ktstr/workload/spawn/
mod.rs

1//! Spawn pipeline: `WorkloadHandle`, `SpawnGuard`, `GroupParams`,
2//! `ThreadWorker`, the report shapes (`WorkerReport`, `WorkerExitInfo`,
3//! `Migration`), and the helpers that thread workers through fork or
4//! `std::thread::spawn`. Split out of `workload/mod.rs` to separate
5//! the production code path from its co-located tests. Tests are
6//! co-located with the production code in topic-grouped sibling
7//! files (`tests_lifecycle`, `tests_grandchild`, `tests_composed`,
8//! ...) that import shared fixtures from `testing.rs` via
9//! `use super::testing::*;`.
10
11use anyhow::{Context, Result};
12use std::collections::{BTreeMap, BTreeSet};
13use std::io::{Read, Write};
14use std::os::unix::io::FromRawFd;
15use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
16use std::time::{Duration, Instant};
17
18use super::affinity::{AffinityIntent, ResolvedAffinity, resolve_affinity, set_thread_affinity};
19use super::config::{CloneMode, MemPolicy, MpolFlags, SchedPolicy, WorkSpec, WorkloadConfig};
20use super::types::*;
21use super::worker::worker_main;
22
23mod cleanup;
24use cleanup::{close_fds_silently, kill_and_killpg};
25
26pub(super) static STOP: AtomicBool = AtomicBool::new(false);
27
28/// A single CPU migration event observed by a worker.
29///
30/// No `Default` impl — a default-constructed Migration would be
31/// `{at_ns: 0, from_cpu: 0, to_cpu: 0}` (a "migration from CPU 0 to
32/// CPU 0 at time 0" is a contradiction: a migration where source ==
33/// dest is not a real migration). Downstream analysis (NUMA migration
34/// counts, scheduler-balance ratios) that assumes `from_cpu !=
35/// to_cpu` would misread default values as real migrations. Construct
36/// every Migration explicitly via [`Migration::new`].
37#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
38pub struct Migration {
39    /// Nanoseconds since worker start.
40    pub at_ns: u64,
41    /// CPU before migration.
42    pub from_cpu: usize,
43    /// CPU after migration.
44    pub to_cpu: usize,
45}
46
47impl Migration {
48    /// Const constructor for a Migration event. Pair-symmetric with
49    /// the struct fields; lets `Migration::new(ns, from, to)` replace
50    /// `Migration { at_ns: ns, from_cpu: from, to_cpu: to }` in
51    /// hand-built test fixtures.
52    pub const fn new(at_ns: u64, from_cpu: usize, to_cpu: usize) -> Self {
53        Self {
54            at_ns,
55            from_cpu,
56            to_cpu,
57        }
58    }
59}
60
61/// Build a nodemask bitmask and maxnode value for `set_mempolicy(2)`
62/// and `mbind(2)`.
63///
64/// Returns `(nodemask_vec, maxnode)`. The nodemask is a bitmask of
65/// `c_ulong` words where bit N corresponds to NUMA node N. `maxnode`
66/// must be `max_node + 2` because the kernel's `get_nodes()` does
67/// `--maxnode` before reading the bitmask.
68pub fn build_nodemask(nodes: &BTreeSet<usize>) -> (Vec<libc::c_ulong>, libc::c_ulong) {
69    if nodes.is_empty() {
70        return (vec![], 0);
71    }
72    let max_node = nodes.iter().copied().max().unwrap_or(0);
73    let mask_bits = max_node + 2;
74    let bits_per_word = std::mem::size_of::<libc::c_ulong>() * 8;
75    let mask_words = mask_bits.div_ceil(bits_per_word);
76    let mut nodemask = vec![0 as libc::c_ulong; mask_words];
77    for &node in nodes {
78        nodemask[node / bits_per_word] |= 1 << (node % bits_per_word);
79    }
80    (nodemask, mask_bits as libc::c_ulong)
81}
82
83const MPOL_PREFERRED_MANY: i32 = 5;
84const MPOL_WEIGHTED_INTERLEAVE: i32 = 6;
85
86/// Worker-side `futex_wait` timeout for STOP-signal polling across
87/// every blocking workload primitive (FutexPingPong, FutexFanOut,
88/// FanOutCompute, MutexContention). Workers block inside the
89/// per-variant futex with this timespec; on wake (or timeout) they
90/// re-check [`STOP`] and either continue working or exit cleanly.
91/// At 100ms the worst-case shutdown latency a `stop_and_collect`
92/// caller must budget for is ~100ms above the flush/IO cost; see
93/// [`WorkloadHandle::stop_and_collect`]'s "Shutdown latency"
94/// paragraph for the caller-facing contract.
95pub(super) const WORKER_STOP_POLL_NS: libc::c_long = 100_000_000;
96
97/// Packaged [`libc::timespec`] for every worker-side `futex_wait`
98/// across the blocking workload primitives. Duplicating the struct
99/// literal per call site drifted the `tv_nsec` field between variants
100/// during earlier edits; a single const keeps the shutdown-latency
101/// budget documented on [`WORKER_STOP_POLL_NS`] authoritative.
102pub(super) const FUTEX_WAIT_TIMEOUT: libc::timespec = libc::timespec {
103    tv_sec: 0,
104    tv_nsec: WORKER_STOP_POLL_NS,
105};
106
107/// Post-wake spin count used by the fan-out messenger variants
108/// ([`WorkType::FutexFanOut`] and [`WorkType::FanOutCompute`]) AFTER
109/// each broadcast wake. Gives receivers a short uncontended window
110/// to run to their reservoir-push before the next wake cycle
111/// arrives. Threaded through `spin_burst` rather than a raw
112/// `std::hint::spin_loop` so the messenger also contributes to
113/// `work_units` — matching FanOutCompute's existing pattern so
114/// both variants' messengers report comparable throughput to
115/// downstream assertions.
116pub(super) const FAN_OUT_POST_WAKE_SPIN_ITERS: u64 = 256;
117
118/// Outcome of [`apply_mempolicy_with_flags`].
119///
120/// Makes each branch of the apply path observable to tests and any
121/// future caller that wants to react to the result (the production
122/// worker-setup caller in `worker::worker_main` discards it — applying
123/// the policy is best-effort there, matching the `setpriority` /
124/// `set_sched_policy` soft-fail idiom). The variants partition the
125/// function's exits exactly:
126///
127/// - [`SkippedDefault`](MempolicyOutcome::SkippedDefault): `policy ==
128///   MemPolicy::Default` — no syscall (inherit the parent's policy).
129/// - [`SkippedEmpty`](MempolicyOutcome::SkippedEmpty): a
130///   node-set-bearing variant (`Bind` / `Interleave` /
131///   `PreferredMany` / `WeightedInterleave`) carried an empty set — no
132///   `set_mempolicy(2)` is issued. Issuing one with an empty mask
133///   would be a different (and wrong) request to the kernel, so the
134///   skip is the contract, not merely "did not crash".
135/// - [`Applied`](MempolicyOutcome::Applied): `set_mempolicy(2)`
136///   returned 0.
137/// - [`Failed`](MempolicyOutcome::Failed): `set_mempolicy(2)` returned
138///   non-zero; the errno is logged to stderr and carried here.
139#[derive(Debug, Clone, Copy, PartialEq, Eq)]
140pub(super) enum MempolicyOutcome {
141    /// `MemPolicy::Default` — no syscall.
142    SkippedDefault,
143    /// A node-set variant with an empty set — no syscall.
144    SkippedEmpty,
145    /// `set_mempolicy(2)` succeeded.
146    Applied,
147    /// `set_mempolicy(2)` failed; carries the errno.
148    Failed(i32),
149}
150
151/// Call `set_mempolicy(2)` for the current process with mode flags.
152///
153/// No-op for `MemPolicy::Default` (returns
154/// [`MempolicyOutcome::SkippedDefault`]) and for a node-set variant
155/// with an empty set (returns [`MempolicyOutcome::SkippedEmpty`]).
156/// Logs a warning on syscall failure and returns
157/// [`MempolicyOutcome::Failed`]; returns [`MempolicyOutcome::Applied`]
158/// on success.
159pub(super) fn apply_mempolicy_with_flags(policy: &MemPolicy, flags: MpolFlags) -> MempolicyOutcome {
160    let (mode, node_set): (i32, BTreeSet<usize>) = match policy {
161        MemPolicy::Default => return MempolicyOutcome::SkippedDefault,
162        MemPolicy::Bind(nodes) => (libc::MPOL_BIND, nodes.clone()),
163        MemPolicy::Preferred(node) => (libc::MPOL_PREFERRED, [*node].into_iter().collect()),
164        MemPolicy::Interleave(nodes) => (libc::MPOL_INTERLEAVE, nodes.clone()),
165        MemPolicy::PreferredMany(nodes) => (MPOL_PREFERRED_MANY, nodes.clone()),
166        MemPolicy::WeightedInterleave(nodes) => (MPOL_WEIGHTED_INTERLEAVE, nodes.clone()),
167        MemPolicy::Local => {
168            let rc = unsafe {
169                libc::syscall(
170                    libc::SYS_set_mempolicy,
171                    libc::MPOL_LOCAL | flags.bits() as i32,
172                    std::ptr::null::<libc::c_ulong>(),
173                    0 as libc::c_ulong,
174                )
175            };
176            if rc != 0 {
177                let err = std::io::Error::last_os_error();
178                eprintln!("ktstr: set_mempolicy(MPOL_LOCAL) failed: {err}");
179                return MempolicyOutcome::Failed(err.raw_os_error().unwrap_or(0));
180            }
181            return MempolicyOutcome::Applied;
182        }
183    };
184    if node_set.is_empty() {
185        eprintln!("ktstr: set_mempolicy: empty node set, skipping");
186        return MempolicyOutcome::SkippedEmpty;
187    }
188    let (mask, maxnode) = build_nodemask(&node_set);
189    let effective_mode = mode | flags.bits() as i32;
190    let rc = unsafe {
191        libc::syscall(
192            libc::SYS_set_mempolicy,
193            effective_mode,
194            mask.as_ptr(),
195            maxnode,
196        )
197    };
198    if rc != 0 {
199        let err = std::io::Error::last_os_error();
200        eprintln!("ktstr: set_mempolicy(mode={mode}, nodes={node_set:?}) failed: {err}");
201        return MempolicyOutcome::Failed(err.raw_os_error().unwrap_or(0));
202    }
203    MempolicyOutcome::Applied
204}
205
206/// Terminate the calling forked-child worker with success status (code 0).
207///
208/// Uses the `_exit(2)` syscall via `libc::_exit` directly rather than [`std::process::exit`] or
209/// returning from `main` so that:
210/// 1. Rust drop glue does NOT run on inherited parent state (open file
211///    descriptors, mmaps, atexit handlers, thread-local destructors).
212///    Forked children share address space with the parent at fork
213///    time; running destructors on the child side would also tear down
214///    parent-owned resources via the shared FD table / OS handles.
215/// 2. `exit_group(2)` semantics (which `_exit` invokes) tear down only
216///    the calling tgid. Forked children have their own tgid distinct
217///    from the test-runner's, so the parent test runner survives.
218///
219/// Code 0 signals to the parent's collect_results path that the worker
220/// completed the work-loop normally — the WorkerReport (if any was
221/// written) is valid and should be merged into the test verdict.
222#[inline]
223fn exit_child_success() -> ! {
224    // SAFETY: `libc::_exit` is `unsafe` because it bypasses Rust drop
225    // execution + C stdlib atexit. That bypass IS the contract here
226    // for forked children: running destructors on shared inherited
227    // state corrupts parent-owned resources. exit_group(2) tear-down
228    // of the child's tgid is sound — the parent is in a different
229    // tgid.
230    unsafe { libc::_exit(0) }
231}
232
233/// Terminate the calling forked-child worker with error status (code 1).
234///
235/// Same `_exit`-vs-Rust-exit rationale as [`exit_child_success`]. Code 1
236/// signals to the parent's collect_results path that the worker failed
237/// before completing the work-loop normally — the parent treats the
238/// WorkerReport as missing or invalid and surfaces the failure to the
239/// test verdict.
240///
241/// Use this on any error path inside the forked-child closure: poll
242/// timeout, syscall failure, panic via `catch_unwind`, write-report
243/// failure, orphan detection.
244#[inline]
245fn exit_child_error() -> ! {
246    // SAFETY: see [`exit_child_success`] — identical contract.
247    unsafe { libc::_exit(1) }
248}
249
250/// Apply `nice` to the calling worker via `setpriority(2)`.
251///
252/// Always invokes `setpriority(PRIO_PROCESS, 0, nice)` — including
253/// for `nice == 0`, which writes 0 explicitly rather than
254/// inheriting. The "skip the syscall, inherit the parent's nice"
255/// state lives one layer up: callers gate the call on
256/// [`WorkSpec::nice`](crate::workload::WorkSpec::nice) /
257/// [`WorkloadConfig::nice`](crate::workload::WorkloadConfig::nice)
258/// being `Some(_)` and pass through the inner value (so `Some(0)`
259/// becomes a real `setpriority(0)` and `None` skips the call
260/// entirely). The kernel clamps `niceval` to
261/// `[MIN_NICE, MAX_NICE]` (-20..19) inside `setpriority`, so any
262/// out-of-range input is normalised by the syscall itself rather
263/// than rejected.
264///
265/// Failures are logged once via stderr and do not abort the
266/// worker — matches the [`apply_mempolicy_with_flags`] /
267/// [`set_thread_affinity`] / `set_sched_policy` error idiom in
268/// `worker_main`. The expected failure mode is `EACCES` from
269/// `set_one_prio` → `can_nice` when an unprivileged worker tries
270/// to lower nice (negative niceval) without `CAP_SYS_NICE`.
271pub(super) fn apply_nice(nice: i32) {
272    let rc = unsafe { libc::setpriority(libc::PRIO_PROCESS, 0, nice) };
273    if rc != 0 {
274        warn_setpriority_failed_once();
275    }
276}
277
278/// Print a single `setpriority` failure warning for the lifetime
279/// of the process. Same rationale as
280/// `warn_schedstat_unavailable_once`: dozens of workers will fail
281/// once each on an unprivileged host that requested negative nice,
282/// and a per-worker line floods the test log.
283pub(super) fn warn_setpriority_failed_once() {
284    static WARNED: std::sync::Once = std::sync::Once::new();
285    WARNED.call_once(|| {
286        let errno = std::io::Error::last_os_error();
287        eprintln!(
288            "workload: setpriority(PRIO_PROCESS) failed: {errno}; nice value not applied (CAP_SYS_NICE may be required for negative nice)"
289        );
290    });
291}
292
293/// Telemetry collected from a worker process after it stops.
294///
295/// Normal reports: each field is populated by the worker itself
296/// (inside the VM) and serialized via a pipe to the parent process.
297/// Sentinel reports: sentinel reports synthesized by
298/// [`WorkloadHandle::stop_and_collect`] on worker-exit carry
299/// parent-populated `exit_info` with the remaining fields at their
300/// [`Default`] values (the worker never emitted on the pipe, so
301/// the parent is the sole source of truth for the surfaced
302/// outcome).
303///
304/// # Default trade-off
305///
306/// [`Default`] produces a zero/empty report. The trade-off:
307///
308/// - **Pro:** sentinel/test code can spread `..WorkerReport::default()`
309///   so adding a field does not require touching every sentinel site.
310/// - **Con:** zero-valued fields are valid report outputs (e.g. a
311///   worker that never blocked has `wake_latencies_ns: vec![]`), so
312///   a missing field cannot be distinguished from a real-zero field at
313///   the reader. Consumers that need "was this field actually set"
314///   must track presence out-of-band (e.g. whether the work type
315///   populates the field per `wake_latencies_ns`'s doc).
316///
317/// Decision: keep the `Default` impl. Sentinel ergonomics outweigh
318/// the distinguishability cost — every real consumer already knows
319/// which fields a given `WorkType` populates, and the alternative
320/// (removing `Default` and hand-listing every field at sentinel
321/// sites) introduces a worse drift problem that silently skips new
322/// telemetry instead of reporting it as zero.
323///
324/// Callers building a sentinel report should spread
325/// `..WorkerReport::default()` rather than listing every field by hand
326/// -- the sentinel drifts silently when a field is added.
327#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize, crate::Claim)]
328pub struct WorkerReport {
329    /// Kernel TID from `gettid(2)`. For [`CloneMode::Fork`] each
330    /// worker is its own thread-group leader so `gettid() == getpid()
331    /// == tgid`; the report's tid is interchangeable with the
332    /// worker's pid in libc / cgroup APIs. For [`CloneMode::Thread`]
333    /// every worker shares the parent's tgid and `gettid()` is the
334    /// only identifier that discriminates per-task identity, so the
335    /// report's tid is what feeds `sched_setaffinity(tid, ...)` and
336    /// `cgroup.threads` writes (NOT `cgroup.procs` — see the warning
337    /// on [`WorkloadHandle::worker_pids`]). Stored as `pid_t` (i32)
338    /// to match the kernel's native type and avoid the silent
339    /// u32→i32 sign-cast wraparound at libc boundaries
340    /// (kill/waitpid/Pid::from_raw).
341    pub tid: i32,
342    /// Cumulative work iterations (incremented by `spin_burst` or I/O loops).
343    /// Read by the fairness/no-progress gate (`assert_not_stuck` /
344    /// `min_work_units`) and `assert_throughput_parity`; NOT summed into
345    /// `CgroupStats::total_iterations`, which reads [`iterations`](Self::iterations).
346    /// A `Custom` worker that wants throughput assertions must also populate
347    /// [`iterations`](Self::iterations).
348    pub work_units: u64,
349    /// Thread CPU time from `CLOCK_THREAD_CPUTIME_ID` (ns).
350    pub cpu_time_ns: u64,
351    /// Wall-clock time from worker-start to stop flag (ns).
352    /// Measured from the worker's first `Instant::now()` in
353    /// `worker_main` (immediately after the start handshake) to the
354    /// outer-loop exit (when the per-worker `stop` flag is observed
355    /// `true`); covers both Fork-mode workers (signal-driven flag)
356    /// and Thread-mode workers (parent-driven flag).
357    pub wall_time_ns: u64,
358    /// `wall_time_ns - cpu_time_ns`: total off-CPU time (ns).
359    ///
360    /// Includes all time the worker was not executing on a CPU: runnable
361    /// queue wait, voluntary sleep, I/O wait, futex wait, etc.
362    pub off_cpu_ns: u64,
363    /// Number of observed CPU migrations (checked every 1024 work units).
364    pub migration_count: u64,
365    /// Set of all CPUs this worker ran on.
366    pub cpus_used: BTreeSet<usize>,
367    /// Ordered list of CPU migration events with timestamps.
368    pub migrations: Vec<Migration>,
369    /// Longest worker CPU-TIME gap observed between 1024-work-unit
370    /// checkpoints (ms) — the `CLOCK_THREAD_CPUTIME_ID` delta the worker
371    /// consumed without crossing a progress checkpoint. This is the Stuck
372    /// gate's measure (`max_gap_ms` thresholds compare against it): a high
373    /// value means compute-without-progress (livelock, a pathologically
374    /// expensive iteration), which is real at ANY host dilation — CPU time
375    /// excludes host preemption and guest run-queue wait by construction.
376    /// A BLOCKED or starved-runnable worker accrues no CPU here; detecting
377    /// blocked-stuck is the cell watchdog's domain. Granularity: multiples
378    /// of the checkpoint interval (1024 work units, plus one extra sample
379    /// at each phase transition).
380    pub max_gap_ms: u64,
381    /// Longest WALL-CLOCK gap between checkpoints (ms) — the historical
382    /// pre-CPU-conversion gate measure, kept verbatim (including its
383    /// blocking-site baseline resets) as EVIDENCE: under host dilation or
384    /// guest starvation wall-without-CPU shows up here while
385    /// [`max_gap_ms`](Self::max_gap_ms) stays low. Independent maximum —
386    /// its peak may come from a different checkpoint than the CPU peak.
387    /// `#[serde(default)]`: sidecar JSON written before this field existed
388    /// reads 0 (guest→host postcard is same-binary, always in sync).
389    #[serde(default)]
390    pub max_gap_wall_ms: u64,
391    /// CPU where the longest (CPU-time) gap happened.
392    pub max_gap_cpu: usize,
393    /// When the longest (CPU-time) gap happened (wall ms from start).
394    pub max_gap_at_ms: u64,
395    /// Per-wakeup latency samples (ns). Measures off-CPU time
396    /// between the call that blocks (any blocking primitive — pipe
397    /// `read`, futex wait, `poll`, `sched_yield`, `nanosleep`, etc.)
398    /// and the wakeup that resumes execution; not a yield-specific
399    /// measure.
400    /// Populated for blocking work types: Bursty, PipeIo, FutexPingPong,
401    /// FutexFanOut, FanOutCompute, CacheYield, CachePipe, IoSyncWrite,
402    /// IoRandRead, IoConvoy, NiceSweep,
403    /// AffinityChurn, PolicyChurn, MutexContention, ForkExit (parent's
404    /// waitpid wait), Sequence with Sleep/Yield/Io phases.
405    ///
406    /// Distinct from [`iteration_costs_ns`](Self::iteration_costs_ns):
407    /// this field measures the OFF-CPU gap between blocks (scheduler
408    /// wake latency); `iteration_costs_ns` measures the wall-clock
409    /// duration of a single compute iteration. The three pure-compute
410    /// variants that populate `iteration_costs_ns` —
411    /// [`WorkType::AluHot`], [`WorkType::SmtSiblingSpin`], and
412    /// [`WorkType::IpcVariance`] — never block and report
413    /// `wake_latencies_ns: vec![]`. Other compute variants
414    /// (e.g. SpinWait, YieldHeavy, Mixed) populate neither
415    /// reservoir.
416    pub wake_latencies_ns: Vec<u64>,
417    /// Total number of wake-latency observations the worker
418    /// recorded, INCLUDING any that were dropped by the reservoir
419    /// sampler. `wake_latencies_ns` is reservoir-clamped to at
420    /// most `MAX_WAKE_SAMPLES` (100_000) entries; on a long run
421    /// that accumulates more than that many wake events, the
422    /// vector stays at its cap while this counter keeps climbing.
423    /// Host-side consumers that want to report "total wakeups
424    /// observed" (vs. "entries in the sample") read this field;
425    /// percentile / CV computations read `wake_latencies_ns`.
426    pub wake_sample_total: u64,
427    /// Per-iteration wall-clock duration of one compute iteration (ns),
428    /// including any scheduler preemption. Measured via
429    /// `Instant::now()` (CLOCK_MONOTONIC), so a sample includes any
430    /// off-CPU time the kernel inserted mid-iteration. The variance
431    /// across iterations is the load-bearing scheduler signal —
432    /// preemption inflates samples and that inflation is the
433    /// observable.
434    ///
435    /// Reservoir-sampled at the same cap (`MAX_WAKE_SAMPLES` =
436    /// 100_000) as [`wake_latencies_ns`](Self::wake_latencies_ns),
437    /// using the same Algorithm-R sampler.
438    ///
439    /// Populated for pure compute work types where the worker
440    /// never blocks: [`WorkType::AluHot`], [`WorkType::SmtSiblingSpin`],
441    /// and [`WorkType::IpcVariance`]. Each sample is the elapsed
442    /// time from the start to the end of one outer-loop iteration's
443    /// compute burst.
444    ///
445    /// Distinct from [`wake_latencies_ns`](Self::wake_latencies_ns):
446    /// the wake-latency reservoir captures off-CPU time (futex /
447    /// pipe / nanosleep wakeups); this reservoir captures the
448    /// wall-clock duration of one compute iteration (which
449    /// includes any scheduler preemption inside the iteration).
450    /// The two are NOT comparable across variants — a
451    /// scheduler-A/B test that wants iteration cost for a compute
452    /// variant reads this field; a test that wants wake latency
453    /// for a blocking variant reads `wake_latencies_ns`.
454    pub iteration_costs_ns: Vec<u64>,
455    /// Total number of iteration-cost observations the worker
456    /// recorded, INCLUDING any that were dropped by the reservoir
457    /// sampler. Mirrors [`wake_sample_total`](Self::wake_sample_total)
458    /// but for [`iteration_costs_ns`](Self::iteration_costs_ns):
459    /// host-side consumers that want "total compute iterations
460    /// observed" read this field; distribution computations read
461    /// `iteration_costs_ns` directly.
462    pub iteration_cost_sample_total: u64,
463    /// Per-timer-cycle latency samples (ns) for
464    /// [`crate::workload::WorkType::TimerLatency`]: the observed
465    /// `clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME)` wake time minus the
466    /// absolute deadline, floored at 0. Reservoir-clamped to at most
467    /// `MAX_WAKE_SAMPLES`, distinct from `wake_latencies_ns` so cyclictest-style
468    /// timer latency does not blur with the blocking variants' wake latency.
469    /// `vec![]` for every non-`TimerLatency` variant.
470    pub timer_latencies_ns: Vec<u64>,
471    /// Total timer-cycle observations, INCLUDING any the reservoir dropped —
472    /// the true population for unbiased cross-phase weighting. Mirrors
473    /// [`wake_sample_total`](Self::wake_sample_total) for `timer_latencies_ns`.
474    pub timer_sample_total: u64,
475    /// Outer-loop iteration count. What `CgroupStats::total_iterations` sums
476    /// and what the derived `iteration_rate`, `iterations_per_worker`, and
477    /// `migration_ratio` divide by; NOT read by
478    /// the zero-work-units gate, which reads [`work_units`](Self::work_units). A
479    /// `Custom` worker that wants the starvation / `min_work_units` gate
480    /// honored must also populate [`work_units`](Self::work_units).
481    pub iterations: u64,
482    /// Delta of /proc/self/schedstat field 2 (run_delay) over the work loop.
483    pub schedstat_run_delay_ns: u64,
484    /// Delta of /proc/self/schedstat field 3 (pcount — number of
485    /// times the task was scheduled in over the work loop). This is
486    /// NOT a context-switch count; `/proc/<pid>/status`'s
487    /// `voluntary_ctxt_switches` / `nonvoluntary_ctxt_switches` are
488    /// the true context-switch counters and are not read here.
489    pub schedstat_run_count: u64,
490    /// Delta of /proc/self/schedstat field 1 (cpu_time) over the work loop.
491    pub schedstat_cpu_time_ns: u64,
492    /// `true` when the worker reached its natural end — either the
493    /// outer work loop observed STOP and exited cleanly, or a
494    /// custom-closure payload returned from its `run` function. A
495    /// sentinel report synthesised by
496    /// [`WorkloadHandle::stop_and_collect`]'s decode-failure
497    /// fallback (see `exit_info` below) carries `false`. Lets downstream
498    /// consumers distinguish "worker ran to completion and
499    /// observed zero iterations" (`completed: true, iterations: 0`
500    /// — legitimate for pathologically short test windows) from
501    /// "worker died / timed out before recording anything"
502    /// (`completed: false, iterations: 0` — the sentinel shape).
503    pub completed: bool,
504    /// Per-NUMA-node page counts from `/proc/self/numa_maps` after workload.
505    /// Keyed by node ID. Empty when numa_maps is unavailable. numa_maps reports
506    /// the per-node RESIDENT pages of the calling task's mm. For
507    /// [`CloneMode::Fork`] workers (the scenario-engine default) each worker has
508    /// a disjoint mm, so SUMming across workers is the true cgroup page total;
509    /// for [`CloneMode::Thread`] siblings share one mm and each reports the SAME
510    /// residency, so a SUM counts shared pages once per thread. Consumers
511    /// (cgroup_stats / phase_cgroup_stats) SUM
512    /// this — correct for the Fork default; the Thread-mode caveat is inherited
513    /// identically by both reducers (no per-phase divergence).
514    pub numa_pages: BTreeMap<usize, u64>,
515    /// Delta of `/proc/vmstat` `numa_pages_migrated` over the work loop. This is
516    /// the SYSTEM-WIDE vmstat `NUMA_PAGE_MIGRATE` vm_event (summed across all
517    /// CPUs), NOT the per-task `task_struct` field of the same name surfaced in
518    /// `/proc/PID/sched`. Because every worker reads the same system-wide
519    /// counter, consumers fold it as MAX across workers (a SUM would inflate it
520    /// by the worker count) — see `PhaseCgroupStats::cross_node_migrated`. The vm_event is bumped only by
521    /// NUMA balancing (the source of NUMA page migrations), so the delta is 0 on
522    /// kernels/configs without it (a measurement-availability caveat, not a wrong
523    /// value).
524    pub vmstat_numa_pages_migrated: u64,
525    /// Diagnostic attached only to sentinel reports — populated when
526    /// `stop_and_collect` synthesized the entry because no (or
527    /// unparseable) postcard payload came back on the report pipe.
528    /// `None` on every real worker-produced report. Lets operators
529    /// distinguish the four failure shapes that all collapse to
530    /// "empty pipe + no report":
531    ///
532    /// - [`WorkerExitInfo::Exited`] with a non-zero code: worker
533    ///   reached `_exit(code)` without writing the report —
534    ///   typically the `catch_unwind` Err arm in the worker-child
535    ///   closure (panic under `panic = "unwind"`) or the 30s
536    ///   poll-start timeout's early `_exit(1)`.
537    /// - [`WorkerExitInfo::Signaled`]: worker was killed — SIGABRT
538    ///   under `panic = "abort"`, SIGKILL from the still-alive
539    ///   escalation in `stop_and_collect`, or an external signal
540    ///   (OOM killer, operator SIGKILL).
541    /// - [`WorkerExitInfo::TimedOut`]: worker never exited within the
542    ///   5s collection deadline and the WNOHANG reap observed
543    ///   `StillAlive` — escalated via SIGKILL + `waitpid(None)`.
544    /// - [`WorkerExitInfo::WaitFailed`]: `waitpid` itself returned an
545    ///   error (ECHILD / EINTR). Typically a plumbing bug — the child
546    ///   was reaped by an external signal handler, a double-reap
547    ///   regression, or the pid was recycled.
548    ///
549    /// No `skip_serializing_if`: postcard is a positional, schemaless
550    /// format — every Serialize call must emit every field in the
551    /// same order or the decoder reads the next field's bytes off
552    /// the wire (silent data corruption). The Option<…> tag itself
553    /// (one byte) is the only overhead on the live-worker path.
554    pub exit_info: Option<WorkerExitInfo>,
555    /// `true` when this worker served as the messenger for a
556    /// wake-fanout work type ([`WorkType::FutexFanOut`] or
557    /// [`WorkType::FanOutCompute`]) — the single writer that
558    /// advances the shared generation and issues `futex_wake` for
559    /// its group. `false` for receivers and for every non-fanout
560    /// work type.
561    ///
562    /// Populated from the `is_messenger` flag on the
563    /// `futex: Option<(*mut u32, bool)>` parameter threaded into
564    /// `worker_main`. A sentinel report synthesized by the
565    /// decode-failure fallback in
566    /// [`WorkloadHandle::stop_and_collect`] carries `false` via
567    /// [`Default`], matching its `completed: false` shape.
568    ///
569    /// Enables per-worker latency-participation assertions in
570    /// tests — a receiver worker produces `wake_latencies_ns`
571    /// entries while its messenger pair records wake-side work but
572    /// no wake latency. Without this field, tests had to
573    /// cross-reference per-group indexing or guess from the empty
574    /// vector — ambiguous on groups where the messenger legitimately
575    /// exits before producing a report.
576    pub is_messenger: bool,
577    /// Index of the worker group this report belongs to.
578    ///
579    /// `0` denotes the primary group described by
580    /// [`WorkloadConfig`]'s top-level `work_type` / `num_workers` /
581    /// `affinity` / `sched_policy` fields. `1..=N` denotes
582    /// composed groups in the order they appear in
583    /// [`WorkloadConfig::composed`]. Reports collected by
584    /// [`WorkloadHandle::stop_and_collect`] are tagged with the
585    /// `group_idx` of the spawning [`WorkSpec`] (or `0` for the
586    /// primary), so per-group filtering in test assertions can
587    /// cleanly partition the vector.
588    ///
589    /// Sentinel reports (synthesized on missing or undecodable
590    /// payload / panic / timeout) carry the `group_idx` of the
591    /// worker whose pid the sentinel replaces, so a "this composed
592    /// group failed" assertion still works on an outright crash.
593    pub group_idx: usize,
594    /// Rendered error from the worker's `set_thread_affinity`
595    /// call, or `None` when affinity setup succeeded (or the
596    /// worker had no affinity to apply). Populated by
597    /// `worker_main` when the pre-loop
598    /// `set_thread_affinity(tid, cpus)` returns `Err` — the
599    /// worker continues with the inherited (or kernel-default)
600    /// cpumask so the test still produces an observable outcome,
601    /// but the failure is now surfaced in the report instead of
602    /// being silently dropped via `let _ = …`. The expected
603    /// failure shape is EINVAL from a requested cpu that is
604    /// outside the cpuset cgroup's `cpus.allowed` mask or the
605    /// kernel's online mask; EPERM is reachable when a more
606    /// privileged tracer set the worker's cpus_allowed and a
607    /// container policy denies further widening. Sentinel
608    /// reports synthesised by
609    /// [`WorkloadHandle::stop_and_collect`] leave this field at
610    /// its default `None` — a worker that died before
611    /// `worker_main` ran has no affinity-error observation.
612    ///
613    /// No `skip_serializing_if`: postcard is positional and
614    /// schemaless, so every Serialize call must emit every field
615    /// in the same order — skipping a field shifts the decoder
616    /// onto the next field's bytes (silent corruption). The
617    /// Option<…> tag (one byte) is the only overhead on the
618    /// success path.
619    pub affinity_error: Option<String>,
620    /// `set_sched_policy` failure text (`{e:#}`), or `None` when the
621    /// per-worker scheduling-policy set succeeded (or the policy was the
622    /// `Normal` no-op). Load-bearing for the verifier dispatch probe: a
623    /// probe worker configured `SchedPolicy::Ext` whose
624    /// `sched_setattr(SCHED_EXT)` was rejected (e.g. the scheduler set
625    /// `scx.disallow` on it) stays SCHED_OTHER, so its `iterations`
626    /// progress does NOT prove the BPF scheduler dispatched it —
627    /// `run_and_confirm_dispatch` excludes any worker with a
628    /// `sched_policy_error` from the dispatch proof. Sentinel reports
629    /// synthesised by [`WorkloadHandle::stop_and_collect`] leave this
630    /// `None` (no policy set was attempted). Same positional-serde
631    /// reasoning as [`Self::affinity_error`]: no `skip_serializing_if`.
632    pub sched_policy_error: Option<String>,
633    /// Per-phase telemetry slices for a backdrop (persistent) worker
634    /// that spanned scenario steps. EMPTY for step-local workers and
635    /// for a backdrop worker that remained in the pre-step baseline or
636    /// inter-step sentinel epoch. A worker drains its open measured
637    /// epoch either when the parent-driven `phase_epoch` changes or at
638    /// shutdown, including when a delayed worker first runs after the
639    /// parent has already published a measured epoch. Each slice carries
640    /// the per-phase subset of the whole-run telemetry above, scoped to
641    /// one phase's hold window; the host expands these into per-epoch
642    /// `PhaseBucket` entries — the per-phase attribution a backdrop
643    /// worker otherwise lacks, since it is collected once with
644    /// `step_index = None`.
645    ///
646    /// Appended LAST so the positional postcard decode order of every
647    /// prior field is unchanged. `#[claim(skip)]`: there is no
648    /// test-author claim surface for the raw wire slices — assertions
649    /// run against the host-expanded `PhaseCgroupStats` carriers, which
650    /// carry their own `#[derive(Claim)]`.
651    #[claim(skip)]
652    pub phase_slices: Vec<PhaseSlice>,
653    /// Whole-run taobench COUNTER aggregate — `Some` only for a Taobench worker,
654    /// `None` otherwise. Shipped so the host can derive run-level qps/hit Rate keys
655    /// (`taobench_*_ops_per_cpu_sec_whole` / `taobench_hit_fraction` /
656    /// `taobench_command_hit_rate`) for `--noise-adjust` spread analysis. The
657    /// per-phase `PhaseSlice::taobench` carriers feed the per-phase metrics + the
658    /// serve-latency distribution; this whole-run carrier holds COUNTERS only
659    /// ([`TaobenchStats`](crate::workload::taobench::run::TaobenchStats)) — the
660    /// serve histogram is per-phase data, and the per-phase `elapsed_ns` is
661    /// MAX-merged across concurrent threads so summing phase windows is the wrong
662    /// qps denominator, hence the engine's authoritative whole-run counter
663    /// aggregate is shipped directly. Appended LAST (after `phase_slices`) to keep
664    /// the positional postcard decode order of every prior field unchanged. `pub`
665    /// (every `WorkerReport` field is `pub`, so external custom workers can
666    /// struct-literal construct it via [`crate::workload::WorkType::custom`]; a
667    /// non-taobench worker sets `None`). `#[claim(skip)]`: framework wire plumbing,
668    /// not a test-author claim surface; assertions run against the host-derived
669    /// run-level `taobench_*` Rate metrics.
670    #[claim(skip)]
671    pub taobench_whole: Option<crate::workload::taobench::run::TaobenchStats>,
672}
673
674/// Per-phase telemetry for one backdrop worker over one scenario
675/// step's HOLD window `[StepStart(k), StepEnd(k))`. A backdrop worker
676/// spans every phase, so it accumulates a fresh slice between each
677/// parent-driven `phase_epoch` transition and finalizes it when the
678/// epoch moves (drain-on-change). Carries the per-phase subset of
679/// [`WorkerReport`]'s whole-run telemetry that has a host-side
680/// per-cgroup carrier (`PhaseCgroupStats`), so the host can pool slices
681/// across workers into per-epoch `PhaseBucket`s. Counter fields are
682/// per-phase deltas (`end_snap − start_snap`, re-baselined at each
683/// boundary); `cpus_used` and `numa_pages` are per-phase observations
684/// (gauges). Excludes `iteration_costs_ns` — that reservoir has no
685/// per-cgroup carrier at any level (it feeds only the run-level
686/// benchmark gate).
687#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
688pub struct PhaseSlice {
689    /// The 1-indexed phase this slice covers (scenario Step k → k+1;
690    /// `0` = BASELINE, `u32::MAX` = inter-step gap). The host maps this
691    /// to a `PhaseBucket.step_index`; gap- and baseline-tagged slices
692    /// have no host bucket and are discarded.
693    pub phase_epoch: u32,
694    /// CPUs this worker ran on DURING this phase.
695    pub cpus_used: BTreeSet<usize>,
696    /// Per-wakeup latency samples (ns) recorded during this phase,
697    /// reservoir-clamped to at most `MAX_PHASE_WAKE_SAMPLES` — a
698    /// smaller cap than the whole-run `MAX_WAKE_SAMPLES` so a worker
699    /// spanning many phases stays within the report pipe. The host
700    /// re-caps the merged per-cgroup pool at the larger
701    /// `MAX_WAKE_SAMPLES`.
702    pub wake_latencies_ns: Vec<u64>,
703    /// Total wake observations during this phase, INCLUDING any the
704    /// reservoir dropped — the true population for unbiased
705    /// cross-phase weighting.
706    pub wake_sample_total: u64,
707    /// Per-timer-cycle latency samples (ns) recorded during this phase by a
708    /// [`crate::workload::WorkType::TimerLatency`] worker, reservoir-clamped to
709    /// `MAX_PHASE_WAKE_SAMPLES` (like `wake_latencies_ns`). Distinct carrier so
710    /// timer latency does not blur with wake latency.
711    pub timer_latencies_ns: Vec<u64>,
712    /// Total timer-cycle observations during this phase, INCLUDING any the
713    /// reservoir dropped — the true population for unbiased cross-phase
714    /// weighting. Mirrors `wake_sample_total` for `timer_latencies_ns`.
715    pub timer_sample_total: u64,
716    /// `/proc/self/schedstat` run_delay (field 2) delta over this phase (ns).
717    pub run_delay_ns: u64,
718    /// Off-CPU time during this phase (ns): `wall_ns − cpu_time` over
719    /// the phase. Paired with `wall_ns` so the host derives the
720    /// off-CPU fraction and skips the not-measured `wall_ns == 0` case
721    /// rather than dividing by zero.
722    pub off_cpu_ns: u64,
723    /// Wall-clock duration of this phase as the worker observed it
724    /// (ns) — the denominator for the off-CPU fraction. `0` for a
725    /// phase the worker never ran in (not-measured).
726    pub wall_ns: u64,
727    /// CPU migrations observed during this phase.
728    pub migration_count: u64,
729    /// Outer-loop iterations during this phase.
730    pub iterations: u64,
731    /// `/proc/self/schedstat` cpu_time (field 1, sum_exec_runtime)
732    /// delta over this phase (ns).
733    pub schedstat_cpu_time_ns: u64,
734    /// Per-NUMA-node resident page counts observed at this phase's end
735    /// (`/proc/self/numa_maps`). A gauge, not a delta.
736    pub numa_pages: BTreeMap<usize, u64>,
737    /// `/proc/vmstat numa_pages_migrated` delta over this phase
738    /// (system-wide counter; host folds as MAX across workers).
739    pub vmstat_numa_pages_migrated: u64,
740    /// Longest checkpoint-to-checkpoint wall gap during this phase (ms).
741    pub max_gap_ms: u64,
742    /// CPU where this phase's longest gap happened — coupled with
743    /// `max_gap_ms` as an argmax pair (a bare max would desync the gap
744    /// from its CPU).
745    pub max_gap_cpu: usize,
746    /// Per-phase schbench engine metrics, present ONLY for a
747    /// `WorkType::Schbench` backdrop worker (`None` for every other work
748    /// type — the generic drain leaves it `None`). Carries the phase's merged
749    /// wakeup/request histograms + run-delay raw pairs; the host re-pools these
750    /// across workers ([`crate::workload::schbench::run::SchbenchPhaseStats`])
751    /// and derives per-phase percentiles into `PhaseBucket.metrics`.
752    /// `pub(crate)`: an internal carrier (test authors read `PhaseBucket`, not
753    /// `PhaseSlice`) whose element type is `pub(crate)`. Integer-only, so it
754    /// preserves `PhaseSlice`'s `Eq`.
755    pub(crate) schbench: Option<crate::workload::schbench::run::SchbenchPhaseStats>,
756    /// Per-phase taobench engine metrics, present ONLY for a
757    /// `WorkType::Taobench` backdrop worker (`None` for every other work type —
758    /// the generic drain leaves it `None`). Carries the phase's request- and
759    /// response-time op counters plus the open-loop serve-latency histogram
760    /// ([`crate::workload::taobench::run::TaobenchPhaseStats`]); the host derives
761    /// per-phase qps / hit-ratio / serve-latency percentiles into
762    /// `PhaseBucket.metrics`. `pub(crate)`: an internal carrier (test authors read
763    /// `PhaseBucket`, not `PhaseSlice`). `PhaseSlice`'s `Eq` is preserved because
764    /// `PlatStats` is `Eq` (the schbench carrier above already holds histograms),
765    /// not because the field is integer-only.
766    pub(crate) taobench: Option<crate::workload::taobench::run::TaobenchPhaseStats>,
767}
768
769/// Reason a sentinel [`WorkerReport`] was synthesized — attached to
770/// the report's `exit_info` field so operators can triage a missing
771/// or undecodable postcard payload without cross-referencing
772/// parent-side logs.
773///
774/// Invariant: every variant carries the `waitpid`-derived status for
775/// the worker PID as of the end of `stop_and_collect`. Ordered from
776/// most-informative (exit code) to least (plumbing failure).
777/// No `Default` impl — every variant carries observed-outcome state.
778/// A default-constructed value would have to pick one variant (TimedOut
779/// was the obvious candidate) but that risks silent-pass: a test
780/// fixture using `..Default::default()` on a struct containing
781/// `WorkerExitInfo` gets "worker never exited within the deadline,"
782/// which an operator triaging the failure would chase for minutes
783/// before realizing the value came from a missing field. Construct
784/// every WorkerExitInfo explicitly via the variant the test scenario
785/// expects (e.g. `WorkerExitInfo::Exited(0)` for a clean success).
786#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
787pub enum WorkerExitInfo {
788    /// `WIFEXITED=true` with the given exit code. Non-zero under
789    /// `panic = "unwind"` means catch_unwind caught a panic in the
790    /// worker-child closure and `_exit(1)` fired, or the 30s
791    /// parent-ready poll timed out. Zero means the worker ran to
792    /// completion but failed to write / serialize the report — a
793    /// postcard encode or pipe-write failure that didn't panic.
794    Exited(i32),
795    /// `WIFSIGNALED=true` with the given signal number. Under
796    /// `panic = "abort"` a worker panic raises SIGABRT (signal 6);
797    /// other values indicate external kill, OOM killer, or the
798    /// still-alive-escalation SIGKILL (signal 9) from this function.
799    Signaled(i32),
800    /// Worker was still running after the 5s shared collection
801    /// deadline; escalated via SIGKILL + blocking `waitpid`. The
802    /// child's final status is not retained — the reap happened past
803    /// the point where operator diagnostics would differ between a
804    /// clean timeout and a signal storm.
805    TimedOut,
806    /// `waitpid` itself returned `Err` — typically ECHILD (child
807    /// already reaped by an external signal handler or a double-reap
808    /// regression) or EINTR. Message is the rendered `errno` string.
809    WaitFailed(String),
810    /// Thread-mode worker panicked. `JoinHandle::join()` returned
811    /// `Err`; the inner payload is downcast to a `&str` / `String`
812    /// (the canonical `panic!` payload shapes) and recorded here so
813    /// the operator can triage without scraping the test log. This
814    /// variant is exclusive to [`CloneMode::Thread`] — fork workers
815    /// surface panics via `Exited(1)` or `Signaled(SIGABRT)`
816    /// depending on the panic strategy.
817    Panicked(String),
818}
819
820/// Pure mapping from a `waitpid` outcome to the diagnostic
821/// [`WorkerExitInfo`] attached to a sentinel [`WorkerReport`].
822///
823/// Split out of [`WorkloadHandle::stop_and_collect`] so the four
824/// shapes each resolve to a `WorkerExitInfo` variant without pulling
825/// in the full collection loop's state (pipe drain, SIGKILL
826/// escalation, pid lifetime). Pure input → output means the variant
827/// matrix is directly testable without spawning children.
828///
829/// Shape → variant:
830/// - `Ok(Exited(_, code))` → [`WorkerExitInfo::Exited`]
831/// - `Ok(Signaled(_, sig, _))` → [`WorkerExitInfo::Signaled`]
832/// - `Ok(StillAlive)` → [`WorkerExitInfo::TimedOut`]
833/// - `Ok(_ exotic)` → [`WorkerExitInfo::TimedOut`] (Stopped /
834///   PtraceEvent / PtraceSyscall / Continued; not reachable for a
835///   plain forked worker with no ptrace parent, but collapsed rather
836///   than silently dropped so coverage stays exhaustive)
837/// - `Err(errno)` → [`WorkerExitInfo::WaitFailed`]
838pub(super) fn classify_wait_outcome(
839    source: Result<nix::sys::wait::WaitStatus, nix::errno::Errno>,
840) -> WorkerExitInfo {
841    match source {
842        Ok(nix::sys::wait::WaitStatus::Exited(_, code)) => WorkerExitInfo::Exited(code),
843        Ok(nix::sys::wait::WaitStatus::Signaled(_, sig, _)) => WorkerExitInfo::Signaled(sig as i32),
844        Ok(nix::sys::wait::WaitStatus::StillAlive) => WorkerExitInfo::TimedOut,
845        Ok(_) => WorkerExitInfo::TimedOut,
846        Err(e) => WorkerExitInfo::WaitFailed(e.to_string()),
847    }
848}
849
850/// Extract a human-readable panic payload from a
851/// [`std::thread::Result`] `Err` value. The two canonical shapes
852/// are `&'static str` (`panic!("literal")`) and `String`
853/// (`panic!("{x}")` post-formatting); anything else falls back to
854/// a fixed sentinel.
855///
856/// Pure mapping (no IO, no allocation past `String::clone`) so the
857/// stop_and_collect path can call it on every joined-and-panicked
858/// thread without performance cliffs.
859pub(super) fn extract_panic_payload(payload: Box<dyn std::any::Any + Send>) -> String {
860    if let Some(s) = payload.downcast_ref::<&'static str>() {
861        (*s).to_string()
862    } else if let Some(s) = payload.downcast_ref::<String>() {
863        s.clone()
864    } else {
865        "<non-string panic payload>".to_string()
866    }
867}
868
869/// Wall-clock time budget for joining a thread-mode worker after
870/// its per-task `stop` has been flipped. Mirrors the fork-mode
871/// `stop_and_collect` 5s shared deadline so neither dispatch path
872/// can serially exhaust the test runtime by hanging on a single
873/// stuck worker. The 100ms `FUTEX_WAIT_TIMEOUT` inside
874/// `worker_main`'s blocking primitives means a well-behaved worker
875/// observes `stop=true` within 100ms of the parent's flip; the 5s
876/// budget covers IO drain, scheduling delays under contention, and
877/// post-loop cleanup (NUMA stat reads, schedstat snapshots).
878pub(super) const THREAD_JOIN_TIMEOUT: Duration = Duration::from_secs(5);
879
880/// Block until `join` reports finished or `timeout` elapses.
881/// Returns `Some(thread_result)` on successful join, `None` on
882/// timeout.
883///
884/// Implementation: wait on `exit_evt` (the worker's "I'm about to
885/// return" eventfd, bumped from a Drop guard inside the thread
886/// closure) via `epoll_wait` with a `timerfd` for the safety
887/// deadline. A spurious wake (e.g. EINTR or a stale eventfd-counter
888/// drain) loops back into the wait without orphaning the worker —
889/// the timerfd carries the absolute deadline.
890///
891/// Std lacks a native timed-join API; an alternative side-thread
892/// "joiner + channel" pattern would orphan the joiner on timeout
893/// (joining is non-cancellable in std), which keeps the thread
894/// alive past `WorkloadHandle::drop` and prevents process exit.
895/// The eventfd path replaces the previous 10ms sleep-poll loop
896/// without that orphan cost.
897pub(super) fn join_thread_with_timeout(
898    join: std::thread::JoinHandle<WorkerReport>,
899    exit_evt: &vmm_sys_util::eventfd::EventFd,
900    timeout: Duration,
901) -> Option<std::thread::Result<WorkerReport>> {
902    use std::os::unix::io::AsRawFd;
903    use vmm_sys_util::epoll::{ControlOperation, Epoll, EpollEvent, EventSet};
904    use vmm_sys_util::timerfd::TimerFd;
905
906    if join.is_finished() {
907        return Some(join.join());
908    }
909
910    let epoll = match Epoll::new() {
911        Ok(e) => e,
912        Err(e) => {
913            tracing::warn!(%e, "join_thread_with_timeout: epoll_create1 failed");
914            return None;
915        }
916    };
917    if let Err(e) = epoll.ctl(
918        ControlOperation::Add,
919        exit_evt.as_raw_fd(),
920        EpollEvent::new(EventSet::IN, 0),
921    ) {
922        tracing::warn!(%e, "join_thread_with_timeout: add exit_evt to epoll");
923        return None;
924    }
925    let mut timer = match TimerFd::new() {
926        Ok(t) => t,
927        Err(e) => {
928            tracing::warn!(%e, "join_thread_with_timeout: timerfd_create failed");
929            return None;
930        }
931    };
932    if let Err(e) = timer.reset(timeout, None) {
933        tracing::warn!(%e, "join_thread_with_timeout: timerfd_settime failed");
934        return None;
935    }
936    if let Err(e) = epoll.ctl(
937        ControlOperation::Add,
938        timer.as_raw_fd(),
939        EpollEvent::new(EventSet::IN, 1),
940    ) {
941        tracing::warn!(%e, "join_thread_with_timeout: add timerfd to epoll");
942        return None;
943    }
944
945    let deadline = Instant::now() + timeout;
946    let mut events = [EpollEvent::default(); 2];
947    loop {
948        if join.is_finished() {
949            return Some(join.join());
950        }
951        let remaining = deadline.saturating_duration_since(Instant::now());
952        if remaining.is_zero() {
953            return None;
954        }
955        match epoll.wait(-1, &mut events) {
956            Ok(_) => {}
957            Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
958            Err(e) => {
959                tracing::warn!(%e, "join_thread_with_timeout: epoll_wait failed");
960                return None;
961            }
962        }
963    }
964}
965
966/// `worker_main`'s loop-check predicate: returns `true` when the
967/// worker should stop iterating. Reads BOTH the per-worker `stop`
968/// flag and the global [`STOP`] flag; either set request causes
969/// exit.
970///
971/// Why both:
972/// - Per-worker `stop` is what `WorkloadHandle::stop_and_collect`
973///   flips for graceful shutdown. For Fork mode the per-worker
974///   `stop` IS the global [`STOP`] (the SIGUSR1 handler flips it).
975///   For Thread mode each worker has its own `Arc<AtomicBool>`
976///   passed via `&AtomicBool`.
977/// - The global [`STOP`] is what the SIGUSR1 handler sets. For
978///   Fork mode the worker's per-process [`STOP`] is the same
979///   AtomicBool the handler writes. For Thread mode every thread
980///   shares the parent process's address space, so a SIGUSR1
981///   delivered to the parent (e.g. Ctrl-C / a test harness signal)
982///   flips the shared global [`STOP`] but NOT the per-worker
983///   `stop` Arcs. Without this disjunction, Thread workers would
984///   silently keep running through a parent-level shutdown
985///   request.
986///
987/// `#[inline]` because the call site is two atomic loads + an OR.
988/// Relaxed ordering on both reads matches every existing site —
989/// no cross-field happens-before edge to establish.
990#[inline]
991pub(super) fn stop_requested(stop: &AtomicBool) -> bool {
992    stop.load(Ordering::Relaxed) || STOP.load(Ordering::Relaxed)
993}
994
995/// Per-thread worker state for [`CloneMode::Thread`] dispatch.
996///
997/// Thread workers cannot be reaped via `waitpid` (they share a tgid
998/// with the parent), so the lifecycle uses Rust's [`std::thread`]
999/// primitives instead of pid-based syscalls:
1000///
1001/// - `tid` is published by the worker thread post-spawn via
1002///   `gettid()` so the parent can address the kernel task for
1003///   `sched_setaffinity(tid, ...)` and report it from
1004///   [`WorkloadHandle::worker_pids`]. `Arc<AtomicI32>` because the
1005///   thread closure owns the publisher and the parent reads it
1006///   without joining.
1007/// - `stop` replaces the global [`STOP`] signal-flag for thread
1008///   mode: the parent flips it from
1009///   [`WorkloadHandle::stop_and_collect`], the worker observes it
1010///   inside `worker_main`'s `stop.load(Relaxed)` checks. SIGUSR1 is
1011///   process-wide and useless for per-thread stop control.
1012/// - `start_tx` is the rendezvous channel: the parent calls
1013///   `send(())` from [`WorkloadHandle::start`]; the thread blocks
1014///   in `recv()` until then. `Option` so `start` can take it and
1015///   drop it (idempotent re-call is a no-op when `None`).
1016/// - `join` holds the [`std::thread::JoinHandle`] returned by
1017///   `thread::spawn`; `stop_and_collect` joins each handle to
1018///   retrieve the [`WorkerReport`]. `Option` so `stop_and_collect`
1019///   can take ownership and `Drop` does not double-join.
1020pub(super) struct ThreadWorker {
1021    tid: std::sync::Arc<std::sync::atomic::AtomicI32>,
1022    stop: std::sync::Arc<AtomicBool>,
1023    pub(super) start_tx: Option<std::sync::mpsc::SyncSender<()>>,
1024    join: Option<std::thread::JoinHandle<WorkerReport>>,
1025    /// Eventfd bumped by the worker thread's `WorkerExitSignal` Drop
1026    /// guard before the thread returns from its closure. Lets
1027    /// [`join_thread_with_timeout`] block in `epoll_wait` instead of
1028    /// sleep-polling [`std::thread::JoinHandle::is_finished`]. Counter
1029    /// mode (not semaphore) — the value never matters; only the edge
1030    /// from 0 to non-zero does. The Arc is cloned into the closure
1031    /// for the Drop guard; the parent retains the original here.
1032    exit_evt: std::sync::Arc<vmm_sys_util::eventfd::EventFd>,
1033}
1034
1035/// Defense-in-depth Drop for [`ThreadWorker`]. Rust's
1036/// [`std::thread::JoinHandle`] does NOT join its thread on drop —
1037/// it detaches, and the thread continues running until completion.
1038/// `WorkloadHandle::drop`, `WorkloadHandle::stop_and_collect`, and
1039/// `SpawnGuard::drop` already explicitly `take()` the JoinHandle and
1040/// route it through [`join_thread_with_timeout`]; this impl exists
1041/// for the case where some future refactor lets a `ThreadWorker`
1042/// fall out of scope without going through one of those paths.
1043///
1044/// Behavior: if `join` is still `Some` when this Drop fires, flip
1045/// `stop` (so the worker exits cleanly), drop `start_tx` (in case
1046/// the worker is still parked on `recv()`), and join with the
1047/// shared 5s budget. Errors / timeouts are swallowed because Drop
1048/// has nothing to assert against; the upstream paths produce the
1049/// auditable diagnostics.
1050impl Drop for ThreadWorker {
1051    fn drop(&mut self) {
1052        if let Some(j) = self.join.take() {
1053            self.stop.store(true, Ordering::Relaxed);
1054            self.start_tx.take();
1055            let _ = join_thread_with_timeout(j, &self.exit_evt, THREAD_JOIN_TIMEOUT);
1056        }
1057    }
1058}
1059
1060/// Per-fork-child report-decoding shape and bookkeeping recorded
1061/// alongside each entry in [`SpawnGuard::children`] and
1062/// [`WorkloadHandle::children`].
1063///
1064/// Two variants:
1065///
1066/// - [`ForkedChildKind::Worker`]: the conventional fork-mode worker
1067///   (one process per worker, single postcard `WorkerReport`).
1068///   Carries the worker's `group_idx` so a sentinel report on a
1069///   missing payload can be tagged correctly.
1070/// - [`ForkedChildKind::PcommContainer`]: a single thread-group
1071///   leader that owns `num_workers` worker threads across one or
1072///   more logical groups. The leader sets its own `comm` to `pcomm`
1073///   via `prctl(PR_SET_NAME)` before spawning the threads, so every
1074///   worker thread's `task->group_leader->comm` is `pcomm` exactly
1075///   (the [`WorkSpec::pcomm`] builder rejects > 15 bytes —
1076///   `TASK_COMM_LEN - 1` — so the framework never feeds the kernel
1077///   a name `__set_task_comm` would truncate). Reports are a single
1078///   `serde_json` `Vec<WorkerReport>` (one entry per worker thread;
1079///   per-thread `group_idx` lives inside each `WorkerReport`).
1080///
1081/// Note the per-variant wire-format split: `Worker` uses postcard and
1082/// `PcommContainer` uses serde_json. The encodings are dispatched by
1083/// `ForkedChildKind` at decode time on the parent.
1084#[derive(Clone, Debug)]
1085pub(super) enum ForkedChildKind {
1086    /// Conventional fork-mode worker (one process per worker).
1087    /// Report wire format: bare `b'r'` ready byte followed by a
1088    /// `postcard::to_stdvec(&WorkerReport)` document.
1089    Worker { group_idx: usize },
1090    /// pcomm thread-group leader hosting worker threads across one
1091    /// or more logical groups. `groups` records the per-group
1092    /// `(group_idx, num_workers)` layout in the order threads were
1093    /// spawned: `groups[0]` contributes the first
1094    /// `groups[0].1` worker reports, `groups[1]` the next
1095    /// `groups[1].1`, and so on. Total expected reports =
1096    /// `groups.iter().map(|(_, n)| n).sum()`. The layout drives
1097    /// sentinel distribution: when the JSON payload is missing or
1098    /// short, the parent emits sentinels tagged with the right
1099    /// per-group `group_idx` so per-group filters partition
1100    /// correctly.
1101    ///
1102    /// Report wire format: bare `b'r'` ready byte followed by a
1103    /// `serde_json::to_vec(&Vec<WorkerReport>)` document, one entry
1104    /// per worker thread (in `(group_idx, within-group index)`
1105    /// traversal order matching the `groups` layout).
1106    PcommContainer { groups: Vec<(usize, usize)> },
1107}
1108
1109/// Bookkeeping for a single forked-child process owned by the spawn
1110/// pipeline. Replaces the prior `(pid, report_fd, start_fd)` tuple so
1111/// per-child decoding metadata ([`ForkedChildKind`]) lives alongside
1112/// the pipe fds and pid.
1113#[derive(Clone, Debug)]
1114pub(super) struct ForkedChild {
1115    /// Forked child's pid (== tgid for both `Worker` and
1116    /// `PcommContainer`: the worker is its own thread-group leader,
1117    /// the container is the thread-group leader of its inner threads).
1118    pub pid: libc::pid_t,
1119    /// Parent-side read end of the report pipe. The child writes one
1120    /// `b'r'` ready byte followed by either a postcard-encoded single
1121    /// `WorkerReport` (Worker) or a `serde_json` `Vec<WorkerReport>`
1122    /// (PcommContainer) per the [`ForkedChildKind`] tag.
1123    pub report_fd: std::os::unix::io::RawFd,
1124    /// Parent-side write end of the start pipe. Closed by
1125    /// [`crate::workload::WorkloadHandle::start`] after writing the
1126    /// start byte; set to `-1` thereafter.
1127    pub start_fd: std::os::unix::io::RawFd,
1128    /// Per-child decoding shape. See [`ForkedChildKind`].
1129    pub kind: ForkedChildKind,
1130}
1131
1132/// Handle to spawned worker tasks. Workers block until
1133/// [`start()`](Self::start) is called.
1134///
1135/// The [`CloneMode`] in the [`WorkloadConfig`] selects how each
1136/// worker is created. Within one [`WorkloadHandle`] every worker
1137/// uses the same mode, so exactly one of `children` or `threads`
1138/// is populated; the other is empty. This avoids per-worker mode
1139/// dispatch on the hot path and keeps each vec's per-mode
1140/// invariants (pid-based vs JoinHandle-based reaping) cohesive.
1141///
1142/// - [`CloneMode::Fork`] populates `children` — separate process
1143///   per worker, reaped via `waitpid`, signaled via SIGUSR1.
1144/// - [`CloneMode::Thread`] populates `threads` — separate kernel
1145///   task in the parent's thread group via [`std::thread::spawn`],
1146///   joined via `JoinHandle`. Workers share the parent's tgid;
1147///   per-worker cgroup placement requires `cgroup.threads`
1148///   (cgroup v2 thread mode), which ktstr scenarios do not
1149///   currently configure — Thread-mode workers inherit the
1150///   parent's cgroup.
1151#[must_use = "dropping a WorkloadHandle immediately tears down all worker tasks"]
1152pub struct WorkloadHandle {
1153    /// Forked-child processes owned by this handle. Each entry is a
1154    /// [`ForkedChild`] carrying pid, parent-side pipe fds, and a
1155    /// [`ForkedChildKind`] tag that drives report decoding (single
1156    /// postcard `WorkerReport` for `Worker`; `serde_json`
1157    /// `Vec<WorkerReport>` for `PcommContainer`). Empty when every
1158    /// group used [`CloneMode::Thread`] without `pcomm`.
1159    children: Vec<ForkedChild>,
1160    /// Thread-mode workers. Empty when `clone_mode` is not
1161    /// [`CloneMode::Thread`].
1162    threads: Vec<ThreadWorker>,
1163    started: bool,
1164    /// Shared mmap regions for futex-based work types (one per worker group). Unmapped on drop.
1165    futex_ptrs: Vec<*mut u32>,
1166    /// Per-region byte length, parallel to `futex_ptrs`. Each
1167    /// region was sized at spawn time to its source group's
1168    /// natural width (4 for FutexPingPong / FutexFanOut /
1169    /// MutexContention / etc., 16 for FanOutCompute, 32 + Q*8 for
1170    /// ProducerConsumerImbalance — see [`futex_region_size_for`]).
1171    /// `futex_ptrs[i]` and `futex_region_sizes[i]` describe the
1172    /// same region; both are consumed pairwise on `Drop` so each
1173    /// `munmap` call receives the matching length.
1174    futex_region_sizes: Vec<usize>,
1175    /// MAP_SHARED region of per-worker iteration counters. Workers
1176    /// atomically store their iteration count; parent reads via
1177    /// `snapshot_iterations()`. Pointer to the first element; length
1178    /// is the active worker collection's len. Typed as
1179    /// `*mut AtomicU64` rather than `*mut u64` so the 8-byte
1180    /// alignment guarantee (inherited from the page-aligned
1181    /// iter_counters mmap site in `WorkloadHandle::spawn`) and the
1182    /// atomic-only-access invariant are encoded in the type system
1183    /// instead of prose. `AtomicU64` is layout-compatible with `u64`:
1184    /// `std::mem::size_of::<AtomicU64>() == std::mem::align_of::<AtomicU64>() == 8`
1185    /// on every supported target, so casting the `*mut c_void`
1186    /// returned by `mmap` to `*mut AtomicU64` is sound.
1187    iter_counters: *mut AtomicU64,
1188    /// Number of AtomicU64 slots in iter_counters (== num_workers at spawn time).
1189    iter_counter_len: usize,
1190    /// MAP_SHARED single-word phase-epoch region. The scenario engine
1191    /// stores the current 1-indexed phase here (0 = BASELINE, Step k →
1192    /// k + 1, `u32::MAX` = inter-step gap); every backdrop worker of
1193    /// this handle reads the SAME pointer (not a per-worker slot — the
1194    /// epoch is a broadcast value, all the handle's workers transition
1195    /// phases together) to attribute its per-phase `PhaseSlice`s. A
1196    /// single `AtomicU32`, not the per-worker `AtomicU64` layout of
1197    /// `iter_counters`, because one shared word suffices. Null until
1198    /// the spawn-site mmap installs it.
1199    phase_epoch: *mut AtomicU32,
1200    /// Byte length of the `phase_epoch` mmap, for `munmap` on drop
1201    /// (`size_of::<AtomicU32>()`; the kernel rounds the mapping up to a
1202    /// page).
1203    phase_epoch_bytes: usize,
1204    /// Inter-worker paired pipes `(ab, ba)` for PipeIo / CachePipe.
1205    /// Transferred from [`SpawnGuard`] on success; closed by
1206    /// [`WorkloadHandle::drop`] AFTER worker shutdown so Thread-mode
1207    /// workers (which share the parent's fd table) can finish their
1208    /// pipe ops before the close. Under Fork mode each child holds
1209    /// its own fd-table copy via `fork()`, so the parent's late
1210    /// close is a no-op for the children. Empty when `work_type`
1211    /// is neither PipeIo nor CachePipe.
1212    pipe_pairs: Vec<([i32; 2], [i32; 2])>,
1213    /// Per-chain pipe rings for `WakeChain { wake: WakeMechanism::Pipe }`.
1214    /// Outer Vec is one entry per chain (= `num_workers / depth`);
1215    /// inner Vec is `depth` pipes per chain. Same ownership rule as
1216    /// `pipe_pairs`: transferred from [`SpawnGuard`] on success,
1217    /// closed by [`WorkloadHandle::drop`] AFTER worker shutdown so
1218    /// Thread-mode chain workers don't observe `EBADF` mid-run.
1219    /// Empty when `work_type` is not `WakeChain { wake: Pipe }`.
1220    chain_pipes: Vec<Vec<[i32; 2]>>,
1221    /// Shared first-iteration signal eventfd. `Some` only when the
1222    /// workload set [`WorkloadConfig::signal_first_iteration`] AND the
1223    /// dispatch is fork with at least one worker; each fork worker
1224    /// `write(2)`s a `1` after its first counted iteration and the parent
1225    /// blocks on this fd in
1226    /// [`Self::wait_first_iteration_all`]. The [`std::os::fd::OwnedFd`]
1227    /// closes the parent's copy on `Drop`; fork children hold their own
1228    /// fd-table copies (the fd is `EFD_CLOEXEC`, harmless — fork workers
1229    /// never `exec`). `None` for every other workload — the polled
1230    /// `snapshot_iterations` path is unchanged.
1231    first_iter_signal: Option<std::os::fd::OwnedFd>,
1232}
1233
1234/// Per-variant byte length for the MAP_SHARED futex region.
1235///
1236/// Each WorkType that needs a shared region has a fixed natural
1237/// size:
1238///
1239/// - [`WorkType::FanOutCompute`] needs 16 bytes — futex `u32` at
1240///   offset 0, wake-timestamp `u64` at offset 8.
1241/// - [`WorkType::ProducerConsumerImbalance`] needs a ring buffer:
1242///   reserve-head `u64` @ 0, tail `u64` @ 8, producer-wake `u32`
1243///   @ 16, consumer-wake `u32` @ 20, publish-head `u64` @ 24,
1244///   then `Q` × `u64` ring slots starting at offset 32. Total
1245///   bytes = `32 + Q*8`. The two head counters split the MPMC
1246///   protocol: producers CAS-advance the reserve head to claim a
1247///   unique slot index, write the slot, then in slot-index FIFO
1248///   order release-store the publish head; consumers
1249///   acquire-load the publish head so a slot is visible to a
1250///   reader only after its producer's data write
1251///   synchronizes-with the consumer through the publish-head
1252///   release. `queue_depth_target` is `u64` to match the variant;
1253///   an `as usize` truncation on a 32-bit host could silently
1254///   produce a sub-page region with a malformed queue, so the
1255///   conversion is clamped at `usize::MAX/8 - 4` to keep the
1256///   layout well-defined (one fewer slot than before to make
1257///   room for `pub_head`). Realistic configs use Q in the
1258///   hundreds-to-thousands; the clamp only triggers on a
1259///   degenerate input that itself fails admission control
1260///   elsewhere (the queue is far larger than RAM).
1261/// - [`WorkType::SignalStorm`] needs 8 bytes — two `u32` tid
1262///   slots (worker 0's tid @ offset 0, worker 1's @ offset 4).
1263/// - Everything else: `u32` (4 bytes).
1264///
1265/// Returning the same byte count for every WorkType variant lets
1266/// the caller mmap exactly what's needed for THIS group rather
1267/// than the MAX across all groups, so a small-variant group
1268/// composed alongside a large `ProducerConsumerImbalance` no
1269/// longer pays the large group's per-region overhead.
1270pub(super) fn futex_region_size_for(work_type: &WorkType) -> usize {
1271    match work_type {
1272        WorkType::FanOutCompute { .. } => 16,
1273        WorkType::ProducerConsumerImbalance {
1274            queue_depth_target, ..
1275        } => {
1276            let q = std::cmp::min(*queue_depth_target as usize, usize::MAX / 8 - 4);
1277            32 + q * 8
1278        }
1279        // SignalStorm exchanges two u32 tid slots through the region
1280        // (worker 0's tid @ offset 0, worker 1's @ offset 4), so it
1281        // needs 8 bytes, not the default single u32.
1282        WorkType::SignalStorm { .. } => 2 * std::mem::size_of::<u32>(),
1283        _ => std::mem::size_of::<u32>(),
1284    }
1285}
1286
1287/// Block on a counter-mode eventfd `raw` until its accumulated count
1288/// reaches `need`, or `deadline` passes. Returns `true` when the count is
1289/// reached, `false` on timeout / unexpected error. Extracted from
1290/// [`WorkloadHandle::wait_first_iteration_all`] so the poll/accumulate
1291/// logic is unit-testable against a raw eventfd without spawning workers.
1292///
1293/// Each `read(2)` on a counter-mode eventfd returns the current counter
1294/// and resets it to 0, so the value is accumulated across reads. `need ==
1295/// 0` returns `true` immediately (vacuous). EINTR (poll or read) and a
1296/// spurious EFD_NONBLOCK EAGAIN re-poll against the same deadline; any
1297/// other error returns `false`.
1298fn poll_eventfd_reaches(raw: i32, deadline: Instant, need: u64) -> bool {
1299    if need == 0 {
1300        return true;
1301    }
1302    let mut acc: u64 = 0;
1303    loop {
1304        let now = Instant::now();
1305        if now >= deadline {
1306            return false;
1307        }
1308        let remaining = deadline - now;
1309        // poll(2) timeout in ms: truncating `as_millis` under-waits by
1310        // <1ms (the outer deadline re-check bounds the total), and the
1311        // `.max(1)` avoids a busy-spin on a sub-ms remainder.
1312        let timeout_ms = remaining.as_millis().min(i32::MAX as u128).max(1) as i32;
1313        let mut pfd = libc::pollfd {
1314            fd: raw,
1315            events: libc::POLLIN,
1316            revents: 0,
1317        };
1318        let ret = unsafe { libc::poll(&mut pfd, 1, timeout_ms) };
1319        if ret < 0 {
1320            let err = std::io::Error::last_os_error();
1321            // EINTR (e.g. SIGCHLD coalescing under the probe's forked
1322            // workers): re-poll against the same deadline.
1323            if err.raw_os_error() == Some(libc::EINTR) {
1324                continue;
1325            }
1326            return false;
1327        }
1328        if ret == 0 {
1329            // Timed out this slice; the loop top re-checks the deadline.
1330            continue;
1331        }
1332        if pfd.revents & libc::POLLIN != 0 {
1333            // Counter-mode read: returns the summed counter and resets it
1334            // to 0, so accumulate across reads.
1335            let mut buf: u64 = 0;
1336            let r = unsafe {
1337                libc::read(
1338                    raw,
1339                    &mut buf as *mut u64 as *mut libc::c_void,
1340                    std::mem::size_of::<u64>(),
1341                )
1342            };
1343            if r == std::mem::size_of::<u64>() as isize {
1344                acc = acc.saturating_add(buf);
1345                if acc >= need {
1346                    return true;
1347                }
1348            } else if r < 0 {
1349                let err = std::io::Error::last_os_error();
1350                // EAGAIN: EFD_NONBLOCK spurious wake with no pending count;
1351                // EINTR: signal mid-read. Either way re-poll.
1352                if matches!(err.raw_os_error(), Some(libc::EAGAIN) | Some(libc::EINTR)) {
1353                    continue;
1354                }
1355                return false;
1356            }
1357        }
1358    }
1359}
1360
1361/// Scope guard that owns every resource acquired during
1362/// [`WorkloadHandle::spawn`]'s partial setup. If `spawn` returns
1363/// early (via `?` or `bail!`), the guard's `Drop` kills and reaps any
1364/// already-forked children, closes every open pipe fd, and munmaps
1365/// every shared region — so a mid-setup failure never leaks fds,
1366/// zombie processes, or anonymous-shared pages.
1367///
1368/// On success, [`SpawnGuard::into_handle`] moves every live
1369/// resource — children/threads, futex regions, iter-counter
1370/// region, AND `pipe_pairs` / `chain_pipes` — into the returned
1371/// [`WorkloadHandle`]. The guard's subsequent `Drop` runs against
1372/// empty Vecs/null pointers and is a no-op on the success path.
1373/// On the early-bail path (an `?` inside `WorkloadHandle::spawn`)
1374/// the guard still owns whatever it allocated and `Drop` cleans
1375/// it all up — fds, processes, threads, mmaps. Pipe fds are
1376/// closed by the handle (not the guard) because Thread-mode
1377/// workers share the parent's fd table; closing the fds before
1378/// worker shutdown would surface as `EBADF` on every pipe op a
1379/// thread runs after spawn returns.
1380pub(super) struct SpawnGuard {
1381    /// Inter-worker paired pipes `(ab, ba)` for PipeIo/CachePipe.
1382    /// Transferred to [`WorkloadHandle`] on success; closed by the
1383    /// guard only on the early-bail path. Under Fork mode each
1384    /// child holds its own fd-table copy via `fork()`; under
1385    /// Thread mode every worker thread shares these fds with the
1386    /// parent.
1387    pipe_pairs: Vec<([i32; 2], [i32; 2])>,
1388    /// Per-chain pipe rings for `WakeChain { wake: WakeMechanism::Pipe }`. Outer
1389    /// Vec is one entry per chain (= `num_workers / depth`); inner
1390    /// Vec is `depth` pipes per chain. Pipe `i` connects stage `i`
1391    /// (writer) to stage `(i + 1) % depth` (reader). Same ownership
1392    /// shape as `pipe_pairs`: transferred to the handle on success,
1393    /// closed by the guard only on the early-bail path.
1394    chain_pipes: Vec<Vec<[i32; 2]>>,
1395    /// Shared-memory futex regions (transferred to handle on success).
1396    futex_ptrs: Vec<*mut u32>,
1397    /// Per-region byte length, parallel to `futex_ptrs`. Each
1398    /// region is sized to its source group's natural width
1399    /// (4 / 16 / 32+Q*8 — see [`futex_region_size_for`]) and
1400    /// recorded here at `spawn_group` time so munmap on Drop
1401    /// can call `libc::munmap(ptr, len)` with the matching length
1402    /// even when groups with different natural sizes co-exist.
1403    /// `futex_ptrs[i]` and `futex_region_sizes[i]` describe the
1404    /// same region.
1405    futex_region_sizes: Vec<usize>,
1406    /// Per-worker iteration counter region (transferred on success).
1407    /// Typed matches the handle field; see `WorkloadHandle::iter_counters`.
1408    iter_counters: *mut AtomicU64,
1409    iter_counter_bytes: usize,
1410    /// Per-handle phase-epoch region (transferred to handle on
1411    /// success). Single `AtomicU32` word shared by all the handle's
1412    /// workers; see `WorkloadHandle::phase_epoch`.
1413    phase_epoch: *mut AtomicU32,
1414    phase_epoch_bytes: usize,
1415    /// Already-forked children (either conventional workers or
1416    /// pcomm containers) with their parent-side pipe fds and
1417    /// per-child decoding shape (transferred to handle on success).
1418    children: Vec<ForkedChild>,
1419    /// Already-spawned thread workers (transferred on success).
1420    /// Cleanup on early-exit flips each `stop` and joins each
1421    /// thread, since threads share the parent's address space and
1422    /// must be drained cooperatively (no `kill` equivalent).
1423    threads: Vec<ThreadWorker>,
1424    /// First-iteration signal eventfd (transferred to the handle on
1425    /// success; see [`WorkloadHandle::first_iter_signal`]). The
1426    /// [`std::os::fd::OwnedFd`] closes on the guard's `Drop`, so the
1427    /// early-bail path releases the fd exactly like the iter_counters
1428    /// mmap it is allocated beside — no explicit cleanup needed.
1429    first_iter_signal: Option<std::os::fd::OwnedFd>,
1430}
1431
1432impl SpawnGuard {
1433    fn new() -> Self {
1434        Self {
1435            pipe_pairs: Vec::new(),
1436            chain_pipes: Vec::new(),
1437            futex_ptrs: Vec::new(),
1438            futex_region_sizes: Vec::new(),
1439            iter_counters: std::ptr::null_mut(),
1440            iter_counter_bytes: 0,
1441            phase_epoch: std::ptr::null_mut(),
1442            phase_epoch_bytes: 0,
1443            children: Vec::new(),
1444            threads: Vec::new(),
1445            first_iter_signal: None,
1446        }
1447    }
1448
1449    /// Transfer live resources into a [`WorkloadHandle`]. Leaves the
1450    /// guard's `children`, `threads`, `futex_ptrs`,
1451    /// `futex_region_sizes`, `iter_counters`, `pipe_pairs`, and
1452    /// `chain_pipes` empty, so the guard's subsequent `Drop` is a
1453    /// no-op on the success path. The handle is now the sole owner
1454    /// of every resource — its own `Drop` closes the pipe fds
1455    /// AFTER worker shutdown completes, which is the ordering
1456    /// Thread mode requires (workers share the parent's fd table;
1457    /// closing pre-shutdown would surface as `EBADF` on every
1458    /// worker's pipe op). Fork mode is unaffected either way: each
1459    /// child holds its own fd-table copy via `fork()`, so the
1460    /// parent's close timing is invisible to the child.
1461    fn into_handle(mut self) -> WorkloadHandle {
1462        let children = std::mem::take(&mut self.children);
1463        let threads = std::mem::take(&mut self.threads);
1464        let futex_ptrs = std::mem::take(&mut self.futex_ptrs);
1465        let futex_region_sizes = std::mem::take(&mut self.futex_region_sizes);
1466        let iter_counters = std::mem::replace(&mut self.iter_counters, std::ptr::null_mut());
1467        let iter_counter_bytes = std::mem::replace(&mut self.iter_counter_bytes, 0);
1468        let iter_counter_len = iter_counter_bytes / std::mem::size_of::<AtomicU64>();
1469        let phase_epoch = std::mem::replace(&mut self.phase_epoch, std::ptr::null_mut());
1470        let phase_epoch_bytes = std::mem::replace(&mut self.phase_epoch_bytes, 0);
1471        let pipe_pairs = std::mem::take(&mut self.pipe_pairs);
1472        let chain_pipes = std::mem::take(&mut self.chain_pipes);
1473        let first_iter_signal = self.first_iter_signal.take();
1474        WorkloadHandle {
1475            children,
1476            threads,
1477            started: false,
1478            futex_ptrs,
1479            futex_region_sizes,
1480            iter_counters,
1481            iter_counter_len,
1482            phase_epoch,
1483            phase_epoch_bytes,
1484            pipe_pairs,
1485            chain_pipes,
1486            first_iter_signal,
1487        }
1488    }
1489}
1490
1491impl Drop for SpawnGuard {
1492    fn drop(&mut self) {
1493        // Kill and reap any already-forked children first, so their
1494        // pipe ends are not left blocked when we close the parent
1495        // side. `nix` wrappers replace the raw libc calls — kill
1496        // returns `Result<()>` (we swallow ECHILD/ESRCH in the
1497        // already-exited case), waitpid returns `Result<WaitStatus>`
1498        // (we discard the status in the cleanup path), close returns
1499        // `Result<()>` (we swallow EBADF for fds an earlier arm may
1500        // have already closed).
1501        for child in &self.children {
1502            let npid = nix::unistd::Pid::from_raw(child.pid);
1503            let _ = nix::sys::signal::kill(npid, nix::sys::signal::Signal::SIGKILL);
1504            let _ = nix::sys::wait::waitpid(npid, None);
1505        }
1506        // Close each child's parent-side report/start fds.
1507        for child in &self.children {
1508            close_fds_silently(&[child.report_fd, child.start_fd]);
1509        }
1510        // Stop and join any partially-spawned threads. Threads
1511        // share our address space, so `kill` does not reach them
1512        // and the only safe teardown is "flip stop, drop the start
1513        // channel (in case worker is still parked on `recv`), then
1514        // join". Dropping `start_tx` causes `recv` on the worker
1515        // side to return `Err(Disconnected)`, unblocking a thread
1516        // that has not yet been signaled. After both signals
1517        // (stop=true and start_tx dropped), `worker_main`'s outer
1518        // loop exits at the next `stop.load(Relaxed)` check (max
1519        // ~100ms latency from the `FUTEX_WAIT_TIMEOUT` poll
1520        // cadence) and the thread completes. `join` returns the
1521        // partial `WorkerReport` (or `Err` on panic, which we
1522        // swallow because mid-spawn cleanup has nothing to assert).
1523        for tw in &mut self.threads {
1524            tw.stop.store(true, Ordering::Relaxed);
1525            // Drop start_tx FIRST so a worker still parked on
1526            // recv() unblocks via Disconnected.
1527            tw.start_tx.take();
1528            if let Some(j) = tw.join.take() {
1529                // SpawnGuard cleanup uses the same `THREAD_JOIN_TIMEOUT`
1530                // budget as `stop_and_collect` and `WorkloadHandle::drop`
1531                // so a stuck worker can't pin mid-spawn error recovery.
1532                // Errors (panic / timeout) are silently dropped — the
1533                // mid-spawn path has nothing to assert against beyond
1534                // not leaking, and the spawn-side bail message has
1535                // already named the failure mode that triggered cleanup.
1536                let _ = join_thread_with_timeout(j, &tw.exit_evt, THREAD_JOIN_TIMEOUT);
1537            }
1538        }
1539        // Early-bail pipe close. On the success path, into_handle
1540        // moved both `pipe_pairs` and `chain_pipes` into the handle,
1541        // so these Vecs are empty here and these loops iterate
1542        // nothing. On the early-bail path the guard still owns the
1543        // partially-allocated pipes and must close them now — the
1544        // child arm of each fork already closed any inherited
1545        // copies it held, and Thread-mode early-bail joined any
1546        // partially-spawned threads above before this loop runs.
1547        for (ab, ba) in &self.pipe_pairs {
1548            close_fds_silently(&[ab[0], ab[1], ba[0], ba[1]]);
1549        }
1550        for chain in &self.chain_pipes {
1551            for pipe in chain {
1552                close_fds_silently(&[pipe[0], pipe[1]]);
1553            }
1554        }
1555        // Munmap shared regions. `futex_ptrs[i]` and
1556        // `futex_region_sizes[i]` describe the same region, so each
1557        // munmap receives the exact length used for the matching
1558        // mmap. The two vectors are appended in lockstep inside
1559        // `spawn_group`, so they have identical lengths in every
1560        // observable state.
1561        for (&ptr, &size) in self.futex_ptrs.iter().zip(self.futex_region_sizes.iter()) {
1562            unsafe {
1563                libc::munmap(ptr as *mut libc::c_void, size);
1564            }
1565        }
1566        if !self.iter_counters.is_null() && self.iter_counter_bytes > 0 {
1567            unsafe {
1568                libc::munmap(
1569                    self.iter_counters as *mut libc::c_void,
1570                    self.iter_counter_bytes,
1571                );
1572            }
1573        }
1574        if !self.phase_epoch.is_null() && self.phase_epoch_bytes > 0 {
1575            unsafe {
1576                libc::munmap(
1577                    self.phase_epoch as *mut libc::c_void,
1578                    self.phase_epoch_bytes,
1579                );
1580            }
1581        }
1582    }
1583}
1584
1585// SAFETY: futex_ptrs, iter_counters, and phase_epoch are MAP_SHARED anonymous pages
1586// created before fork, so every forked child inherits a pointer copy
1587// of the same underlying kernel object. Children read/write their own
1588// futex word — via `std::ptr::read_volatile`/`write_volatile` for
1589// most WorkType variants, or via `AtomicU32`/`AtomicU64` references
1590// re-derived from the raw pointer for FanOutCompute, which needs
1591// release-acquire ordering to publish `wake_ns` alongside the
1592// generation advance — and atomically store into their dedicated
1593// iter_counters slot (via a shared `&AtomicU64` reference derived
1594// from the `*mut AtomicU64` region pointer); the parent reads
1595// all slots via `snapshot_iterations` and is the sole process that
1596// munmaps the region, on WorkloadHandle::drop after every child has
1597// been reaped.
1598//
1599// Per-mode aliasing rationale:
1600//
1601// - Fork mode: each forked child constructs its own process-local
1602//   `&AtomicU32`/`&AtomicU64` shared reference into the MAP_SHARED
1603//   page from the inherited raw pointer. No reference value ever
1604//   crosses a process boundary — each process synthesises its own
1605//   reference from the same underlying kernel object. Interior
1606//   mutation through a shared atomic reference is permitted by
1607//   Rust's aliasing model because `AtomicU32`/`AtomicU64` wrap an
1608//   `UnsafeCell`; the post-fork alias relation is therefore not an
1609//   aliasing-rule violation.
1610//
1611// - Thread mode: under [`CloneMode::Thread`] every worker thread
1612//   shares the parent process's single address space — the same
1613//   raw `*mut AtomicU32`/`*mut AtomicU64` pointer is dereferenced
1614//   from multiple threads concurrently, and the resulting
1615//   `&AtomicU32`/`&AtomicU64` shared references coexist for
1616//   overlapping lifetimes. This is sound for the same reason
1617//   `Arc<AtomicU64>` is sound: atomic types' `UnsafeCell`-wrapped
1618//   storage permits concurrent shared-reference access by design,
1619//   and the underlying load/store instructions are by construction
1620//   non-tearing on every supported target. No `&mut` reference is
1621//   ever materialised; every access is via the atomic API. The
1622//   MAP_SHARED region is allocated once before any worker spawns
1623//   and `munmap`ped after every worker has been joined, so the
1624//   underlying kernel object outlives every alias.
1625//
1626// `phase_epoch` follows the same MAP_SHARED aliasing rules with the
1627// roles reversed: the parent (scenario engine) is the sole WRITER of
1628// the single `AtomicU32` word (Release stores at each step boundary)
1629// and the backdrop workers are READERS (Relaxed loads). It is
1630// telemetry — no happens-before edge is required, matching the
1631// `iter_counters` Relaxed-store / Relaxed-read precedent.
1632unsafe impl Send for WorkloadHandle {}
1633unsafe impl Sync for WorkloadHandle {}
1634
1635/// Pointer-sized addresses passed across a thread-spawn boundary.
1636///
1637/// Rust's auto-`Send` inference on closures conservatively treats
1638/// `*mut T` as `!Send` even inside a wrapper struct destructured in
1639/// the closure body — the destructured field type leaks into the
1640/// closure's auto-trait check. The simplest workaround is to round-
1641/// trip the pointers through `usize` (Send + Copy) and re-cast on
1642/// the receiver side. Soundness is identical: thread-mode workers
1643/// share the parent's address space, so the addresses retain
1644/// meaning across the thread boundary, and the underlying
1645/// MAP_SHARED regions are owned by the guard / handle for the full
1646/// duration of every worker.
1647///
1648/// `SendFutexPtr` carries a (futex_address, pos) tuple wrapped in
1649/// `Option`; `None` is the "no futex required" case for work types
1650/// that don't need shared memory. `SendIterSlotPtr` carries a single
1651/// address (zero ⇒ no iter_slot publish).
1652#[derive(Clone, Copy)]
1653pub(super) struct SendFutexPtr(Option<(usize, usize)>);
1654
1655#[derive(Clone, Copy)]
1656pub(super) struct SendIterSlotPtr(usize);
1657
1658impl SendFutexPtr {
1659    fn new(p: Option<(*mut u32, usize)>) -> Self {
1660        SendFutexPtr(p.map(|(ptr, pos)| (ptr as usize, pos)))
1661    }
1662
1663    /// Re-cast back into the `*mut u32` + `pos` tuple `worker_main`
1664    /// expects.
1665    fn into_raw(self) -> Option<(*mut u32, usize)> {
1666        self.0.map(|(addr, pos)| (addr as *mut u32, pos))
1667    }
1668}
1669
1670impl SendIterSlotPtr {
1671    fn new(p: *mut AtomicU64) -> Self {
1672        SendIterSlotPtr(p as usize)
1673    }
1674
1675    fn into_raw(self) -> *mut AtomicU64 {
1676        self.0 as *mut AtomicU64
1677    }
1678}
1679
1680/// Send-safe carrier for the single shared `phase_epoch` word (the SAME
1681/// pointer for every worker in a group), mirroring [`SendIterSlotPtr`].
1682/// Round-trips the address through `usize` so a worker closure's capture
1683/// set stays `Send` (no raw-pointer field in the closure type).
1684#[derive(Clone, Copy)]
1685pub(super) struct SendPhaseEpochPtr(usize);
1686
1687impl SendPhaseEpochPtr {
1688    fn new(p: *mut AtomicU32) -> Self {
1689        SendPhaseEpochPtr(p as usize)
1690    }
1691
1692    fn into_raw(self) -> *mut AtomicU32 {
1693        self.0 as *mut AtomicU32
1694    }
1695}
1696
1697/// Per-group resolved view of [`WorkloadConfig`] used by the
1698/// spawn pipeline.
1699///
1700/// [`WorkloadHandle::spawn`] iterates one `GroupParams` per group
1701/// it spawns: the primary group (`group_idx == 0`) is built from
1702/// the top-level [`WorkloadConfig`] fields via
1703/// [`Self::primary`], and each composed [`WorkSpec`] entry is
1704/// resolved into its own `GroupParams` (with `group_idx ==
1705/// 1..=N`) via [`Self::from_composed`]. Both paths funnel through
1706/// [`Self::from_work_spec`] for the actual field copy.
1707///
1708/// `GroupParams` is the post-resolution shape — `num_workers` is a
1709/// concrete `usize` (not the `Option<usize>` that [`WorkSpec`]
1710/// carries), `affinity` is a concrete [`ResolvedAffinity`] (not
1711/// the [`AffinityIntent`] that [`WorkSpec`] carries). The spawn
1712/// pipeline operates on `GroupParams` exclusively so it never has
1713/// to deal with the unresolved intent/optional shapes that the
1714/// user-facing types expose.
1715///
1716/// `clone_mode` is shared across every group — the top-level
1717/// [`WorkloadConfig::clone_mode`] selects fork vs thread dispatch
1718/// for the entire workload, and [`WorkSpec`] carries no
1719/// `clone_mode` field of its own (composed entries inherit the
1720/// parent's mode; the [`SpawnGuard`]'s lifecycle assumes a single
1721/// dispatch path).
1722#[derive(Clone)]
1723pub(super) struct GroupParams {
1724    work_type: WorkType,
1725    /// Park each worker after `n` counted iterations rather than
1726    /// running to stop. Only the primary group can carry a non-`None`
1727    /// value ([`WorkloadConfig::park_after_iterations`] has no
1728    /// [`WorkSpec`] counterpart), so composed groups always inherit
1729    /// `None` through [`Self::from_work_spec`].
1730    park_after: Option<u64>,
1731    /// Signal the handle's first-iteration eventfd after each fork
1732    /// worker's first counted iteration. Like `park_after`, only the
1733    /// primary group can carry `true`
1734    /// ([`WorkloadConfig::signal_first_iteration`] has no [`WorkSpec`]
1735    /// counterpart), so composed groups always inherit `false` through
1736    /// [`Self::from_work_spec`].
1737    signal_first_iteration: bool,
1738    sched_policy: SchedPolicy,
1739    mem_policy: MemPolicy,
1740    mpol_flags: MpolFlags,
1741    nice: Option<i32>,
1742    comm: Option<String>,
1743    uid: Option<u32>,
1744    gid: Option<u32>,
1745    numa_node: Option<u32>,
1746    affinity: ResolvedAffinity,
1747    num_workers: usize,
1748    group_idx: usize,
1749}
1750
1751impl GroupParams {
1752    /// Extract a [`GroupParams`] from a [`WorkSpec`] given the
1753    /// resolved sibling values. This is the single field-extraction
1754    /// site — both [`Self::primary`] and [`Self::from_composed`]
1755    /// funnel through here, so the field-by-field copy lives in one
1756    /// place.
1757    ///
1758    /// The caller is responsible for resolving the
1759    /// [`WorkSpec::num_workers`] `Option<usize>` to a concrete
1760    /// `usize` and the [`WorkSpec::affinity`] [`AffinityIntent`] to
1761    /// a concrete [`ResolvedAffinity`]. The remaining fields
1762    /// (`work_type`, `sched_policy`, `mem_policy`, `mpol_flags`,
1763    /// `nice`) are copied verbatim — they need no resolution
1764    /// because both [`WorkSpec`] and [`GroupParams`] carry them in
1765    /// their final runtime form (`nice` round-trips as
1766    /// `Option<i32>` so `None` continues to mean "skip the
1767    /// `setpriority(2)` call" at the spawn-time gate).
1768    fn from_work_spec(
1769        spec: &WorkSpec,
1770        group_idx: usize,
1771        resolved_affinity: ResolvedAffinity,
1772        resolved_num_workers: usize,
1773    ) -> Self {
1774        Self {
1775            work_type: spec.work_type.clone(),
1776            // WorkSpec has no park-after knob; `primary` overrides this
1777            // from `WorkloadConfig::park_after_iterations`, composed
1778            // groups keep `None`.
1779            park_after: None,
1780            // Same shape as `park_after`: no WorkSpec counterpart, so
1781            // `primary` overrides from `WorkloadConfig::signal_first_iteration`
1782            // and composed groups keep `false`.
1783            signal_first_iteration: false,
1784            sched_policy: spec.sched_policy,
1785            mem_policy: spec.mem_policy.clone(),
1786            mpol_flags: spec.mpol_flags,
1787            nice: spec.nice,
1788            comm: spec.comm.as_ref().map(|c| c.to_string()),
1789            uid: spec.uid,
1790            gid: spec.gid,
1791            numa_node: spec.numa_node,
1792            affinity: resolved_affinity,
1793            num_workers: resolved_num_workers,
1794            group_idx,
1795        }
1796    }
1797
1798    /// Resolve an [`AffinityIntent`] to a [`ResolvedAffinity`] under
1799    /// the spawn-time gate: only `Inherit`, `Exact`, and
1800    /// `RandomSubset` carry enough information to resolve without
1801    /// scenario context (the caller supplies the `from` pool for
1802    /// `RandomSubset`, so per-worker sampling stays self-contained).
1803    /// Topology-aware variants (`SingleCpu`, `LlcAligned`,
1804    /// `CrossCgroup`, `SmtSiblingPair`) require a
1805    /// [`crate::topology::TestTopology`] / cpuset state that
1806    /// [`WorkloadHandle::spawn`] does not have, so they bail with an
1807    /// actionable diagnostic.
1808    ///
1809    /// `site` names the location of the affinity field for the bail
1810    /// message — `"WorkloadConfig::affinity"` for the primary group,
1811    /// `"composed[N].affinity"` for entries inside `composed`. Pinned
1812    /// across both call sites so the gate matches exactly and a
1813    /// future variant addition is rejected uniformly.
1814    pub(super) fn resolve_spawn_affinity(
1815        intent: &AffinityIntent,
1816        site: &str,
1817    ) -> Result<ResolvedAffinity> {
1818        match intent {
1819            AffinityIntent::Inherit => Ok(ResolvedAffinity::None),
1820            AffinityIntent::Exact(cpus) => {
1821                if cpus.is_empty() {
1822                    anyhow::bail!(
1823                        "{site} = AffinityIntent::Exact with empty CPU set \
1824                         would produce EINVAL from sched_setaffinity; \
1825                         use AffinityIntent::Inherit for no affinity \
1826                         constraint",
1827                    );
1828                }
1829                Ok(ResolvedAffinity::Fixed(cpus.clone()))
1830            }
1831            AffinityIntent::RandomSubset { from, count } => {
1832                if from.is_empty() {
1833                    anyhow::bail!(
1834                        "{site} = AffinityIntent::RandomSubset with empty \
1835                         pool; use AffinityIntent::Inherit for no affinity \
1836                         constraint",
1837                    );
1838                }
1839                if *count == 0 {
1840                    anyhow::bail!(
1841                        "{site} = AffinityIntent::RandomSubset with \
1842                         count=0; use AffinityIntent::Inherit for no \
1843                         affinity constraint",
1844                    );
1845                }
1846                Ok(ResolvedAffinity::Random {
1847                    from: from.clone(),
1848                    count: *count,
1849                })
1850            }
1851            AffinityIntent::SingleCpu
1852            | AffinityIntent::LlcAligned
1853            | AffinityIntent::CrossCgroup
1854            | AffinityIntent::SmtSiblingPair => {
1855                anyhow::bail!(
1856                    "{site} = {:?} requires scenario context; use \
1857                     AffinityIntent::Exact(set), \
1858                     AffinityIntent::RandomSubset {{ from, count }}, \
1859                     or AffinityIntent::Inherit when spawning directly \
1860                     via WorkloadHandle::spawn. Topology-aware variants \
1861                     resolve automatically inside #[ktstr_test] \
1862                     scenarios.",
1863                    intent,
1864                );
1865            }
1866        }
1867    }
1868
1869    /// Build the primary group's parameters from the top-level
1870    /// [`WorkloadConfig`] fields. `group_idx` is fixed to `0`.
1871    ///
1872    /// Synthesises a [`WorkSpec`] view of the top-level config
1873    /// fields and funnels through [`Self::from_work_spec`] so the
1874    /// field-by-field copy lives in exactly one place. The
1875    /// synthesised spec mirrors the resolved sibling values
1876    /// (`num_workers: Some(n)`, `affinity: Inherit`) — the spawn
1877    /// pipeline never reads it.
1878    ///
1879    /// `WorkloadConfig::affinity` is an [`AffinityIntent`]
1880    /// (type-unified with [`WorkSpec::affinity`]); resolution to
1881    /// [`ResolvedAffinity`] runs through
1882    /// [`Self::resolve_spawn_affinity`] under the same gate as
1883    /// [`Self::from_composed`]. Topology-aware variants
1884    /// (`SingleCpu`, `LlcAligned`, `CrossCgroup`, `SmtSiblingPair`)
1885    /// require scenario context; the scenario engine pre-resolves
1886    /// them via
1887    /// `crate::scenario::intent_for_spawn` (which round-trips
1888    /// `RandomSubset` verbatim and flattens topology-aware variants
1889    /// to `Exact`) before building [`WorkloadConfig`], so the gate
1890    /// only ever sees `Inherit`, `Exact`, or `RandomSubset` from
1891    /// this path.
1892    fn primary(config: &WorkloadConfig) -> Result<Self> {
1893        let resolved_affinity =
1894            Self::resolve_spawn_affinity(&config.affinity, "WorkloadConfig::affinity")?;
1895        let spec = WorkSpec {
1896            work_type: config.work_type.clone(),
1897            sched_policy: config.sched_policy,
1898            num_workers: Some(config.num_workers),
1899            affinity: AffinityIntent::Inherit,
1900            mem_policy: config.mem_policy.clone(),
1901            mpol_flags: config.mpol_flags,
1902            nice: config.nice,
1903            comm: config.comm.clone(),
1904            uid: config.uid,
1905            gid: config.gid,
1906            numa_node: config.numa_node,
1907            pcomm: None,
1908            workers_pct: None,
1909        };
1910        let mut params = Self::from_work_spec(&spec, 0, resolved_affinity, config.num_workers);
1911        // The park-after knob lives on WorkloadConfig, not the
1912        // synthesised WorkSpec above, so carry it in after the copy.
1913        params.park_after = config.park_after_iterations;
1914        // Same for the first-iteration signal knob.
1915        params.signal_first_iteration = config.signal_first_iteration;
1916        Ok(params)
1917    }
1918
1919    /// Resolve a composed [`WorkSpec`] into per-group parameters,
1920    /// applying the spawn-time rules documented on
1921    /// [`WorkloadConfig::composed`]:
1922    ///
1923    /// - `num_workers` must be `Some(n)`; the `None` default
1924    ///   resolved by the scenario engine via
1925    ///   `Ctx::workers_per_cgroup` is unreachable here. A `None`
1926    ///   value is rejected with an actionable diagnostic.
1927    /// - `affinity` resolution runs through
1928    ///   [`Self::resolve_spawn_affinity`] —
1929    ///   [`AffinityIntent::Inherit`] (mapped to
1930    ///   [`ResolvedAffinity::None`]),
1931    ///   [`AffinityIntent::Exact`] (mapped to
1932    ///   [`ResolvedAffinity::Fixed`]), and
1933    ///   [`AffinityIntent::RandomSubset`] (mapped to
1934    ///   [`ResolvedAffinity::Random`]) are accepted; topology-aware
1935    ///   variants are rejected.
1936    ///
1937    /// Composed entries inherit the parent
1938    /// [`WorkloadConfig::clone_mode`]; [`WorkSpec`] has no
1939    /// `clone_mode` field of its own.
1940    fn from_composed(spec: &WorkSpec, group_idx: usize) -> Result<Self> {
1941        if spec.pcomm.is_some() {
1942            anyhow::bail!(
1943                "composed[{}].pcomm: pcomm via WorkloadHandle::spawn is not supported; \
1944                 use WorkloadHandle::spawn_pcomm_cgroup or CgroupDef (apply_setup) — \
1945                 spawn always forks one process per worker and never coalesces into \
1946                 a thread-group leader",
1947                group_idx - 1,
1948            );
1949        }
1950        let num_workers = spec.num_workers.ok_or_else(|| {
1951            anyhow::anyhow!(
1952                "composed[{}].num_workers must be set explicitly at spawn time \
1953                 (the Some/None resolution via Ctx::workers_per_cgroup is only \
1954                 available through the scenario engine; \
1955                 WorkloadHandle::spawn requires a concrete count)",
1956                group_idx - 1,
1957            )
1958        })?;
1959        let site = format!("composed[{}].affinity", group_idx - 1);
1960        let affinity = Self::resolve_spawn_affinity(&spec.affinity, &site)?;
1961        Ok(Self::from_work_spec(spec, group_idx, affinity, num_workers))
1962    }
1963}
1964
1965/// Shared per-group admission rules common to every spawn entry
1966/// point. Validates only the rules that are dispatch-agnostic —
1967/// `worker_group_size` divisibility, `chain_pipe_depth >= 2`,
1968/// `IdleChurn` zero-duration rejections, `IpcVariance` zero-knob
1969/// rejections. Dispatch-specific compatibility checks (CloneMode
1970/// vs WorkType, pcomm vs WorkType) stay at each entry point's
1971/// admission block since their reasoning differs per dispatch.
1972///
1973/// Called from [`WorkloadHandle::spawn`] (per-group inside
1974/// `groups`) and [`WorkloadHandle::spawn_pcomm_cgroup`] (per-group
1975/// inside its own `groups`). Centralises the rules so a future
1976/// addition (e.g. a new `WorkType` zero-rejection) lives in one
1977/// place.
1978pub(super) fn validate_workload_admission(group: &GroupParams) -> Result<()> {
1979    if let Some(group_size) = group.work_type.worker_group_size()
1980        && (group.num_workers == 0 || !group.num_workers.is_multiple_of(group_size))
1981    {
1982        return Err(WorkTypeValidationError::NonDivisibleWorkerCount {
1983            name: group.work_type.name().to_string(),
1984            group_idx: group.group_idx,
1985            group_size,
1986            num_workers: group.num_workers,
1987        }
1988        .into());
1989    }
1990    if let Some(depth) = group.work_type.chain_pipe_depth()
1991        && depth < 2
1992    {
1993        return Err(WorkTypeValidationError::InsufficientWakeChainDepth {
1994            depth,
1995            group_idx: group.group_idx,
1996        }
1997        .into());
1998    }
1999    if let WorkType::IdleChurn {
2000        burst_duration,
2001        sleep_duration,
2002        ..
2003    } = group.work_type
2004    {
2005        if burst_duration.is_zero() {
2006            return Err(WorkTypeValidationError::ZeroBurstDuration {
2007                group_idx: group.group_idx,
2008            }
2009            .into());
2010        }
2011        if sleep_duration.is_zero() {
2012            return Err(WorkTypeValidationError::ZeroSleepDuration {
2013                group_idx: group.group_idx,
2014            }
2015            .into());
2016        }
2017    }
2018    if let WorkType::IpcVariance {
2019        hot_iters,
2020        cold_iters,
2021        period_iters,
2022    } = group.work_type
2023    {
2024        if hot_iters == 0 {
2025            return Err(WorkTypeValidationError::ZeroIpcVarianceParam {
2026                field: "hot_iters",
2027                group_idx: group.group_idx,
2028            }
2029            .into());
2030        }
2031        if cold_iters == 0 {
2032            return Err(WorkTypeValidationError::ZeroIpcVarianceParam {
2033                field: "cold_iters",
2034                group_idx: group.group_idx,
2035            }
2036            .into());
2037        }
2038        if period_iters == 0 {
2039            return Err(WorkTypeValidationError::ZeroIpcVarianceParam {
2040                field: "period_iters",
2041                group_idx: group.group_idx,
2042            }
2043            .into());
2044        }
2045    }
2046    if let WorkType::TimerLatency { interval_us } = group.work_type
2047        && interval_us == 0
2048    {
2049        return Err(WorkTypeValidationError::ZeroTimerInterval {
2050            group_idx: group.group_idx,
2051        }
2052        .into());
2053    }
2054    if let WorkType::NetTraffic { frame_bytes, .. } = group.work_type
2055        && !(60..=1514).contains(&frame_bytes)
2056    {
2057        return Err(WorkTypeValidationError::NetTrafficFrameBytes {
2058            frame_bytes,
2059            group_idx: group.group_idx,
2060        }
2061        .into());
2062    }
2063    if let WorkType::IrqWake { frame_bytes, .. } = group.work_type
2064        && !(60..=1514).contains(&frame_bytes)
2065    {
2066        return Err(WorkTypeValidationError::IrqWakeFrameBytes {
2067            frame_bytes,
2068            group_idx: group.group_idx,
2069        }
2070        .into());
2071    }
2072    Ok(())
2073}
2074
2075/// Spawn a single thread-mode worker via [`std::thread::Builder`].
2076///
2077/// The thread closure runs `worker_main` directly with the same
2078/// per-worker arguments the fork dispatch passes, except `stop` is
2079/// a per-worker `Arc<AtomicBool>` instead of the global [`STOP`].
2080/// Start rendezvous uses an `mpsc::sync_channel(0)` because every
2081/// worker needs to block until the parent calls
2082/// [`WorkloadHandle::start`]; the parent then sends `()` to each
2083/// worker's `start_tx` to unblock them in order.
2084///
2085/// Also uses a capacity-1 `mpsc::sync_channel(1)` tid-publish
2086/// channel so this function blocks (up to 2 s) until the worker
2087/// thread reaches the `gettid()` publish point. That makes
2088/// [`WorkloadHandle::worker_pids`] safe to call immediately after
2089/// [`WorkloadHandle::spawn`] returns without first calling
2090/// [`WorkloadHandle::start`] — the publish happens BEFORE the
2091/// start handshake, so post-spawn tid reads always observe the
2092/// gettid() value (or spawn would have bailed with a "did not
2093/// publish gettid() within 2 s" error).
2094///
2095/// SIGUSR1 is process-wide and useless for per-thread stop control,
2096/// so this path does not install a signal handler. The parent flips
2097/// `stop` directly from [`WorkloadHandle::stop_and_collect`].
2098#[allow(clippy::too_many_arguments)]
2099pub(super) fn spawn_thread_worker(
2100    guard: &mut SpawnGuard,
2101    group: &GroupParams,
2102    affinity: Option<BTreeSet<usize>>,
2103    worker_pipe_fds: Option<(i32, i32)>,
2104    worker_futex: Option<(*mut u32, usize)>,
2105    iter_slot: *mut AtomicU64,
2106    phase_epoch: *mut AtomicU32,
2107    // First-iteration signal fd (`-1` = absent). Always `-1` on this
2108    // thread path: the eventfd is allocated only for fork dispatch, so a
2109    // thread-mode worker never signals (a plain `i32` copy, captured by
2110    // the closure below).
2111    first_iter_signal_fd: i32,
2112) -> Result<()> {
2113    use std::sync::Arc;
2114    use std::sync::atomic::AtomicI32;
2115    use std::sync::mpsc;
2116
2117    // SyncSender(0) — bounded rendezvous channel. The thread blocks
2118    // in `recv()` until the parent sends `()`; if the parent drops
2119    // the sender first (mid-spawn cleanup or early bail), `recv()`
2120    // returns `Err(Disconnected)` and the closure exits cleanly.
2121    let (start_tx, start_rx) = mpsc::sync_channel::<()>(0);
2122    // tid-publish handshake. Capacity 1 so the worker's send is
2123    // non-blocking; the parent's `recv_timeout` below blocks until
2124    // the worker reaches the publish point. Without this, the
2125    // post-spawn `WorkloadHandle::worker_pids()` race-reads the
2126    // initial 0 sentinel because the worker thread hasn't been
2127    // scheduled to the `tid_thread.store(my_tid, Release)` site yet
2128    // — surfaced as a spurious `Thread SpinWait worker reported
2129    // non-positive tid=0` test failure in
2130    // tests/worker_thread_integration.rs.
2131    let (tid_pub_tx, tid_pub_rx) = mpsc::sync_channel::<()>(1);
2132    let stop = Arc::new(AtomicBool::new(false));
2133    let tid = Arc::new(AtomicI32::new(0));
2134    // Per-worker exit eventfd: bumped by a Drop guard inside the
2135    // closure right before the thread returns its `WorkerReport`. The
2136    // parent's `join_thread_with_timeout` blocks in `epoll_wait` on
2137    // this fd instead of sleep-polling `is_finished`. Created with
2138    // `EFD_NONBLOCK` so the Drop-time `write` cannot block; counter
2139    // mode so a missed read just accumulates without losing the edge.
2140    let exit_evt = Arc::new(
2141        vmm_sys_util::eventfd::EventFd::new(libc::EFD_NONBLOCK)
2142            .context("create thread-worker exit eventfd")?,
2143    );
2144
2145    // Clone Arcs for the closure. The thread takes ownership of the
2146    // closure-side handles; the parent retains the originals via
2147    // ThreadWorker for stop signaling and tid reading.
2148    let stop_thread = Arc::clone(&stop);
2149    let tid_thread = Arc::clone(&tid);
2150    let exit_evt_thread = Arc::clone(&exit_evt);
2151    let work_type = group.work_type.clone();
2152    let park_after = group.park_after;
2153    let sched_policy = group.sched_policy;
2154    let mem_policy = group.mem_policy.clone();
2155    let mpol_flags = group.mpol_flags;
2156    let nice = group.nice;
2157    let comm = group.comm.clone();
2158    let uid = group.uid;
2159    let gid = group.gid;
2160    let numa_node = group.numa_node;
2161    let group_idx = group.group_idx;
2162    let num_workers = group.num_workers;
2163
2164    // The closure must be `Send` to cross the thread boundary.
2165    // `worker_pipe_fds` is `Option<(i32, i32)>` (Copy + Send), but
2166    // `worker_futex` and `iter_slot` are raw pointers and not
2167    // `Send` by default. The module-level `SendFutexPtr` and
2168    // `SendIterSlotPtr` newtypes round-trip the addresses through
2169    // `usize` so the closure's capture set is genuinely Send (no
2170    // raw-pointer field appears in the closure type).
2171    let futex_send = SendFutexPtr::new(worker_futex);
2172    let iter_slot_send = SendIterSlotPtr::new(iter_slot);
2173    let phase_epoch_send = SendPhaseEpochPtr::new(phase_epoch);
2174
2175    let join = std::thread::Builder::new()
2176        .name(format!("ktstr-worker-g{group_idx}-{}", guard.threads.len()))
2177        .spawn(move || {
2178            // Drop guard: signal the exit eventfd as the closure
2179            // unwinds, regardless of whether `worker_main` returned
2180            // normally or panicked. The parent's
2181            // `join_thread_with_timeout` blocks in `epoll_wait` on
2182            // this fd; a panic that bypassed the explicit signal
2183            // would otherwise leave the parent waiting until the
2184            // safety timerfd fires. Drop runs even under unwinding,
2185            // so this guard captures both the normal and panic
2186            // paths.
2187            struct WorkerExitSignal(std::sync::Arc<vmm_sys_util::eventfd::EventFd>);
2188            impl Drop for WorkerExitSignal {
2189                fn drop(&mut self) {
2190                    let _ = self.0.write(1);
2191                }
2192            }
2193            let _exit_signal = WorkerExitSignal(exit_evt_thread);
2194
2195            // Publish gettid() so the parent can address this task
2196            // for sched_setaffinity and report it from worker_pids.
2197            // gettid() is the kernel TID; getpid() would return the
2198            // shared tgid, which collides across threads.
2199            let my_tid: libc::pid_t = unsafe { libc::syscall(libc::SYS_gettid) as libc::pid_t };
2200            // Release pairs with Acquire on the parent's
2201            // `tid.load()` sites so any reader observing a non-zero
2202            // tid also sees the worker's post-start state. Cheap on
2203            // every supported target (release-store on the Arc's
2204            // underlying AtomicI32 is a single instruction).
2205            tid_thread.store(my_tid, Ordering::Release);
2206            // Notify the parent that tid is published. Capacity-1
2207            // channel means this send is non-blocking; the parent's
2208            // `recv_timeout` below blocks until this fires so
2209            // post-spawn callers see populated tids. Drop the
2210            // sender after send so any future replays are no-ops.
2211            let _ = tid_pub_tx.send(());
2212            drop(tid_pub_tx);
2213
2214            // Block on start rendezvous. `Err(_)` means the parent
2215            // dropped start_tx before sending — return a sentinel
2216            // WorkerReport without doing any work.
2217            if start_rx.recv().is_err() {
2218                return WorkerReport {
2219                    tid: my_tid,
2220                    completed: false,
2221                    group_idx,
2222                    ..WorkerReport::default()
2223                };
2224            }
2225
2226            // Re-cast usize addresses back into raw pointers for
2227            // worker_main. SAFETY: the ownership and lifetime
2228            // arguments documented on `SendFutexPtr` /
2229            // `SendIterSlotPtr` ensure these pointers are still
2230            // live when worker_main dereferences them.
2231            let futex = futex_send.into_raw();
2232            let slot = iter_slot_send.into_raw();
2233            let epoch = phase_epoch_send.into_raw();
2234
2235            worker_main(
2236                affinity,
2237                work_type,
2238                sched_policy,
2239                mem_policy,
2240                mpol_flags,
2241                nice,
2242                comm.as_deref(),
2243                uid,
2244                gid,
2245                numa_node,
2246                worker_pipe_fds,
2247                futex,
2248                park_after,
2249                first_iter_signal_fd,
2250                slot,
2251                epoch,
2252                &stop_thread,
2253                group_idx,
2254            )
2255        })
2256        .with_context(|| {
2257            format!(
2258                "thread::spawn for worker {}/{} (group {}) failed",
2259                guard.threads.len() + 1,
2260                num_workers,
2261                group_idx,
2262            )
2263        })?;
2264
2265    // Block until the worker reaches the gettid() publish point.
2266    // 2 s deadline is generous: a fresh std::thread on a healthy
2267    // host reaches the closure body within microseconds; under
2268    // VM/CI pressure ~ms. Anything past 2 s indicates the worker
2269    // thread failed to schedule at all — surface as a typed error
2270    // rather than letting a downstream `worker_pids()` reader race-
2271    // read the initial 0 sentinel and tip over a tid-check assertion.
2272    match tid_pub_rx.recv_timeout(std::time::Duration::from_secs(2)) {
2273        Ok(()) => {}
2274        Err(mpsc::RecvTimeoutError::Timeout) => {
2275            anyhow::bail!(
2276                "spawn_thread_worker: worker {} (group {}) did not publish gettid() \
2277                 within 2 s of thread::spawn — the worker thread failed to schedule. \
2278                 Likely cause: host fd / thread-stack exhaustion, or the std runtime \
2279                 thread pool is wedged. tid stays at 0; subsequent worker_pids() \
2280                 reads would surface the sentinel.",
2281                guard.threads.len() + 1,
2282                group_idx,
2283            );
2284        }
2285        Err(mpsc::RecvTimeoutError::Disconnected) => {
2286            anyhow::bail!(
2287                "spawn_thread_worker: worker {} (group {}) closed tid-publish channel \
2288                 without sending — the closure panicked before reaching the gettid() \
2289                 publish point. Check the thread name's panic output for the cause.",
2290                guard.threads.len() + 1,
2291                group_idx,
2292            );
2293        }
2294    }
2295
2296    guard.threads.push(ThreadWorker {
2297        tid,
2298        stop,
2299        start_tx: Some(start_tx),
2300        join: Some(join),
2301        exit_evt,
2302    });
2303    Ok(())
2304}
2305
2306/// Per-group resource indices into the [`SpawnGuard`]'s flat
2307/// vectors used by [`spawn_pcomm_container`] to compute per-thread
2308/// pipe / chain / futex / iter-slot addresses inside the forked
2309/// container. Built by [`WorkloadHandle::spawn_pcomm_cgroup`] in
2310/// lockstep with the per-group resource allocation pass so the
2311/// container's threads can address their own group's resources from
2312/// the inherited shared regions.
2313#[derive(Clone, Copy, Debug)]
2314pub(super) struct PcommGroupResources {
2315    /// Offset into the iter_counters MAP_SHARED region for this
2316    /// group's first worker. Subsequent workers occupy
2317    /// `iter_offset + i` for `i in 0..num_workers`.
2318    pub iter_offset: usize,
2319    /// Index of this group's first pipe pair in `guard.pipe_pairs`
2320    /// (only meaningful when `needs_pipes` is true).
2321    pub pipe_pair_base: usize,
2322    /// Index of this group's first chain pipe in `guard.chain_pipes`
2323    /// (only meaningful when `chain_depth` is `Some(_)`).
2324    pub chain_pipes_base: usize,
2325    /// Index of this group's first futex region in `guard.futex_ptrs`
2326    /// (only meaningful when `needs_futex` is true).
2327    pub futex_ptrs_base: usize,
2328    /// True when the group's `WorkType` requires inter-worker pipe
2329    /// pairs (PipeIo / CachePipe).
2330    pub needs_pipes: bool,
2331    /// `Some(depth)` when the group's `WorkType` requires a chain
2332    /// pipe ring (WakeChain { wake: Pipe }); `None` otherwise.
2333    pub chain_depth: Option<usize>,
2334    /// True when the group's `WorkType` requires a MAP_SHARED futex
2335    /// region (FutexPingPong / FutexFanOut / FanOutCompute /
2336    /// MutexContention / etc.).
2337    pub needs_futex: bool,
2338    /// Worker-group-size for per-worker `pos` computation inside
2339    /// the futex region (e.g. 2 for FutexPingPong, the messenger /
2340    /// receiver split for FutexFanOut). Defaulted to 2 when the
2341    /// `WorkType` does not declare a group_size.
2342    pub futex_group_size: usize,
2343}
2344
2345/// Fork one thread-group leader hosting every worker thread for the
2346/// supplied groups: fork the leader, set its `comm` to `pcomm` via
2347/// `prctl(PR_SET_NAME)`, then spawn `groups[k].num_workers` worker
2348/// threads per group inside it. The [`WorkSpec::pcomm`] builder
2349/// rejects > 15 bytes (TASK_COMM_LEN-1), so the framework never
2350/// feeds the kernel a name `__set_task_comm` would truncate. Every
2351/// worker thread shares the leader's tgid, so
2352/// `task->group_leader->comm == pcomm` byte-for-byte. Models real
2353/// workloads like `chrome` (pcomm) hosting `ThreadPoolForeg` and
2354/// `GPU Process` (per-thread comm via [`WorkSpec::comm`]) or `java`
2355/// (pcomm) hosting `GC Thread` and `C2 CompilerThre`.
2356///
2357/// `groups` carries per-group [`GroupParams`] and `resources` carries
2358/// the parallel [`PcommGroupResources`] pre-built by
2359/// [`WorkloadHandle::spawn_pcomm_cgroup`] from the surrounding
2360/// per-group resource-allocation pass; the two slices have identical
2361/// length and `resources[k]` describes the indices owned by
2362/// `groups[k]`.
2363///
2364/// `container_uid` / `container_gid` are applied via `setresuid` /
2365/// `setresgid` once on the leader before spawning threads — the
2366/// leader takes the configured process credentials, and each worker
2367/// thread additionally re-applies its own merged credentials inside
2368/// `worker_main` at thread creation time (idempotent when the merge
2369/// produced the same value, which is the common case for a
2370/// CgroupDef-level default flowing through `merged_works`).
2371///
2372/// # Wire format
2373///
2374/// The parent collects via the same report pipe used by conventional
2375/// fork workers:
2376///
2377/// 1. After the start handshake, the leader writes one `b'r'` ready
2378///    byte to the report pipe after all threads are spawned, a
2379///    stronger guarantee than the conventional fork worker's
2380///    pre-loop byte.
2381/// 2. After all worker threads have joined, the leader serializes
2382///    `Vec<WorkerReport>` (one entry per worker thread, in
2383///    `(group_idx, within-group order)` traversal order) via
2384///    `serde_json::to_vec` and writes the bytes to the report pipe
2385///    before `_exit(0)`. The parent decodes via
2386///    `serde_json::from_slice::<Vec<WorkerReport>>`. The report pipe
2387///    is sized at 8 MiB; parent `read_to_end` drains concurrently
2388///    with the leader's `write_all` so payloads exceeding the pipe
2389///    size still drain correctly.
2390///
2391/// On a decode-failure or short payload, the parent emits one
2392/// sentinel report per expected worker so per-group filtering and
2393/// `assert_not_stuck` see the correct cardinality.
2394///
2395/// # Lifecycle
2396///
2397/// - `PR_SET_PDEATHSIG(SIGKILL)`: when the parent dies, the kernel
2398///   sends SIGKILL to the LEADER thread (the calling task at fork
2399///   time). SIGKILL on any thread is fatal-to-tgid: the kernel
2400///   routes it through `complete_signal` → `do_group_exit` →
2401///   `zap_other_threads`, which sets `SIGNAL_GROUP_EXIT` on every
2402///   sibling thread and walks the thread list signalling each. So
2403///   the cascade to worker threads happens via
2404///   `zap_other_threads`, NOT via per-task PDEATHSIG inheritance
2405///   (PDEATHSIG itself is per-task and is cleared on clone). Net:
2406///   parent death → leader's SIGKILL → all worker threads die,
2407///   so the container cannot outlive the harness.
2408/// - `setpgid(0, 0)`: the leader becomes its own process group
2409///   leader so the parent's stop/kill `killpg` reaches any
2410///   descendants the workers spawn.
2411/// - SIGUSR1 handler installed pre-thread-spawn: any
2412///   `kill(leader_pid, SIGUSR1)` from the parent flips the leader's
2413///   STOP, which every worker thread observes via
2414///   `stop_requested(&STOP)` on its next loop check.
2415/// - `prctl(PR_SET_NAME, pcomm)`: sets the leader's `task->comm`,
2416///   which becomes `task->group_leader->comm` for every spawned
2417///   thread byte-for-byte (the [`WorkSpec::pcomm`] builder rejects
2418///   names longer than 15 bytes — `TASK_COMM_LEN - 1` — so the
2419///   framework never feeds the kernel a name `__set_task_comm`
2420///   would truncate).
2421/// - `setresuid(container_uid, ...)` / `setresgid(container_gid,
2422///   ...)` (if specified) apply once before the start-byte poll.
2423/// - Threads spawn AFTER the start byte arrives, so `start()`'s
2424///   serial start-byte writes still gate work entry (the parent
2425///   moves the leader to its cgroup before sending start).
2426#[allow(clippy::too_many_arguments)]
2427pub(super) fn spawn_pcomm_container(
2428    guard: &mut SpawnGuard,
2429    pcomm: &str,
2430    container_uid: Option<u32>,
2431    container_gid: Option<u32>,
2432    groups: &[GroupParams],
2433    resources: &[PcommGroupResources],
2434) -> Result<()> {
2435    debug_assert_eq!(
2436        groups.len(),
2437        resources.len(),
2438        "spawn_pcomm_container: groups / resources must have the same length",
2439    );
2440
2441    // Allocate the container's report and start pipes. Parent holds
2442    // report_fds[0] (read end) and start_fds[1] (write end); the
2443    // container holds the inverse ends post-fork. O_CLOEXEC matches
2444    // the defense-in-depth posture used by every other pipe in the
2445    // spawn pipeline — defends against future exec paths that could
2446    // otherwise leak the fd into a helper.
2447    let mut report_fds = [0i32; 2];
2448    if unsafe { libc::pipe2(report_fds.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
2449        anyhow::bail!(
2450            "pcomm container (pcomm={pcomm:?}): report pipe2 failed: {}",
2451            std::io::Error::last_os_error(),
2452        );
2453    }
2454    // Grow the report pipe to 8 MiB so the container's `write_all` of
2455    // the JSON-encoded `Vec<WorkerReport>` issues without blocking
2456    // for moderate worker counts. For larger payloads the parent's
2457    // `read_to_end` drains concurrently with the container's
2458    // `write_all`, so a pipe overflow degrades to extra wakeups
2459    // rather than truncation. Best-effort failure (older kernel /
2460    // EPERM) leaves the default size in place — large payloads
2461    // still drain via the concurrent flow but with more wake cycles.
2462    const REPORT_PIPE_SIZE: libc::c_int = 8 * 1024 * 1024;
2463    let prev_size = unsafe { libc::fcntl(report_fds[1], libc::F_SETPIPE_SZ, REPORT_PIPE_SIZE) };
2464    if prev_size < 0 {
2465        let err = std::io::Error::last_os_error();
2466        tracing::warn!(
2467            pcomm,
2468            requested_size = REPORT_PIPE_SIZE,
2469            %err,
2470            "F_SETPIPE_SZ on pcomm container report pipe failed; falling back \
2471             to default pipe capacity",
2472        );
2473    }
2474    let mut start_fds = [0i32; 2];
2475    if unsafe { libc::pipe2(start_fds.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
2476        unsafe {
2477            libc::close(report_fds[0]);
2478            libc::close(report_fds[1]);
2479        }
2480        anyhow::bail!(
2481            "pcomm container (pcomm={pcomm:?}): start pipe2 failed: {}",
2482            std::io::Error::last_os_error(),
2483        );
2484    }
2485
2486    // Block SIGUSR1 across fork so the container inherits a blocked
2487    // mask; the post-fork install + restore pattern matches the
2488    // conventional fork worker (see `spawn_group`'s fork dispatch
2489    // for the full rationale).
2490    let mut old_mask: libc::sigset_t = unsafe { std::mem::zeroed() };
2491    let mut block_mask: libc::sigset_t = unsafe { std::mem::zeroed() };
2492    let psm_block_rc = unsafe {
2493        libc::sigemptyset(&mut block_mask);
2494        libc::sigaddset(&mut block_mask, libc::SIGUSR1);
2495        libc::pthread_sigmask(libc::SIG_BLOCK, &block_mask, &mut old_mask)
2496    };
2497    if psm_block_rc != 0 {
2498        tracing::warn!(
2499            rc = psm_block_rc,
2500            pcomm,
2501            "pthread_sigmask(SIG_BLOCK, SIGUSR1) failed pre-fork for pcomm container; \
2502             container inherits unblocked SIGUSR1 and may terminate on default \
2503             action before installing handler",
2504        );
2505    }
2506
2507    // Sum of all groups' workers — matches `WorkerReport` cardinality
2508    // the parent expects.
2509    let total_workers: usize = groups.iter().map(|g| g.num_workers).sum();
2510
2511    let pid = unsafe { libc::fork() };
2512    match pid {
2513        -1 => {
2514            let psm_restore_rc = unsafe {
2515                libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut())
2516            };
2517            if psm_restore_rc != 0 {
2518                tracing::warn!(
2519                    rc = psm_restore_rc,
2520                    "pthread_sigmask(SIG_SETMASK) failed restoring mask after pcomm \
2521                     container fork failure; SIGUSR1 may remain blocked in this thread",
2522                );
2523            }
2524            unsafe {
2525                libc::close(report_fds[0]);
2526                libc::close(report_fds[1]);
2527                libc::close(start_fds[0]);
2528                libc::close(start_fds[1]);
2529            }
2530            anyhow::bail!(
2531                "pcomm container (pcomm={pcomm:?}): fork failed: {}",
2532                std::io::Error::last_os_error(),
2533            );
2534        }
2535        0 => {
2536            // Container child. Mirror the conventional fork worker's
2537            // post-fork init: PDEATHSIG, getppid orphan check,
2538            // setpgid, SIGUSR1 handler install + mask restore. This
2539            // is the exact same defensive sequence; the only
2540            // structural difference is what happens AFTER the
2541            // start-byte handshake (thread spawn instead of inline
2542            // worker_main).
2543            unsafe {
2544                libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL);
2545            }
2546            if std::env::var_os(crate::KTSTR_GUEST_INIT_ENV).is_none()
2547                && unsafe { libc::getppid() } == 1
2548            {
2549                exit_child_success();
2550            }
2551            unsafe {
2552                libc::setpgid(0, 0);
2553            }
2554            STOP.store(false, Ordering::Relaxed);
2555            let sig_prev = unsafe {
2556                libc::signal(
2557                    libc::SIGUSR1,
2558                    sigusr1_handler as *const () as libc::sighandler_t,
2559                )
2560            };
2561            if sig_prev == libc::SIG_ERR {
2562                let errno = std::io::Error::last_os_error();
2563                eprintln!(
2564                    "ktstr: signal(SIGUSR1) install failed in pcomm container: {errno}; \
2565                     graceful stop unavailable, killpg escalation will reap"
2566                );
2567            }
2568            let psm_unblock_rc = unsafe {
2569                libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut())
2570            };
2571            if psm_unblock_rc != 0 {
2572                eprintln!(
2573                    "ktstr: pthread_sigmask(SIG_SETMASK) unblock failed in pcomm \
2574                     container: rc={psm_unblock_rc}; SIGUSR1 stays blocked, killpg \
2575                     escalation will reap"
2576                );
2577            }
2578            // Close the parent's ends. Keep report_fds[1] (the
2579            // container's write end) and start_fds[0] (the
2580            // container's read end).
2581            unsafe {
2582                libc::close(report_fds[0]);
2583                libc::close(start_fds[1]);
2584            }
2585
2586            // Inherited-fd sweep. The container forked from the
2587            // harness inherits every fd in the parent's table —
2588            // including pipe fds from OTHER live `WorkloadHandle`s
2589            // and any harness-only fds (tracing subscribers etc.).
2590            // Close everything not in the keep-set:
2591            //   - 0 / 1 / 2 (stdio).
2592            //   - `report_fds[1]` (this container's write end).
2593            //   - `start_fds[0]` (this container's read end).
2594            //   - This spawn's `pipe_pairs` (per-pair inter-worker
2595            //     fds) and `chain_pipes` (per-chain ring fds) —
2596            //     worker threads will use them once they start.
2597            //
2598            // MAP_SHARED regions (futex / iter_counters) are mmap
2599            // memory inherited via CLONE_VM, not fd entries; nothing
2600            // to keep on that axis. The sweep walks
2601            // `/proc/self/fd` once and uses raw `libc::close` on
2602            // each non-kept fd; failure to close (EBADF) is benign
2603            // — the entry is gone by the time we get to it. Heap
2604            // allocation here is acceptable post-fork (the
2605            // conventional fork worker also heap-allocates in
2606            // `worker_main`).
2607            let mut keep_fds: std::collections::BTreeSet<i32> = std::collections::BTreeSet::new();
2608            keep_fds.insert(0);
2609            keep_fds.insert(1);
2610            keep_fds.insert(2);
2611            keep_fds.insert(report_fds[1]);
2612            keep_fds.insert(start_fds[0]);
2613            for (ab, ba) in &guard.pipe_pairs {
2614                keep_fds.insert(ab[0]);
2615                keep_fds.insert(ab[1]);
2616                keep_fds.insert(ba[0]);
2617                keep_fds.insert(ba[1]);
2618            }
2619            for chain in &guard.chain_pipes {
2620                for pipe in chain {
2621                    keep_fds.insert(pipe[0]);
2622                    keep_fds.insert(pipe[1]);
2623                }
2624            }
2625            if let Ok(entries) = std::fs::read_dir("/proc/self/fd") {
2626                let mut to_close: Vec<i32> = Vec::new();
2627                for entry in entries.flatten() {
2628                    if let Some(name) = entry.file_name().to_str()
2629                        && let Ok(fd) = name.parse::<i32>()
2630                        && !keep_fds.contains(&fd)
2631                    {
2632                        to_close.push(fd);
2633                    }
2634                }
2635                for fd in to_close {
2636                    unsafe {
2637                        libc::close(fd);
2638                    }
2639                }
2640            } else {
2641                eprintln!(
2642                    "ktstr: pcomm container (pcomm={pcomm:?}): /proc/self/fd \
2643                     read_dir failed; inherited-fd sweep skipped",
2644                );
2645            }
2646
2647            // prctl(PR_SET_NAME, pcomm). Setting it on the container's
2648            // tgid leader makes `task->group_leader->comm == pcomm`
2649            // for every spawned thread (kernel/sys.c::prctl_set_name).
2650            // The WorkSpec::pcomm builder rejects empty strings,
2651            // interior NULs, and names > 15 bytes (TASK_COMM_LEN - 1)
2652            // at declaration time, so neither CString construction
2653            // nor `__set_task_comm`'s `min(strlen, TASK_COMM_LEN-1)`
2654            // truncation can fire for any value that arrives here.
2655            // `unwrap_or_default` is a defensive fall-through to an
2656            // empty CString — the kernel accepts it; the leader's
2657            // comm is set to the empty string and the failure is
2658            // surfaced to stderr.
2659            let c_pcomm = std::ffi::CString::new(pcomm).unwrap_or_default();
2660            let prctl_rc = unsafe { libc::prctl(libc::PR_SET_NAME, c_pcomm.as_ptr()) };
2661            if prctl_rc != 0 {
2662                let errno = std::io::Error::last_os_error();
2663                eprintln!(
2664                    "ktstr: prctl(PR_SET_NAME, {pcomm:?}) failed on pcomm container: {errno}",
2665                );
2666            }
2667
2668            // Apply container-level credentials. Order matters:
2669            // setresgid first (while still uid=0 we can change the
2670            // primary group), setresuid second (after the uid drop
2671            // the process loses CAP_SETGID and a later gid change
2672            // would EPERM). Failures are surfaced to stderr but do
2673            // not halt the container — the threads inside still
2674            // run with the inherited credentials, and worker_main's
2675            // own setresuid / setresgid calls (which run after this)
2676            // act as a second-line defense for any per-thread merge
2677            // that produced a different value than the container
2678            // default.
2679            if let Some(gid) = container_gid {
2680                let rc = unsafe { libc::setresgid(gid, gid, gid) };
2681                if rc != 0 {
2682                    let errno = std::io::Error::last_os_error();
2683                    eprintln!("ktstr: setresgid({gid}) failed on pcomm container: {errno}");
2684                }
2685            }
2686            if let Some(uid) = container_uid {
2687                let rc = unsafe { libc::setresuid(uid, uid, uid) };
2688                if rc != 0 {
2689                    let errno = std::io::Error::last_os_error();
2690                    eprintln!("ktstr: setresuid({uid}) failed on pcomm container: {errno}");
2691                }
2692            }
2693
2694            // Wait for the parent's start byte (poll with 30s
2695            // timeout — same budget as the conventional fork
2696            // worker). Match the conventional worker's behaviour:
2697            // on `poll <= 0` call exit_child_error so the parent's
2698            // collect path observes a missing report and emits
2699            // sentinels.
2700            let mut pfd = libc::pollfd {
2701                fd: start_fds[0],
2702                events: libc::POLLIN,
2703                revents: 0,
2704            };
2705            let ret = unsafe { libc::poll(&mut pfd, 1, 30_000) };
2706            if ret <= 0 {
2707                exit_child_error();
2708            }
2709            let mut buf = [0u8; 1];
2710            {
2711                let mut f = unsafe { std::fs::File::from_raw_fd(start_fds[0]) };
2712                let _ = f.read_exact(&mut buf);
2713                drop(f);
2714            }
2715
2716            // Unlike fork workers, the pcomm container spawns threads after
2717            // the start byte. A latched STOP from a SIGUSR1 race during mask
2718            // restore would produce N zero-work threads. Reset so threads
2719            // get a fair start.
2720            STOP.store(false, Ordering::Relaxed);
2721
2722            // Spawn worker threads for every group. Each thread
2723            // computes its own affinity / pipe-fd / futex /
2724            // iter_slot from its (group_idx, within-group index)
2725            // pair, matching the conventional fork loop's per-worker
2726            // computation but inside the container's address space
2727            // (so threads share the parent-allocated MAP_SHARED
2728            // regions and inter-worker pipes inherited via fork).
2729            // Each spawned thread gets its own `EventFd` cloned into
2730            // the closure. A `WorkerExitSignal` Drop guard inside the
2731            // closure writes 1 to that fd as the closure unwinds —
2732            // covering both the normal-return and panic paths. The
2733            // container's join phase below `epoll_wait`s on every
2734            // eventfd, so a thread's exit edge is delivered without
2735            // any polling sleep. This mirrors `join_thread_with_timeout`'s
2736            // pattern but without a timer fd: the parent's
2737            // `stop_and_collect` SIGKILL is the only timeout authority,
2738            // per the user's "event-driven only, no sleeps or
2739            // timeouts" rule.
2740            let mut joins: Vec<(
2741                std::thread::JoinHandle<WorkerReport>,
2742                std::sync::Arc<vmm_sys_util::eventfd::EventFd>,
2743            )> = Vec::with_capacity(total_workers);
2744            // Snapshot the guard's resource bookkeeping we need
2745            // inside each thread closure. The guard is borrowed
2746            // mutably by the surrounding `spawn_pcomm_cgroup` and
2747            // dropped on the parent side after the function returns;
2748            // after fork the container has its own copy of every
2749            // field, and these snapshots are clones owned by the
2750            // container's process so the closures can capture them
2751            // without aliasing the guard.
2752            let pipe_pairs_snapshot: Vec<([i32; 2], [i32; 2])> = guard.pipe_pairs.clone();
2753            let chain_pipes_snapshot: Vec<Vec<[i32; 2]>> = guard.chain_pipes.clone();
2754            let futex_ptrs_snapshot: Vec<*mut u32> = guard.futex_ptrs.clone();
2755            let iter_counters_base = guard.iter_counters;
2756            let phase_epoch_base = guard.phase_epoch;
2757
2758            for (group, res) in groups.iter().zip(resources.iter()) {
2759                let num_workers = group.num_workers;
2760                for i in 0..num_workers {
2761                    // Per-thread affinity resolution. Random-subset
2762                    // affinity samples a fresh subset per thread, so
2763                    // each call produces an independent BTreeSet.
2764                    // Fixed affinity returns the same BTreeSet for
2765                    // every thread.
2766                    //
2767                    // resolve_affinity bails on caller bugs (count==0,
2768                    // empty pool). In the pcomm container's forked-
2769                    // child context we cannot propagate Err up; the
2770                    // convention is eprintln + [`exit_child_error`]
2771                    // (the named helper used by every forked-child
2772                    // failure site in this file — start-byte
2773                    // poll-timeout bail, EventFd::new bail,
2774                    // thread::Builder::spawn bail) so the parent
2775                    // collects a missing-report rather than silently
2776                    // spawning workers without the affinity the test
2777                    // author requested. The prior degrade-to-None
2778                    // behaviour was a silent-drop bug we close here.
2779                    let affinity = match resolve_affinity(&group.affinity) {
2780                        Ok(a) => a,
2781                        Err(e) => {
2782                            eprintln!(
2783                                "ktstr: pcomm container (group {}): resolve_affinity \
2784                                 for thread {i}/{num_workers} failed: {e:#}. \
2785                                 Aborting container — refusing to spawn workers \
2786                                 without the requested affinity (no silent drops).",
2787                                group.group_idx,
2788                            );
2789                            // exit_group(2) tears down every sibling
2790                            // worker thread spawned in earlier loop
2791                            // iterations along with this one — the
2792                            // intended fail-stop behaviour (parent
2793                            // observes a missing report and surfaces
2794                            // a sentinel).
2795                            exit_child_error();
2796                        }
2797                    };
2798
2799                    let worker_pipe_fds: Option<(i32, i32)> = if res.needs_pipes {
2800                        let pair_idx = res.pipe_pair_base + i / 2;
2801                        let (ab, ba) = &pipe_pairs_snapshot[pair_idx];
2802                        if i % 2 == 0 {
2803                            Some((ba[0], ab[1]))
2804                        } else {
2805                            Some((ab[0], ba[1]))
2806                        }
2807                    } else if let Some(depth) = res.chain_depth
2808                        && depth > 0
2809                    {
2810                        let chain_idx = res.chain_pipes_base + i / depth;
2811                        let stage = i % depth;
2812                        let prev_stage = (stage + depth - 1) % depth;
2813                        let chain = &chain_pipes_snapshot[chain_idx];
2814                        Some((chain[prev_stage][0], chain[stage][1]))
2815                    } else {
2816                        None
2817                    };
2818
2819                    let worker_futex: Option<(*mut u32, usize)> = if res.needs_futex {
2820                        let futex_group_idx = res.futex_ptrs_base + i / res.futex_group_size;
2821                        let pos = i % res.futex_group_size;
2822                        Some((futex_ptrs_snapshot[futex_group_idx], pos))
2823                    } else {
2824                        None
2825                    };
2826
2827                    let iter_slot: *mut AtomicU64 = if !iter_counters_base.is_null() {
2828                        unsafe { iter_counters_base.add(res.iter_offset + i) }
2829                    } else {
2830                        std::ptr::null_mut()
2831                    };
2832
2833                    // Round raw pointers through Send-newtypes so
2834                    // the closure's auto-Send check passes (same
2835                    // wrapper pattern `spawn_thread_worker` uses).
2836                    let futex_send = SendFutexPtr::new(worker_futex);
2837                    let iter_slot_send = SendIterSlotPtr::new(iter_slot);
2838                    let phase_epoch_send = SendPhaseEpochPtr::new(phase_epoch_base);
2839
2840                    let work_type = group.work_type.clone();
2841                    let park_after = group.park_after;
2842                    let sched_policy = group.sched_policy;
2843                    let mem_policy = group.mem_policy.clone();
2844                    let mpol_flags = group.mpol_flags;
2845                    let nice = group.nice;
2846                    let comm = group.comm.clone();
2847                    let uid = group.uid;
2848                    let gid = group.gid;
2849                    let numa_node = group.numa_node;
2850                    let group_idx = group.group_idx;
2851
2852                    // Per-thread exit eventfd. Cloned into the
2853                    // closure as `Arc<EventFd>`; the closure-local
2854                    // `WorkerExitSignal` Drop guard writes 1 to the
2855                    // fd on every exit path (normal return AND
2856                    // panic). The container's join phase below
2857                    // `epoll_wait`s on every eventfd to deliver the
2858                    // exit edge without polling. EFD_NONBLOCK keeps
2859                    // the Drop-time `write` from blocking — counter
2860                    // mode means the writer just bumps the counter
2861                    // without losing the edge.
2862                    let exit_evt = match vmm_sys_util::eventfd::EventFd::new(libc::EFD_NONBLOCK) {
2863                        Ok(efd) => std::sync::Arc::new(efd),
2864                        Err(e) => {
2865                            // Hard fail: continuing with fewer
2866                            // threads silently breaks the
2867                            // workload's worker count and the
2868                            // parent's report count. exit_child_error
2869                            // so the parent's sentinel path observes
2870                            // a missing payload and emits one
2871                            // sentinel per expected report. Reuses
2872                            // the same fail-stop contract as the
2873                            // start-byte poll timeout (`ret <=
2874                            // 0`).
2875                            eprintln!(
2876                                "ktstr: pcomm container (group {}): EventFd::new for \
2877                                 worker {}/{num_workers} failed: {e}; aborting container",
2878                                group.group_idx,
2879                                i + 1,
2880                            );
2881                            exit_child_error();
2882                        }
2883                    };
2884                    let exit_evt_thread = std::sync::Arc::clone(&exit_evt);
2885
2886                    let join = std::thread::Builder::new()
2887                        .name(format!("ktstr-pcomm-g{group_idx}-{i}"))
2888                        .spawn(move || {
2889                            // Drop guard: bumps the eventfd as the
2890                            // closure unwinds. The write is
2891                            // non-blocking (EFD_NONBLOCK) and the
2892                            // counter accumulates, so a missed
2893                            // read just queues the next read's
2894                            // value — no edge loss. Drop runs
2895                            // under both normal return and
2896                            // unwinding (panic), so the container's
2897                            // join phase observes EVERY thread's
2898                            // exit, including panicked ones.
2899                            struct WorkerExitSignal(std::sync::Arc<vmm_sys_util::eventfd::EventFd>);
2900                            impl Drop for WorkerExitSignal {
2901                                fn drop(&mut self) {
2902                                    let _ = self.0.write(1);
2903                                }
2904                            }
2905                            let _exit_signal = WorkerExitSignal(exit_evt_thread);
2906
2907                            let futex = futex_send.into_raw();
2908                            let slot = iter_slot_send.into_raw();
2909                            let epoch = phase_epoch_send.into_raw();
2910                            worker_main(
2911                                affinity,
2912                                work_type,
2913                                sched_policy,
2914                                mem_policy,
2915                                mpol_flags,
2916                                nice,
2917                                comm.as_deref(),
2918                                uid,
2919                                gid,
2920                                numa_node,
2921                                worker_pipe_fds,
2922                                futex,
2923                                park_after,
2924                                // pcomm containers run threads inside a
2925                                // forked container; the first-iteration
2926                                // eventfd is allocated only on the plain
2927                                // fork-dispatch path, so pcomm workers never
2928                                // signal.
2929                                -1,
2930                                slot,
2931                                epoch,
2932                                &STOP,
2933                                group_idx,
2934                            )
2935                        });
2936                    match join {
2937                        Ok(j) => joins.push((j, exit_evt)),
2938                        Err(e) => {
2939                            // Hard fail: see EventFd path above.
2940                            // A partially-spawned container leaves
2941                            // the parent guessing about which
2942                            // reports are real vs missing;
2943                            // exit_child_error collapses to the
2944                            // parent's sentinel path with
2945                            // deterministic cardinality.
2946                            eprintln!(
2947                                "ktstr: pcomm container (group {}): thread::spawn for \
2948                                 worker {}/{num_workers} failed: {e}; aborting container",
2949                                group.group_idx,
2950                                i + 1,
2951                            );
2952                            exit_child_error();
2953                        }
2954                    }
2955                }
2956            }
2957
2958            // Publish the ready byte AFTER every worker thread is
2959            // alive. The parent's barrier polls every report fd for
2960            // POLLIN with a bounded deadline; the byte's correct
2961            // semantic is "the container has spawned every worker
2962            // thread and they are now running" — earlier (pre-spawn)
2963            // is observable to the parent before threads exist, which
2964            // races with `set_affinity(idx)` / `worker_pids()` calls
2965            // the harness may issue between the parent's barrier wake
2966            // and the work loop. Each thread starts work immediately
2967            // upon spawn (no per-thread handshake gate) since the
2968            // CONTAINER's start-byte poll above is the workload's
2969            // single start gate; once the container observes the
2970            // start byte, every thread it spawns is by construction
2971            // intended to run. Done as a raw `libc::write` to avoid
2972            // taking ownership of `report_fds[1]` (we still need it
2973            // for the JSON write below).
2974            let ready_byte: u8 = b'r';
2975            unsafe {
2976                libc::write(
2977                    report_fds[1],
2978                    &ready_byte as *const u8 as *const libc::c_void,
2979                    1,
2980                );
2981            }
2982
2983            // Join every thread via `epoll_wait` on per-thread exit
2984            // eventfds. Each thread's `WorkerExitSignal` Drop guard
2985            // writes 1 to its fd as the closure unwinds (normal
2986            // return AND panic), so the container's `epoll_wait`
2987            // observes EVERY thread's exit edge without polling.
2988            // No timeout: per the user's "event-driven only, no
2989            // sleeps or timeouts" rule, the container blocks in
2990            // `epoll_wait(-1)` until each fd fires. The parent's
2991            // `stop_and_collect` 5s collect deadline + SIGKILL is
2992            // the only timeout authority; if a thread hangs (no
2993            // Drop guard fires — no realistic path to that under
2994            // the closures we spawn), the parent SIGKILLs the
2995            // container, the kernel tears down every thread, and
2996            // the container exits via the signal.
2997            //
2998            // Setup-failure fallback: if `Epoll::new` or any
2999            // `epoll.ctl(Add)` fails, fall back to blocking
3000            // `JoinHandle::join` per thread — that itself is
3001            // event-driven from the kernel's perspective (the
3002            // syscall blocks on the thread's exit signal, no
3003            // userspace polling). The fallback path is rare
3004            // (epoll_create1 failure means out of fds or out of
3005            // memory, both of which are pre-existing failure modes
3006            // the conventional fork worker also has).
3007            use std::os::unix::io::AsRawFd;
3008            use vmm_sys_util::epoll::{ControlOperation, Epoll, EpollEvent, EventSet};
3009            let mut indexed_joins: Vec<(
3010                usize,
3011                std::thread::JoinHandle<WorkerReport>,
3012                std::sync::Arc<vmm_sys_util::eventfd::EventFd>,
3013            )> = joins
3014                .into_iter()
3015                .enumerate()
3016                .map(|(i, (j, evt))| (i, j, evt))
3017                .collect();
3018            let mut reports: Vec<WorkerReport> = Vec::with_capacity(indexed_joins.len());
3019
3020            let epoll_setup: Option<Epoll> = match Epoll::new() {
3021                Ok(ep) => {
3022                    let mut ok = true;
3023                    for (idx, _, evt) in &indexed_joins {
3024                        if let Err(e) = ep.ctl(
3025                            ControlOperation::Add,
3026                            evt.as_raw_fd(),
3027                            EpollEvent::new(EventSet::IN, *idx as u64),
3028                        ) {
3029                            eprintln!(
3030                                "ktstr: pcomm container (pcomm={pcomm:?}): epoll.ctl(Add) \
3031                                 for thread {idx} failed: {e}; falling back to blocking \
3032                                 join per thread",
3033                            );
3034                            ok = false;
3035                            break;
3036                        }
3037                    }
3038                    if ok { Some(ep) } else { None }
3039                }
3040                Err(e) => {
3041                    eprintln!(
3042                        "ktstr: pcomm container (pcomm={pcomm:?}): Epoll::new failed: {e}; \
3043                         falling back to blocking join per thread",
3044                    );
3045                    None
3046                }
3047            };
3048
3049            if let Some(epoll) = epoll_setup {
3050                // Event-driven join loop. `epoll_wait(-1, …)` blocks
3051                // until at least one eventfd fires, which only
3052                // happens when a thread's `WorkerExitSignal` Drop
3053                // guard runs. The wake delivers `events[k].data()`
3054                // as the index of the finished thread; we
3055                // deregister, locate the entry, and `join()` it
3056                // (the `join()` call may briefly wait for the OS
3057                // thread to fully exit after the Drop guard, but
3058                // that wait is itself event-driven inside the
3059                // kernel — no userspace polling).
3060                let mut events_buf: Vec<EpollEvent> =
3061                    vec![EpollEvent::default(); indexed_joins.len().max(1)];
3062                while !indexed_joins.is_empty() {
3063                    let n = match epoll.wait(-1, &mut events_buf) {
3064                        Ok(n) => n,
3065                        Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
3066                        Err(e) => {
3067                            eprintln!(
3068                                "ktstr: pcomm container (pcomm={pcomm:?}): epoll.wait \
3069                                 failed: {e}; falling back to blocking join for the \
3070                                 remaining {} thread(s)",
3071                                indexed_joins.len(),
3072                            );
3073                            break;
3074                        }
3075                    };
3076                    for ev in &events_buf[..n] {
3077                        let target_idx = ev.data() as usize;
3078                        let pos = match indexed_joins
3079                            .iter()
3080                            .position(|(idx, _, _)| *idx == target_idx)
3081                        {
3082                            Some(p) => p,
3083                            None => continue,
3084                        };
3085                        let (idx, j, evt) = indexed_joins.swap_remove(pos);
3086                        if let Err(e) = epoll.ctl(
3087                            ControlOperation::Delete,
3088                            evt.as_raw_fd(),
3089                            EpollEvent::default(),
3090                        ) {
3091                            eprintln!(
3092                                "ktstr: pcomm container (pcomm={pcomm:?}): epoll.ctl(Del) \
3093                                 for thread {idx} failed: {e}",
3094                            );
3095                        }
3096                        let _ = evt.read();
3097                        match j.join() {
3098                            Ok(r) => reports.push(r),
3099                            Err(payload) => {
3100                                let msg = extract_panic_payload(payload);
3101                                eprintln!(
3102                                    "ktstr: pcomm container (pcomm={pcomm:?}): thread {idx} \
3103                                     panicked: {msg}",
3104                                );
3105                                reports.push(WorkerReport {
3106                                    completed: false,
3107                                    exit_info: Some(WorkerExitInfo::Panicked(msg)),
3108                                    ..WorkerReport::default()
3109                                });
3110                            }
3111                        }
3112                    }
3113                }
3114            }
3115            // Either the epoll path completed (drained
3116            // `indexed_joins`) or it bailed mid-flight; either way,
3117            // any remaining handles are joined by the kernel-blocking
3118            // `JoinHandle::join` below. `join` is event-driven inside
3119            // the kernel (futex on the thread's TID); no userspace
3120            // polling.
3121            for (idx, j, _evt) in indexed_joins {
3122                match j.join() {
3123                    Ok(r) => reports.push(r),
3124                    Err(payload) => {
3125                        let msg = extract_panic_payload(payload);
3126                        eprintln!(
3127                            "ktstr: pcomm container (pcomm={pcomm:?}): thread {idx} \
3128                             panicked: {msg}",
3129                        );
3130                        reports.push(WorkerReport {
3131                            completed: false,
3132                            exit_info: Some(WorkerExitInfo::Panicked(msg)),
3133                            ..WorkerReport::default()
3134                        });
3135                    }
3136                }
3137            }
3138
3139            // Encode and write the report stream as a single
3140            // `Vec<WorkerReport>` JSON document via `serde_json`.
3141            // serde_json for the pcomm `Vec<WorkerReport>`; the
3142            // fork-mode workers use postcard for the single
3143            // `WorkerReport`. The pcomm container is a
3144            // fork-mode child and its payload sits on the same
3145            // per-child pipe used by every other forked worker.
3146            // The parent decodes via
3147            // `serde_json::from_slice::<Vec<WorkerReport>>`. Encode
3148            // failures fall through to exit_child_success — the
3149            // parent's sentinel path handles missing / truncated
3150            // payloads.
3151            // tracing is unsafe in a forked child (the parent's
3152            // subscriber may hold a lock); use eprintln + raw fd.
3153            let bytes = match serde_json::to_vec(&reports) {
3154                Ok(v) => v,
3155                Err(e) => {
3156                    eprintln!(
3157                        "ktstr: pcomm container (pcomm={pcomm:?}): serde_json encode \
3158                         of {} reports failed: {e}",
3159                        reports.len(),
3160                    );
3161                    Vec::new()
3162                }
3163            };
3164            {
3165                let mut f = unsafe { std::fs::File::from_raw_fd(report_fds[1]) };
3166                if let Err(e) = f.write_all(&bytes) {
3167                    eprintln!(
3168                        "ktstr: pcomm container (pcomm={pcomm:?}): report write_all \
3169                         of {} bytes failed: {e}",
3170                        bytes.len(),
3171                    );
3172                }
3173                drop(f);
3174            }
3175            exit_child_success();
3176        }
3177        child_pid => {
3178            // Parent. Restore signal mask, close the wrong ends,
3179            // record the container in `guard.children` so its
3180            // lifecycle (kill on bail / collect on stop) is shared
3181            // with conventional workers.
3182            let psm_parent_restore_rc = unsafe {
3183                libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut())
3184            };
3185            if psm_parent_restore_rc != 0 {
3186                tracing::warn!(
3187                    rc = psm_parent_restore_rc,
3188                    "pthread_sigmask(SIG_SETMASK) failed restoring mask in parent \
3189                     post-pcomm-fork; SIGUSR1 stays blocked in this thread for \
3190                     the lifetime of the workload",
3191                );
3192            }
3193            unsafe {
3194                libc::close(report_fds[1]);
3195                libc::close(start_fds[0]);
3196            }
3197            // Per-group layout for sentinel distribution. Each
3198            // entry is `(group_idx, num_workers)`; total report
3199            // count is the sum of `num_workers` across the layout
3200            // (== `total_workers` invariant). When the parent
3201            // emits sentinels for a missing JSON payload, the
3202            // layout drives per-group_idx tagging so per-group
3203            // filters partition correctly even on a failure.
3204            let group_layout: Vec<(usize, usize)> = groups
3205                .iter()
3206                .map(|g| (g.group_idx, g.num_workers))
3207                .collect();
3208            debug_assert_eq!(
3209                group_layout.iter().map(|(_, n)| n).sum::<usize>(),
3210                total_workers,
3211                "spawn_pcomm_container: group_layout total must match total_workers",
3212            );
3213            guard.children.push(ForkedChild {
3214                pid: child_pid,
3215                report_fd: report_fds[0],
3216                start_fd: start_fds[1],
3217                kind: ForkedChildKind::PcommContainer {
3218                    groups: group_layout,
3219                },
3220            });
3221            Ok(())
3222        }
3223    }
3224}
3225
3226/// Internal dispatch shape resolved from
3227/// [`WorkloadConfig::clone_mode`] inside [`WorkloadHandle::spawn`].
3228pub(super) enum Dispatch {
3229    Fork,
3230    Thread,
3231}
3232
3233impl WorkloadHandle {
3234    /// Fork one thread-group leader hosting every worker in `works`
3235    /// as worker threads inside a single forked process. Used by
3236    /// `apply_setup` when a `CgroupDef` declares `pcomm` — every
3237    /// `WorkSpec` in the same `CgroupDef` is coalesced into one
3238    /// thread-group leader whose `task->comm` carries `pcomm`
3239    /// exactly (the [`WorkSpec::pcomm`] builder rejects > 15 bytes
3240    /// — `TASK_COMM_LEN - 1` — so the framework never feeds the
3241    /// kernel a name `__set_task_comm` would truncate). Every
3242    /// spawned thread reads its `task->group_leader->comm` as
3243    /// `pcomm` for the leader's lifetime.
3244    ///
3245    /// `works` must already be fully resolved: each entry's
3246    /// `num_workers` must be `Some(_)` and `affinity` must be
3247    /// non-topology-aware (Inherit / Exact / RandomSubset).
3248    /// `apply_setup` runs the standard scenario-engine resolution
3249    /// (`resolve_num_workers` + `intent_for_spawn`) before calling
3250    /// in.
3251    ///
3252    /// `container_uid` / `container_gid` apply to the leader's
3253    /// process credentials via `setresuid` / `setresgid` once,
3254    /// inside the forked leader, before threads spawn. Each worker
3255    /// thread additionally re-applies its merged uid/gid inside
3256    /// `worker_main` at thread creation time; for the common case
3257    /// of a single CgroupDef-level default flowing through
3258    /// `merged_works`, the per-thread call is idempotent.
3259    ///
3260    /// On `works.is_empty()` the function returns a handle with no
3261    /// children — no fork, no resource allocation. Same for the
3262    /// empty-thread case across ALL groups (every `num_workers ==
3263    /// 0`): the leader is skipped because there is no work to host.
3264    /// This matches the `pcomm_zero_workers_no_container_spawn`
3265    /// contract.
3266    ///
3267    /// Group-level admission rejects WorkType variants that conflict
3268    /// with the threaded shape (every worker shares the leader's
3269    /// tgid):
3270    ///
3271    /// - [`WorkType::ForkExit`]: a fork from a thread of a
3272    ///   multi-threaded process inherits all locks held by other
3273    ///   threads at fork time, which the child cannot release;
3274    ///   safe-but-degraded use cases would still need
3275    ///   `CloneMode::Fork` for clean lock-free child state.
3276    /// - [`WorkType::CgroupChurn`]: writing the worker tid to
3277    ///   `cgroup.procs` migrates the entire leader tgid (every
3278    ///   sibling thread) — the test loses control of its own
3279    ///   cgroup placement.
3280    pub fn spawn_pcomm_cgroup(
3281        pcomm: &str,
3282        container_uid: Option<u32>,
3283        container_gid: Option<u32>,
3284        works: &[WorkSpec],
3285    ) -> Result<Self> {
3286        // Empty pcomm input is treated as "no pcomm" by the
3287        // apply_setup caller; reject here as a safety belt — every
3288        // production caller has already filtered empty strings.
3289        if pcomm.is_empty() {
3290            anyhow::bail!(
3291                "spawn_pcomm_cgroup: pcomm must be a non-empty string; \
3292                 the caller (apply_setup) treats `Some(\"\")` as \
3293                 `None` and falls through to the conventional fork \
3294                 spawn — empty here is a programmer error.",
3295            );
3296        }
3297
3298        // Structural mem_policy validation per-spec — same gate as
3299        // [`WorkloadHandle::spawn`]'s [`WorkloadConfig::validate`],
3300        // applied at the second public spawn entry point. Closes the
3301        // bypass that would otherwise allow invalid mem_policy via
3302        // this entry to reach `apply_mempolicy_with_flags`'s silent-
3303        // skip arm. The `exit_group(2)` thread-mode unsoundness
3304        // documented on [`WorkloadConfig::validate`] does not apply
3305        // here (pcomm-container workers run inside a forked container
3306        // with a new tgid distinct from the test runner's — the
3307        // container forks once via `libc::fork`, then workers are
3308        // spawned as threads sharing the container's tgid; a
3309        // hypothetical inner `_exit(1)` would only tear down the
3310        // container, not the test runner), but the validation
3311        // symmetry between the two pub entries prevents future
3312        // refactors from inadvertently re-opening the bypass.
3313        for (i, spec) in works.iter().enumerate() {
3314            spec.mem_policy.validate().map_err(|e| {
3315                anyhow::anyhow!("WorkloadHandle::spawn_pcomm_cgroup: works[{i}].mem_policy: {e}",)
3316            })?;
3317        }
3318
3319        // Build a `GroupParams` per `WorkSpec` using the same
3320        // resolver `WorkloadHandle::spawn` uses for composed
3321        // entries. group_idx is the entry's position in `works` (0-
3322        // based); the parent has no notion of "primary" inside a
3323        // pcomm container — every entry is a peer.
3324        let mut groups: Vec<GroupParams> = Vec::with_capacity(works.len());
3325        for (i, spec) in works.iter().enumerate() {
3326            let num_workers = spec.num_workers.ok_or_else(|| {
3327                anyhow::anyhow!(
3328                    "spawn_pcomm_cgroup: works[{i}].num_workers must be set explicitly \
3329                     (apply_setup runs resolve_num_workers before calling in)",
3330                )
3331            })?;
3332            let site = format!("works[{i}].affinity");
3333            let affinity = GroupParams::resolve_spawn_affinity(&spec.affinity, &site)?;
3334            groups.push(GroupParams::from_work_spec(spec, i, affinity, num_workers));
3335        }
3336
3337        // Per-group admission. Mirrors `WorkloadHandle::spawn` for
3338        // shared rules (worker_group_size divisibility,
3339        // chain-depth-< 2, IdleChurn / IpcVariance zero rejection)
3340        // and adds pcomm-specific rejections (ForkExit,
3341        // CgroupChurn) since the container's threaded shape —
3342        // every worker is a thread of one forked tgid sharing one
3343        // fd table — introduces hazards those variants cannot
3344        // tolerate.
3345        for group in &groups {
3346            if matches!(group.work_type, WorkType::ForkExit) {
3347                anyhow::bail!(
3348                    "WorkSpec::pcomm is incompatible with WorkType::ForkExit \
3349                     (works[{}]): a fork from a thread of a multi-threaded \
3350                     container inherits all locks held by sibling threads at \
3351                     fork time, producing undefined behaviour for any libc \
3352                     primitive in the child. Drop pcomm or pick a different \
3353                     work type.",
3354                    group.group_idx,
3355                );
3356            }
3357            if matches!(group.work_type, WorkType::CgroupChurn { .. }) {
3358                anyhow::bail!(
3359                    "WorkSpec::pcomm is incompatible with WorkType::CgroupChurn \
3360                     (works[{}]): CgroupChurn writes the worker tid to \
3361                     `cgroup.procs`, which the kernel resolves to the whole \
3362                     tgid and migrates the entire pcomm container (every \
3363                     sibling thread). Drop pcomm or pick a different work type.",
3364                    group.group_idx,
3365                );
3366            }
3367            validate_workload_admission(group)?;
3368        }
3369
3370        // No-work shortcut: every group has zero workers (or the
3371        // works list itself is empty). Fork would produce a
3372        // container that blocks on the start byte forever and
3373        // outlives the test handle; skip it. The handle still
3374        // has a sensible Drop (empty children/threads/regions).
3375        let total_workers: usize = groups.iter().map(|g| g.num_workers).sum();
3376        if total_workers == 0 {
3377            return Ok(SpawnGuard::new().into_handle());
3378        }
3379
3380        // All failable acquisitions route through `guard`. If any
3381        // `?` returns early, the guard's Drop SIGKILLs+reaps any
3382        // already-forked container, closes open pipe fds, and
3383        // munmaps the shared regions — so no leak on a mid-spawn
3384        // error path.
3385        let mut guard = SpawnGuard::new();
3386
3387        // Per-worker iteration counter region (MAP_SHARED). Sized
3388        // for ALL groups' workers laid out contiguously; matches
3389        // the `WorkloadHandle::spawn` allocation but with a
3390        // pcomm-specific diagnostic prefix.
3391        let size = total_workers * std::mem::size_of::<AtomicU64>();
3392        let ptr = unsafe {
3393            libc::mmap(
3394                std::ptr::null_mut(),
3395                size,
3396                libc::PROT_READ | libc::PROT_WRITE,
3397                libc::MAP_SHARED | libc::MAP_ANONYMOUS,
3398                -1,
3399                0,
3400            )
3401        };
3402        if ptr == libc::MAP_FAILED {
3403            let errno = std::io::Error::last_os_error();
3404            let hint = mmap_shared_anon_errno_hint(errno.raw_os_error());
3405            anyhow::bail!(
3406                "mmap(MAP_SHARED|MAP_ANONYMOUS, {size} bytes) for the \
3407                 per-worker iter_counters region failed: {errno}{hint}; \
3408                 this region holds one AtomicU64 per worker thread \
3409                 ({total_workers} thread(s) inside the pcomm={pcomm:?} \
3410                 container) so the parent can snapshot iteration \
3411                 counts via `snapshot_iterations()`.",
3412            );
3413        }
3414        guard.iter_counters = ptr as *mut AtomicU64;
3415        guard.iter_counter_bytes = size;
3416
3417        // Single shared phase-epoch word (MAP_SHARED, zero-init to
3418        // 0 = BASELINE) — same role as in `WorkloadHandle::spawn`: the
3419        // scenario engine bumps it per step boundary and backdrop
3420        // workers read it for per-phase attribution.
3421        let epoch_size = std::mem::size_of::<AtomicU32>();
3422        let epoch_ptr = unsafe {
3423            libc::mmap(
3424                std::ptr::null_mut(),
3425                epoch_size,
3426                libc::PROT_READ | libc::PROT_WRITE,
3427                libc::MAP_SHARED | libc::MAP_ANONYMOUS,
3428                -1,
3429                0,
3430            )
3431        };
3432        if epoch_ptr == libc::MAP_FAILED {
3433            let errno = std::io::Error::last_os_error();
3434            let hint = mmap_shared_anon_errno_hint(errno.raw_os_error());
3435            anyhow::bail!(
3436                "mmap(MAP_SHARED|MAP_ANONYMOUS, {epoch_size} bytes) for the \
3437                 phase-epoch word (pcomm={pcomm:?}) failed: {errno}{hint}; \
3438                 this single AtomicU32 broadcasts the current phase to \
3439                 backdrop workers for per-phase attribution.",
3440            );
3441        }
3442        guard.phase_epoch = epoch_ptr as *mut AtomicU32;
3443        guard.phase_epoch_bytes = epoch_size;
3444
3445        // Per-group resource allocation (pipe pairs, chain pipes,
3446        // futex regions). Same shape as `WorkloadHandle::spawn_group`'s
3447        // prologue but inlined here so the per-group bookkeeping
3448        // (`PcommGroupResources`) is built directly for
3449        // `spawn_pcomm_container`. `iter_offset` runs across groups
3450        // and tracks each group's first iter_slot.
3451        let mut resources: Vec<PcommGroupResources> = Vec::with_capacity(groups.len());
3452        let mut iter_offset: usize = 0;
3453        for group in &groups {
3454            let needs_pipes = matches!(
3455                group.work_type,
3456                WorkType::PipeIo { .. } | WorkType::CachePipe { .. }
3457            );
3458            let chain_depth = group.work_type.chain_pipe_depth();
3459            let needs_futex = group.work_type.needs_shared_mem();
3460            let pipe_pair_base = guard.pipe_pairs.len();
3461            let chain_pipes_base = guard.chain_pipes.len();
3462            let futex_ptrs_base = guard.futex_ptrs.len();
3463            let futex_region_size = futex_region_size_for(&group.work_type);
3464            let futex_group_size = group.work_type.worker_group_size().unwrap_or(2);
3465
3466            if needs_pipes {
3467                for _ in 0..group.num_workers / 2 {
3468                    let mut ab = [0i32; 2];
3469                    if unsafe { libc::pipe2(ab.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
3470                        anyhow::bail!(
3471                            "pipe2 (pcomm={pcomm:?}, group {}) failed: {}",
3472                            group.group_idx,
3473                            std::io::Error::last_os_error(),
3474                        );
3475                    }
3476                    let mut ba = [0i32; 2];
3477                    if unsafe { libc::pipe2(ba.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
3478                        unsafe {
3479                            libc::close(ab[0]);
3480                            libc::close(ab[1]);
3481                        }
3482                        anyhow::bail!(
3483                            "pipe2 (pcomm={pcomm:?}, group {}) failed: {}",
3484                            group.group_idx,
3485                            std::io::Error::last_os_error(),
3486                        );
3487                    }
3488                    guard.pipe_pairs.push((ab, ba));
3489                }
3490            }
3491
3492            if let Some(depth) = chain_depth
3493                && depth > 0
3494                && group.num_workers >= depth
3495            {
3496                let chains = group.num_workers / depth;
3497                for _ in 0..chains {
3498                    let mut chain: Vec<[i32; 2]> = Vec::with_capacity(depth);
3499                    let mut alloc_ok = true;
3500                    for _ in 0..depth {
3501                        let mut p = [0i32; 2];
3502                        if unsafe { libc::pipe2(p.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
3503                            alloc_ok = false;
3504                            break;
3505                        }
3506                        chain.push(p);
3507                    }
3508                    if !alloc_ok {
3509                        for p in &chain {
3510                            unsafe {
3511                                libc::close(p[0]);
3512                                libc::close(p[1]);
3513                            }
3514                        }
3515                        anyhow::bail!(
3516                            "WakeChain pipe2 (pcomm={pcomm:?}, group {}) failed: {}",
3517                            group.group_idx,
3518                            std::io::Error::last_os_error(),
3519                        );
3520                    }
3521                    guard.chain_pipes.push(chain);
3522                }
3523            }
3524
3525            if needs_futex {
3526                for _ in 0..group.num_workers / futex_group_size {
3527                    let region = unsafe {
3528                        libc::mmap(
3529                            std::ptr::null_mut(),
3530                            futex_region_size,
3531                            libc::PROT_READ | libc::PROT_WRITE,
3532                            libc::MAP_SHARED | libc::MAP_ANONYMOUS,
3533                            -1,
3534                            0,
3535                        )
3536                    };
3537                    if region == libc::MAP_FAILED {
3538                        let errno = std::io::Error::last_os_error();
3539                        let hint = mmap_shared_anon_errno_hint(errno.raw_os_error());
3540                        anyhow::bail!(
3541                            "mmap(MAP_SHARED|MAP_ANONYMOUS, {futex_region_size} bytes) \
3542                             for a futex shared-memory region (pcomm={pcomm:?}, group {}) \
3543                             failed: {errno}{hint}",
3544                            group.group_idx,
3545                        );
3546                    }
3547                    unsafe {
3548                        std::ptr::write_bytes(region as *mut u8, 0, futex_region_size);
3549                    }
3550                    guard.futex_ptrs.push(region as *mut u32);
3551                    guard.futex_region_sizes.push(futex_region_size);
3552                }
3553            }
3554
3555            resources.push(PcommGroupResources {
3556                iter_offset,
3557                pipe_pair_base,
3558                chain_pipes_base,
3559                futex_ptrs_base,
3560                needs_pipes,
3561                chain_depth,
3562                needs_futex,
3563                futex_group_size,
3564            });
3565            iter_offset += group.num_workers;
3566        }
3567
3568        // Fork the container and spawn its threads. On success the
3569        // container is registered in `guard.children`; on failure the
3570        // guard's Drop reaps it.
3571        spawn_pcomm_container(
3572            &mut guard,
3573            pcomm,
3574            container_uid,
3575            container_gid,
3576            &groups,
3577            &resources,
3578        )?;
3579
3580        Ok(guard.into_handle())
3581    }
3582
3583    /// Spawn worker tasks. Workers block until
3584    /// [`start()`](Self::start) is called, allowing the caller to
3585    /// move fork-mode workers into cgroups first. The worker creation
3586    /// primitive (`fork` or `std::thread::spawn`) is selected by
3587    /// [`WorkloadConfig::clone_mode`].
3588    pub fn spawn(config: &WorkloadConfig) -> Result<Self> {
3589        // Reject invalid mem_policy before any worker context exists;
3590        // see [`WorkloadConfig::validate`] for the silent-skip
3591        // rationale and why a worker-side `_exit(1)` fix is unsound.
3592        config.validate()?;
3593
3594        let dispatch = match &config.clone_mode {
3595            CloneMode::Fork => Dispatch::Fork,
3596            CloneMode::Thread => Dispatch::Thread,
3597        };
3598
3599        // Build the per-group params list: primary first
3600        // (group_idx == 0), then composed[k] resolved into
3601        // group_idx == k+1. The resolver enforces the "spawn-time
3602        // resolution rules" documented on
3603        // [`WorkloadConfig::composed`] (Q1: num_workers must be
3604        // explicit; Q2: only Inherit/Exact affinity reachable from
3605        // spawn() — topology-aware variants need the scenario
3606        // engine).
3607        //
3608        // Every group inherits the parent
3609        // [`WorkloadConfig::clone_mode`]: SpawnGuard's lifecycle
3610        // assumes a single dispatch path (every guard.children
3611        // entry is a fork-mode child reaped via waitpid; every
3612        // guard.threads entry is a thread-mode worker joined via
3613        // JoinHandle). Mixing modes inside one guard would route
3614        // teardown through the wrong code path, so [`WorkSpec`]
3615        // carries no `clone_mode` field — it is a workload-wide
3616        // property fixed by [`WorkloadConfig::clone_mode`].
3617        let mut groups: Vec<GroupParams> = Vec::with_capacity(1 + config.composed.len());
3618        groups.push(GroupParams::primary(config)?);
3619        for (i, spec) in config.composed.iter().enumerate() {
3620            groups.push(GroupParams::from_composed(spec, i + 1)?);
3621        }
3622
3623        // Per-group admission. Each group's work_type is checked
3624        // independently — a malformed composed entry bails the
3625        // whole workload before any resources are acquired.
3626        for group in &groups {
3627            // Thread mode + ForkExit is incompatible. ForkExit's worker
3628            // body calls `libc::fork()` from inside `worker_main` to
3629            // exercise the fork -> exit -> wait lifecycle. Under
3630            // [`CloneMode::Thread`] the worker is one thread of the
3631            // harness's multi-threaded tgid, and a `fork()` from a thread
3632            // of a multi-threaded process duplicates ONLY the calling
3633            // thread: any lock another harness thread holds at fork time
3634            // (glibc malloc arena, internal mutexes) stays locked forever
3635            // in the child, which has no thread left to release it. The
3636            // child here only `_exit`s (async-signal-safe), so it survives
3637            // this in practice — but the hazard is real and the
3638            // fork -> exit primitive is faithfully exercised only when each
3639            // worker is its own process. (The child does NOT share the
3640            // harness tgid: `fork()` omits CLONE_THREAD, so copy_process
3641            // gives it group_leader=self / tgid=pid, a fresh singleton
3642            // group. Its `libc::_exit(0)` issues exit_group(2) ->
3643            // `do_group_exit` -> `zap_other_threads`, but a singleton tgid
3644            // has no other threads for `zap_other_threads` to SIGKILL, so
3645            // the teardown ends only the child; the harness tgid is
3646            // untouched.) Reject at spawn with an actionable diagnostic;
3647            // CloneMode::Fork gives each worker its own tgid and a
3648            // lock-clean address space.
3649            if matches!(dispatch, Dispatch::Thread) && matches!(group.work_type, WorkType::ForkExit)
3650            {
3651                anyhow::bail!(
3652                    "CloneMode::Thread is incompatible with WorkType::ForkExit \
3653                     (group {}) — ForkExit forks inside the worker, and a fork \
3654                     from a thread of the multi-threaded harness inherits \
3655                     sibling-held locks the child cannot release; only a \
3656                     separate process faithfully exercises the fork/exit \
3657                     lifecycle. Use CloneMode::Fork for ForkExit workloads.",
3658                    group.group_idx,
3659                );
3660            }
3661            // Fork mode + EpollStorm is incompatible. EpollStorm
3662            // creates an eventfd + epoll fd inside worker pos 0 and
3663            // publishes their integer fd numbers through the per-group
3664            // shared mmap region (`efd_slot` / `epfd_slot`); siblings
3665            // load those numbers and operate on them as if they
3666            // referred to the same kernel objects. Under
3667            // [`CloneMode::Fork`] each forked child holds its own copy
3668            // of the parent's fd table at fork time, but the eventfd
3669            // and epoll fd are created AFTER the fork on worker pos 0
3670            // — so sibling children's fd tables never contain those
3671            // descriptors. The integer numbers they read from the
3672            // shared region either resolve to unrelated fds the child
3673            // happened to have at the same slot or fail with EBADF.
3674            // The fd table is genuinely shared only under
3675            // [`CloneMode::Thread`]. Reject at spawn time with an
3676            // actionable diagnostic; CloneMode::Thread is the correct
3677            // choice for EpollStorm.
3678            if matches!(dispatch, Dispatch::Fork)
3679                && matches!(group.work_type, WorkType::EpollStorm { .. })
3680            {
3681                anyhow::bail!(
3682                    "CloneMode::Fork is incompatible with WorkType::EpollStorm \
3683                     (group {}) — EpollStorm publishes eventfd/epoll fd numbers \
3684                     through a shared mmap region for siblings to consume, but \
3685                     forked children hold independent fd tables that never \
3686                     contain those post-fork descriptors. Use CloneMode::Thread \
3687                     for EpollStorm workloads.",
3688                    group.group_idx,
3689                );
3690            }
3691            // Thread mode + CgroupChurn is incompatible. CgroupChurn's
3692            // worker body writes its own tid (`SYS_gettid`) to a
3693            // sibling cgroup's `cgroup.procs` file. The kernel's
3694            // `__cgroup_procs_write` resolves the tid to its
3695            // task_struct, then migrates the *entire* thread group
3696            // leader's task and every member of its tgid to the
3697            // target cgroup (see `cgroup_attach_task` /
3698            // `cgroup_migrate` in kernel/cgroup/cgroup.c — procs-file
3699            // semantics are tgid-wide, contrast `cgroup.threads`
3700            // which is per-thread under the threaded controller).
3701            // Under [`CloneMode::Thread`] every worker is a member of
3702            // the test harness's tgid, so the first CgroupChurn write
3703            // migrates the harness itself and every sibling worker
3704            // thread out from under the host. Reject at spawn time
3705            // with an actionable diagnostic; CloneMode::Fork gives
3706            // each worker its own tgid so a procs-file write moves
3707            // only that worker.
3708            if matches!(dispatch, Dispatch::Thread)
3709                && matches!(group.work_type, WorkType::CgroupChurn { .. })
3710            {
3711                anyhow::bail!(
3712                    "CloneMode::Thread is incompatible with WorkType::CgroupChurn \
3713                     (group {}) — CgroupChurn writes the worker tid to \
3714                     `cgroup.procs`, which the kernel resolves to the whole tgid \
3715                     and migrates every sibling thread (including the harness) \
3716                     to the target cgroup. Use CloneMode::Fork for CgroupChurn \
3717                     workloads so each worker is a separate tgid.",
3718                    group.group_idx,
3719                );
3720            }
3721            // Thread mode + CgroupAttachStorm is incompatible because the
3722            // worker installs `SIGCHLD = SIG_IGN` at entry (under
3723            // ReapMode::SigIgn) to auto-reap its forked children. Under
3724            // [`CloneMode::Thread`] the worker is a pthread of the harness
3725            // thread group and shares the harness `sighand` (CLONE_THREAD
3726            // implies CLONE_SIGHAND), so that install changes SIGCHLD
3727            // disposition for the WHOLE harness process — corrupting the
3728            // harness's own child reaping — not just this worker. (fork()
3729            // from a thread of the multithreaded harness is also fragile;
3730            // the forked child is safe here only because it does nothing
3731            // but `_exit`.) Under [`CloneMode::Fork`] each worker is its
3732            // own process with a private sighand, so the SIG_IGN install
3733            // and the forked-child lifecycle stay confined to that worker.
3734            // Note: the storm writes the forked CHILD's pid (its own tgid)
3735            // to cgroup.procs, so — unlike CgroupChurn's own-tid write —
3736            // the migration never moves the harness; the hazard is the
3737            // shared-sighand SIG_IGN install, not tgid migration. Reject at
3738            // spawn time with an actionable diagnostic.
3739            if matches!(dispatch, Dispatch::Thread)
3740                && matches!(group.work_type, WorkType::CgroupAttachStorm { .. })
3741            {
3742                anyhow::bail!(
3743                    "CloneMode::Thread is incompatible with WorkType::CgroupAttachStorm \
3744                     (group {}) — the worker installs SIGCHLD=SIG_IGN to auto-reap its \
3745                     forked children, but a thread-group worker shares the harness \
3746                     sighand, so that install corrupts the harness's own child reaping. \
3747                     Use CloneMode::Fork for CgroupAttachStorm workloads so each worker \
3748                     has its own process and private sighand.",
3749                    group.group_idx,
3750                );
3751            }
3752            // Dispatch-agnostic admission rules
3753            // (worker_group_size divisibility, chain depth >= 2,
3754            // IdleChurn / IpcVariance zero-rejections) live in
3755            // [`validate_workload_admission`] and are shared with
3756            // [`Self::spawn_pcomm_cgroup`]. The
3757            // CloneMode-vs-WorkType compat checks above stay
3758            // dispatch-specific because each dispatch path gates
3759            // independently: Thread+ForkExit (here) and pcomm+ForkExit
3760            // ([`Self::spawn_pcomm_cgroup`]) reject for the SAME reason
3761            // (a fork from a thread of a multi-threaded process inherits
3762            // sibling-held locks the child cannot release) via separate
3763            // per-path gates; Fork+EpollStorm has no pcomm equivalent.
3764            validate_workload_admission(group)?;
3765        }
3766
3767        // futex region sizing is per-group, not MAX'd across all
3768        // groups. Each group's futex region has its own natural
3769        // size determined by [`futex_region_size_for`] (FanOutCompute
3770        // = 16, ProducerConsumerImbalance = 32 + Q*8, everything
3771        // else = 4). Storing the size alongside each pointer in
3772        // `SpawnGuard::futex_region_sizes` lets Drop munmap each
3773        // region with its own length, so a small-variant group
3774        // composed alongside a large ProducerConsumerImbalance group
3775        // is no longer inflated to the large size.
3776        //
3777        // The kernel rounds munmap length up to PAGE_SIZE, so the
3778        // per-region waste for sub-page allocations is bounded at
3779        // one page; the previous MAX-across-groups approach could
3780        // waste many pages per small-variant group when paired with
3781        // a large queue_depth_target.
3782
3783        // All failable acquisitions in this function route through
3784        // `guard`. If any `?`/`bail!` returns early, the guard's Drop
3785        // SIGKILLs+reaps forked children, closes open pipe fds, and
3786        // munmaps the shared regions — so no leak on a mid-spawn
3787        // error path.
3788        let mut guard = SpawnGuard::new();
3789
3790        // Per-worker iteration counter region (MAP_SHARED). Sized
3791        // for ALL groups' workers laid out contiguously: primary
3792        // group occupies slots `[0, primary.num_workers)`, composed
3793        // group `k` occupies slots starting at the running offset
3794        // tracked by `iter_offset` in the per-group spawn loop
3795        // below. Each worker atomically stores its iteration count
3796        // to its assigned slot; the parent reads all slots via
3797        // `snapshot_iterations()`. The mmap base is page-aligned
3798        // (kernel guarantee), so casting to `*mut AtomicU64` is
3799        // sound: page alignment (≥ 4096) ≥ AtomicU64 alignment (8),
3800        // and the region size is an exact multiple of
3801        // `size_of::<AtomicU64>()` (== 8). Each `.add(i)` moves by
3802        // `i * 8` bytes, preserving the 8-byte alignment invariant.
3803        // No non-atomic access to the region exists anywhere in the
3804        // crate, so the atomic-only aliasing rule (workers + parent
3805        // share `&AtomicU64` references derived from the raw
3806        // pointer) holds.
3807        let total_workers: usize = groups.iter().map(|g| g.num_workers).sum();
3808        if total_workers > 0 {
3809            let size = total_workers * std::mem::size_of::<AtomicU64>();
3810            let ptr = unsafe {
3811                libc::mmap(
3812                    std::ptr::null_mut(),
3813                    size,
3814                    libc::PROT_READ | libc::PROT_WRITE,
3815                    libc::MAP_SHARED | libc::MAP_ANONYMOUS,
3816                    -1,
3817                    0,
3818                )
3819            };
3820            if ptr == libc::MAP_FAILED {
3821                let errno = std::io::Error::last_os_error();
3822                let hint = mmap_shared_anon_errno_hint(errno.raw_os_error());
3823                anyhow::bail!(
3824                    "mmap(MAP_SHARED|MAP_ANONYMOUS, {size} bytes) for the \
3825                     per-worker iter_counters region failed: {errno}{hint}; \
3826                     this region holds one AtomicU64 per worker \
3827                     ({total_workers} slots across {} group(s)) so the parent \
3828                     can snapshot iteration counts via \
3829                     `snapshot_iterations()`. Remediation: reduce num_workers \
3830                     (each worker consumes 8 bytes of this region, rounded up \
3831                     to a page) or raise `vm.max_map_count` / the memory \
3832                     cgroup limit.",
3833                    groups.len(),
3834                );
3835            }
3836            guard.iter_counters = ptr as *mut AtomicU64;
3837            guard.iter_counter_bytes = size;
3838            // Single shared phase-epoch word (MAP_SHARED, MAP_ANONYMOUS
3839            // zero-inits it to 0 = BASELINE). The scenario engine bumps
3840            // it at each step boundary; backdrop workers read it to
3841            // attribute per-phase telemetry. Allocated for every handle
3842            // — a step-local pool whose epoch is never bumped simply
3843            // observes no change and emits no PhaseSlices.
3844            let epoch_size = std::mem::size_of::<AtomicU32>();
3845            let epoch_ptr = unsafe {
3846                libc::mmap(
3847                    std::ptr::null_mut(),
3848                    epoch_size,
3849                    libc::PROT_READ | libc::PROT_WRITE,
3850                    libc::MAP_SHARED | libc::MAP_ANONYMOUS,
3851                    -1,
3852                    0,
3853                )
3854            };
3855            if epoch_ptr == libc::MAP_FAILED {
3856                let errno = std::io::Error::last_os_error();
3857                let hint = mmap_shared_anon_errno_hint(errno.raw_os_error());
3858                anyhow::bail!(
3859                    "mmap(MAP_SHARED|MAP_ANONYMOUS, {epoch_size} bytes) for the \
3860                     phase-epoch word failed: {errno}{hint}; this single \
3861                     AtomicU32 lets the scenario engine broadcast the current \
3862                     phase to backdrop workers for per-phase attribution.",
3863                );
3864            }
3865            guard.phase_epoch = epoch_ptr as *mut AtomicU32;
3866            guard.phase_epoch_bytes = epoch_size;
3867        }
3868
3869        // First-iteration signal eventfd (C6). Allocated BEFORE the fork
3870        // loop — like the iter_counters mmap above — so every forked child
3871        // inherits its fd-table copy. Only when the workload opted in AND
3872        // the dispatch is fork with ≥1 worker: thread / pcomm dispatch pass
3873        // a `-1` fd to `worker_main` and never signal, and a zero-worker
3874        // spawn has nothing to wait on. Counter mode (`0` initial value)
3875        // so the parent's `wait_first_iteration_all` reads the SUM of the
3876        // per-worker `1`s. EFD_NONBLOCK keeps the worker's post-first-iter
3877        // `write` non-blocking; EFD_CLOEXEC is safe because fork workers
3878        // never `exec` (a stale fd across an exec would only leak, not
3879        // misbehave). The `OwnedFd` lives on the guard (closed on early
3880        // bail) and transfers to the handle on success.
3881        if config.signal_first_iteration && matches!(dispatch, Dispatch::Fork) && total_workers > 0
3882        {
3883            let raw = unsafe { libc::eventfd(0, libc::EFD_NONBLOCK | libc::EFD_CLOEXEC) };
3884            if raw < 0 {
3885                let errno = std::io::Error::last_os_error();
3886                anyhow::bail!(
3887                    "eventfd(0, EFD_NONBLOCK|EFD_CLOEXEC) for the first-iteration \
3888                     signal failed: {errno}; this fd lets the verifier dispatch \
3889                     probe block on the first-iteration edge instead of polling \
3890                     snapshot_iterations. Remediation: raise the process fd limit \
3891                     (RLIMIT_NOFILE).",
3892                );
3893            }
3894            // SAFETY: `raw` is a fresh, exclusively-owned eventfd from the
3895            // successful `eventfd(2)` above; wrapping it in OwnedFd takes
3896            // sole ownership so it is closed exactly once on Drop.
3897            guard.first_iter_signal = Some(unsafe { std::os::fd::OwnedFd::from_raw_fd(raw) });
3898        }
3899
3900        // Spawn each group in declaration order. `iter_offset`
3901        // tracks the running offset into the iter_counters mmap
3902        // (slot allocation per the layout commented above). Each
3903        // group's pipes / chain_pipes / futex_ptrs are appended to
3904        // the guard's flat vectors; we record per-group base
3905        // offsets so the per-worker fork loop can compute
3906        // global-vector indices from per-group worker indices and
3907        // the close-other-fds child path can iterate the full
3908        // guard while still identifying its own group's resources.
3909        let mut iter_offset: usize = 0;
3910        for group in &groups {
3911            Self::spawn_group(&mut guard, group, &dispatch, iter_offset)?;
3912            iter_offset += group.num_workers;
3913        }
3914
3915        // Success: transfer every live resource (children, threads,
3916        // futex_ptrs, iter_counters, pipe_pairs, chain_pipes) into
3917        // the handle. The guard's subsequent Drop sees empty Vecs
3918        // and a null iter_counters pointer — it is a no-op on this
3919        // path. Pipe fds are closed by `WorkloadHandle::drop` AFTER
3920        // worker shutdown so Thread-mode workers (which share the
3921        // parent's fd table) finish their pipe ops before the close.
3922        Ok(guard.into_handle())
3923    }
3924
3925    /// Spawn a single worker group's resources and per-worker
3926    /// tasks, appending each into the shared [`SpawnGuard`].
3927    ///
3928    /// Each group records its own base offsets into the guard's
3929    /// flat vectors at entry time, then uses those offsets when
3930    /// computing per-worker `pair_idx` / `chain_idx` /
3931    /// `futex_group_idx`. The fork-child close-other-fds block
3932    /// iterates the FULL guard so it sweeps fds belonging to
3933    /// other groups too — without that sweep, a composed-group
3934    /// worker would inherit (and never close) every primary-group
3935    /// pipe fd.
3936    ///
3937    /// Resource ownership is uniform across groups: every
3938    /// allocated pipe / mmap region lives in the guard's flat
3939    /// vectors and is freed by `SpawnGuard::Drop` on early-bail or
3940    /// transferred to [`WorkloadHandle`] on success via
3941    /// [`SpawnGuard::into_handle`].
3942    #[allow(clippy::too_many_arguments)]
3943    fn spawn_group(
3944        guard: &mut SpawnGuard,
3945        group: &GroupParams,
3946        dispatch: &Dispatch,
3947        iter_offset: usize,
3948    ) -> Result<()> {
3949        let needs_pipes = matches!(
3950            group.work_type,
3951            WorkType::PipeIo { .. } | WorkType::CachePipe { .. }
3952        );
3953        let chain_depth = group.work_type.chain_pipe_depth();
3954        let needs_futex = group.work_type.needs_shared_mem();
3955
3956        // Record the bases into the guard's flat vectors BEFORE
3957        // appending this group's allocations. The base values
3958        // identify "where this group's resources start" — the
3959        // per-worker fork loop combines `pipe_pair_base + i / 2`
3960        // (and analogous for chain_idx / futex_group_idx) to
3961        // address its own resources without colliding with another
3962        // group's range.
3963        let pipe_pair_base = guard.pipe_pairs.len();
3964        let chain_pipes_base = guard.chain_pipes.len();
3965        let futex_ptrs_base = guard.futex_ptrs.len();
3966        // Per-group natural size for the futex MAP_SHARED region —
3967        // each region in this group's range gets exactly this many
3968        // bytes (rather than the previous global MAX across every
3969        // group). See `futex_region_size_for` for the per-variant
3970        // sizing rules.
3971        let futex_region_size = futex_region_size_for(&group.work_type);
3972
3973        // For paired work types, create one pipe per worker pair before forking.
3974        // pipe_pairs[pair_idx] = (read_fd, write_fd) for the A->B direction,
3975        // and a second pipe for B->A. Use `pipe2(O_CLOEXEC)` instead
3976        // of bare `pipe(2)`: O_CLOEXEC is the correct default for
3977        // any kernel fd in long-running processes — fds without
3978        // O_CLOEXEC silently leak into any exec path.
3979        if needs_pipes {
3980            for _ in 0..group.num_workers / 2 {
3981                let mut ab = [0i32; 2]; // A writes, B reads
3982                if unsafe { libc::pipe2(ab.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
3983                    anyhow::bail!("pipe2 failed: {}", std::io::Error::last_os_error());
3984                }
3985                let mut ba = [0i32; 2]; // B writes, A reads
3986                if unsafe { libc::pipe2(ba.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
3987                    // Close the ab half we just created: it is not
3988                    // yet owned by the guard, so its Drop won't
3989                    // otherwise reach it.
3990                    unsafe {
3991                        libc::close(ab[0]);
3992                        libc::close(ab[1]);
3993                    }
3994                    anyhow::bail!("pipe2 failed: {}", std::io::Error::last_os_error());
3995                }
3996                guard.pipe_pairs.push((ab, ba));
3997            }
3998        }
3999
4000        // For WakeChain { wake: WakeMechanism::Pipe }, allocate `depth` pipes per
4001        // chain (one pipe per stage). Pipe `i` connects stage `i`
4002        // (writer) to stage `(i + 1) % depth` (reader). On any
4003        // `pipe2()` failure mid-allocation, close the half-built
4004        // chain's pipes before bailing — the chain is not yet
4005        // pushed onto `guard.chain_pipes`, so its Drop won't
4006        // otherwise reach those fds. `O_CLOEXEC` matches the
4007        // defense-in-depth posture documented above on the
4008        // pipe-pair allocation.
4009        if let Some(depth) = chain_depth
4010            && depth > 0
4011            && group.num_workers >= depth
4012        {
4013            let chains = group.num_workers / depth;
4014            for _ in 0..chains {
4015                let mut chain: Vec<[i32; 2]> = Vec::with_capacity(depth);
4016                let mut alloc_ok = true;
4017                for _ in 0..depth {
4018                    let mut p = [0i32; 2];
4019                    if unsafe { libc::pipe2(p.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
4020                        alloc_ok = false;
4021                        break;
4022                    }
4023                    chain.push(p);
4024                }
4025                if !alloc_ok {
4026                    for p in &chain {
4027                        unsafe {
4028                            libc::close(p[0]);
4029                            libc::close(p[1]);
4030                        }
4031                    }
4032                    anyhow::bail!(
4033                        "WakeChain pipe2 allocation failed: {}",
4034                        std::io::Error::last_os_error()
4035                    );
4036                }
4037                guard.chain_pipes.push(chain);
4038            }
4039        }
4040
4041        // For FutexPingPong/FutexFanOut/FanOutCompute/MutexContention, allocate
4042        // one shared region per worker group via MAP_SHARED|MAP_ANONYMOUS
4043        // so all members of the fork see the same physical page.
4044        // Each region is sized exactly to the variant's natural
4045        // need (see [`futex_region_size_for`]) — the per-region
4046        // size is recorded in `guard.futex_region_sizes` parallel
4047        // to `guard.futex_ptrs`, so munmap on Drop receives the
4048        // correct length even when groups with different natural
4049        // sizes co-exist in the same workload.
4050        let futex_group_size = group.work_type.worker_group_size().unwrap_or(2);
4051        if needs_futex {
4052            for _ in 0..group.num_workers / futex_group_size {
4053                let ptr = unsafe {
4054                    libc::mmap(
4055                        std::ptr::null_mut(),
4056                        futex_region_size,
4057                        libc::PROT_READ | libc::PROT_WRITE,
4058                        libc::MAP_SHARED | libc::MAP_ANONYMOUS,
4059                        -1,
4060                        0,
4061                    )
4062                };
4063                if ptr == libc::MAP_FAILED {
4064                    let errno = std::io::Error::last_os_error();
4065                    let hint = mmap_shared_anon_errno_hint(errno.raw_os_error());
4066                    anyhow::bail!(
4067                        "mmap(MAP_SHARED|MAP_ANONYMOUS, {futex_region_size} bytes) \
4068                         for a futex shared-memory region failed: {errno}{hint}; \
4069                         this region backs the {:?} worker-group's (group {}) \
4070                         inter-process futex word and is allocated \
4071                         before fork so every child inherits the same \
4072                         mapping. Remediation: reduce num_workers (each \
4073                         futex group consumes one shared page) or raise \
4074                         `vm.max_map_count` / the memory cgroup limit.",
4075                        group.work_type.name(),
4076                        group.group_idx,
4077                    );
4078                }
4079                unsafe { std::ptr::write_bytes(ptr as *mut u8, 0, futex_region_size) };
4080                guard.futex_ptrs.push(ptr as *mut u32);
4081                guard.futex_region_sizes.push(futex_region_size);
4082            }
4083        }
4084
4085        for i in 0..group.num_workers {
4086            let affinity = resolve_affinity(&group.affinity)?;
4087
4088            // Determine pipe fds for this worker.
4089            //
4090            // Three shapes use the same `Option<(read_fd, write_fd)>`
4091            // parameter:
4092            // - PipeIo / CachePipe (paired): worker A reads `ba[0]`,
4093            //   writes `ab[1]`; worker B reads `ab[0]`, writes
4094            //   `ba[1]`.
4095            // - WakeChain { wake: WakeMechanism::Pipe } (chain ring): stage `s`
4096            //   reads from pipe `(s + depth - 1) % depth` (its
4097            //   predecessor's write end's matching read end) and
4098            //   writes to pipe `s` (its own pipe's write end, which
4099            //   stage `s + 1` reads from).
4100            // - Everything else: `None`.
4101            //
4102            // Indices are computed in the GLOBAL `guard.pipe_pairs`
4103            // / `guard.chain_pipes` space by adding the per-group
4104            // base recorded at the top of `spawn_group`. A composed
4105            // group's pipe-pair-base, for example, equals the sum
4106            // of every prior group's pipe-pair count, so its first
4107            // worker pair is allocated immediately after the
4108            // primary's last entry — no collision, no aliasing.
4109            let worker_pipe_fds: Option<(i32, i32)> = if needs_pipes {
4110                let pair_idx = pipe_pair_base + i / 2;
4111                let (ref ab, ref ba) = guard.pipe_pairs[pair_idx];
4112                if i % 2 == 0 {
4113                    // Worker A: writes to ab[1], reads from ba[0]
4114                    Some((ba[0], ab[1]))
4115                } else {
4116                    // Worker B: writes to ba[1], reads from ab[0]
4117                    Some((ab[0], ba[1]))
4118                }
4119            } else if let Some(depth) = chain_depth
4120                && depth > 0
4121            {
4122                let chain_idx = chain_pipes_base + i / depth;
4123                let stage = i % depth;
4124                let prev_stage = (stage + depth - 1) % depth;
4125                let chain = &guard.chain_pipes[chain_idx];
4126                // Read end of predecessor's pipe; write end of own
4127                // pipe. The kernel pipe pair is `[read_end,
4128                // write_end]` per `libc::pipe`'s manpage.
4129                Some((chain[prev_stage][0], chain[stage][1]))
4130            } else {
4131                None
4132            };
4133
4134            // Futex pointer for this worker. The `pos` is the
4135            // worker's index inside its futex group: `pos == 0`
4136            // is the group's "first" worker (the role that varies
4137            // per-variant — pair-A for FutexPingPong, messenger for
4138            // FutexFanOut/FanOutCompute, waker for ThunderingHerd/
4139            // AsymmetricWaker, chain-head for WakeChain). Variants
4140            // that need finer-grained per-worker positioning
4141            // (PriorityInversion's 3 tiers, ProducerConsumerImbalance's
4142            // producer/consumer split, RtStarvation's RT/CFS split,
4143            // WakeChain's stage index) consume `pos` directly.
4144            let worker_futex: Option<(*mut u32, usize)> = if needs_futex {
4145                let futex_group_idx = futex_ptrs_base + i / futex_group_size;
4146                let pos = i % futex_group_size;
4147                Some((guard.futex_ptrs[futex_group_idx], pos))
4148            } else {
4149                None
4150            };
4151
4152            // Shared iteration counter slot for this worker. The
4153            // group-local index `i` is added to the spawn-time
4154            // `iter_offset` so each group's slot range is disjoint
4155            // from every other group's.
4156            let iter_slot: *mut AtomicU64 = if !guard.iter_counters.is_null() {
4157                unsafe { guard.iter_counters.add(iter_offset + i) }
4158            } else {
4159                std::ptr::null_mut()
4160            };
4161            // Single shared phase-epoch word — the SAME pointer for every
4162            // worker (no per-worker offset). Non-null for every handle; the
4163            // scenario bumps it only for backdrop handles, so step-local
4164            // workers observe no change and emit no slices.
4165            let phase_epoch_ptr: *mut AtomicU32 = guard.phase_epoch;
4166
4167            // First-iteration signal fd for this worker (C6). `-1` (absent)
4168            // unless the group opted in AND the handle allocated the
4169            // eventfd — which happens only for fork dispatch (thread /
4170            // pcomm groups leave `guard.first_iter_signal` None, so this
4171            // resolves to `-1` and the worker never signals). A plain `i32`
4172            // copy, so it crosses the fork/thread closure boundary without
4173            // a Send newtype.
4174            let first_iter_signal_fd: i32 = if group.signal_first_iteration {
4175                use std::os::unix::io::AsRawFd;
4176                guard
4177                    .first_iter_signal
4178                    .as_ref()
4179                    .map(|f| f.as_raw_fd())
4180                    .unwrap_or(-1)
4181            } else {
4182                -1
4183            };
4184
4185            // Per-mode dispatch. Thread-mode workers do not need
4186            // pipes — the rendezvous and report channels are
4187            // in-process Rust primitives (`mpsc::sync_channel(0)` +
4188            // `JoinHandle`). Fork mode uses the pipe-based
4189            // scaffolding below.
4190            match dispatch {
4191                Dispatch::Thread => {
4192                    spawn_thread_worker(
4193                        guard,
4194                        group,
4195                        affinity,
4196                        worker_pipe_fds,
4197                        worker_futex,
4198                        iter_slot,
4199                        phase_epoch_ptr,
4200                        first_iter_signal_fd,
4201                    )?;
4202                    continue;
4203                }
4204                Dispatch::Fork => {
4205                    // fall through to the pipe-based dispatch below
4206                }
4207            }
4208
4209            // Create pipe for report and a second pipe for "start" signal.
4210            // Local cleanup on second-pipe failure: the guard has no
4211            // per-worker tracking of half-allocated pipes, so the first
4212            // half closes here before the bail. `O_CLOEXEC` matches
4213            // the defense-in-depth posture above on the inter-worker
4214            // pipe pairs and chain pipes.
4215            let mut report_fds = [0i32; 2];
4216            if unsafe { libc::pipe2(report_fds.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
4217                anyhow::bail!(
4218                    "worker {}/{} (group {}): report pipe2 failed: {}",
4219                    i + 1,
4220                    group.num_workers,
4221                    group.group_idx,
4222                    std::io::Error::last_os_error(),
4223                );
4224            }
4225            // Grow the report pipe so a worker that produces a worst-
4226            // case report (two `Vec<u64>` reservoirs capped at
4227            // `MAX_WAKE_SAMPLES` (100_000) entries each, plus
4228            // smaller fields) finishes `write_all` without blocking
4229            // for the parent to drain. The default pipe capacity on
4230            // Linux is 16 pages (64 KiB on 4 KiB pages, 256 KiB on
4231            // 16 KiB pages); a postcard-encoded `WorkerReport` with
4232            // both reservoirs full is at most ~1.8 MiB (u64 varint
4233            // is up to 9 bytes per entry, 200_000 entries total).
4234            // Without this grow, the worker blocks inside
4235            // `f.write_all(&bytes)` waiting for the parent's
4236            // `read_to_end`. The parent reads workers serially with
4237            // a shared 5 s deadline (`stop_and_collect`); if the
4238            // first worker's drain consumes the budget, every
4239            // subsequent worker's `poll` budget is 0, the parent
4240            // closes the read fd without reading, and the child's
4241            // blocked write returns EPIPE — the report is lost and
4242            // the failure surfaces as a sentinel `WorkerReport`.
4243            // Sizing the pipe to 8 MiB lets the child write the
4244            // full payload in one non-blocking burst, exit, and
4245            // close its write end; when the parent later polls the
4246            // read fd, all data is already buffered in the kernel
4247            // and `read_to_end` returns immediately with EOF.
4248            //
4249            // F_SETPIPE_SZ rounds the requested size up to the next
4250            // power of two; unprivileged callers are clamped to
4251            // `/proc/sys/fs/pipe-max-size` (typically 1 MiB).
4252            // ktstr always runs as root inside the guest, so the
4253            // grow succeeds. A best-effort failure (older kernel
4254            // without F_SETPIPE_SZ, or an unexpected EPERM) leaves
4255            // the default size in place — workers with small
4256            // reports still complete; workers with large reports
4257            // fall back to the historical blocking-write behaviour.
4258            // Logging the failure surfaces the regression without
4259            // failing the whole spawn.
4260            const REPORT_PIPE_SIZE: libc::c_int = 8 * 1024 * 1024;
4261            // SAFETY: `report_fds[1]` is the freshly opened write
4262            // end returned by `pipe2` above; F_SETPIPE_SZ accepts
4263            // any pipe fd (read or write end — both refer to the
4264            // same kernel `struct pipe_inode_info`).
4265            let prev_size =
4266                unsafe { libc::fcntl(report_fds[1], libc::F_SETPIPE_SZ, REPORT_PIPE_SIZE) };
4267            if prev_size < 0 {
4268                let err = std::io::Error::last_os_error();
4269                tracing::warn!(
4270                    worker = i + 1,
4271                    num_workers = group.num_workers,
4272                    group_idx = group.group_idx,
4273                    requested_size = REPORT_PIPE_SIZE,
4274                    %err,
4275                    "F_SETPIPE_SZ on report pipe failed; falling back to default \
4276                     pipe capacity. Workers producing >64 KiB reports may block \
4277                     in write_all and have their reports truncated by the \
4278                     parent's deadline-driven fd close."
4279                );
4280            }
4281            let mut start_fds = [0i32; 2];
4282            if unsafe { libc::pipe2(start_fds.as_mut_ptr(), libc::O_CLOEXEC) } != 0 {
4283                unsafe {
4284                    libc::close(report_fds[0]);
4285                    libc::close(report_fds[1]);
4286                }
4287                anyhow::bail!(
4288                    "worker {}/{} (group {}): start pipe2 failed: {}",
4289                    i + 1,
4290                    group.num_workers,
4291                    group.group_idx,
4292                    std::io::Error::last_os_error(),
4293                );
4294            }
4295
4296            // Block SIGUSR1 across fork so the child inherits a
4297            // SIGUSR1-blocked mask. Without this, there is a
4298            // window between `fork()` returning in the child and
4299            // the child installing `sigusr1_handler` below where
4300            // SIGUSR1's default disposition (terminate) is in
4301            // effect. A `stop_and_collect` call that races against
4302            // a worker still in mid-init — e.g. if a sibling
4303            // worker spawned just before this one already pushed
4304            // its (pid, ...) into `guard.children` and a parent
4305            // thread initiated stop signaling — could deliver
4306            // SIGUSR1 to this child before its handler is
4307            // installed and terminate it via the default action.
4308            // The block + post-install unblock pattern queues any
4309            // SIGUSR1 sent in the window so the handler observes
4310            // it on the unblock; the worker then sets `STOP` and
4311            // exits gracefully.
4312            //
4313            // pthread_sigmask is the multi-threaded equivalent of
4314            // sigprocmask: ktstr's parent process is genuinely
4315            // multi-threaded (test runner threads, tracing
4316            // threads), and `sigprocmask` semantics are unspecified
4317            // in MT programs per signal-safety(7) /
4318            // pthread_sigmask(3) — pthread_sigmask is the correct
4319            // primitive to use here. The fork()-inherited mask is
4320            // the calling thread's mask at fork time, exactly what
4321            // we want.
4322            //
4323            // Old mask is captured so the parent path can restore
4324            // its prior signal mask after fork — the block must
4325            // be transient on the parent side; the parent's
4326            // SIGUSR1 stays under whatever disposition the host
4327            // application configured (typically the default;
4328            // ktstr does not handle SIGUSR1 in the parent
4329            // process).
4330            let mut old_mask: libc::sigset_t = unsafe { std::mem::zeroed() };
4331            let mut block_mask: libc::sigset_t = unsafe { std::mem::zeroed() };
4332            // pthread_sigmask returns 0 on success, a positive errno
4333            // on failure (per POSIX — does NOT set errno). A non-zero
4334            // return here means the SIGUSR1 mask did not actually get
4335            // installed: the child would inherit an unblocked SIGUSR1
4336            // and the post-fork race window between fork() and the
4337            // child's signal() install would terminate the worker on
4338            // its default action. Log so the failure surfaces.
4339            let psm_block_rc = unsafe {
4340                libc::sigemptyset(&mut block_mask);
4341                libc::sigaddset(&mut block_mask, libc::SIGUSR1);
4342                libc::pthread_sigmask(libc::SIG_BLOCK, &block_mask, &mut old_mask)
4343            };
4344            if psm_block_rc != 0 {
4345                tracing::warn!(
4346                    rc = psm_block_rc,
4347                    "pthread_sigmask(SIG_BLOCK, SIGUSR1) failed pre-fork; child inherits unblocked SIGUSR1 and may terminate on default action before installing handler"
4348                );
4349            }
4350
4351            let pid = unsafe { libc::fork() };
4352            match pid {
4353                -1 => {
4354                    // Fork failed: restore the parent's previous
4355                    // signal mask before bailing so this failure
4356                    // doesn't leave SIGUSR1 blocked in the calling
4357                    // thread for the rest of the process lifetime.
4358                    let psm_restore_rc = unsafe {
4359                        libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut())
4360                    };
4361                    if psm_restore_rc != 0 {
4362                        tracing::warn!(
4363                            rc = psm_restore_rc,
4364                            "pthread_sigmask(SIG_SETMASK) failed restoring mask after fork failure; SIGUSR1 may remain blocked in this thread"
4365                        );
4366                    }
4367                    // Close both fresh pipes so they don't leak
4368                    // before the guard reaps the already-forked
4369                    // siblings.
4370                    unsafe {
4371                        libc::close(report_fds[0]);
4372                        libc::close(report_fds[1]);
4373                        libc::close(start_fds[0]);
4374                        libc::close(start_fds[1]);
4375                    }
4376                    anyhow::bail!(
4377                        "worker {}/{} (group {}): fork failed: {}",
4378                        i + 1,
4379                        group.num_workers,
4380                        group.group_idx,
4381                        std::io::Error::last_os_error(),
4382                    );
4383                }
4384                0 => {
4385                    // Child: set parent-death signal BEFORE any other
4386                    // post-fork setup so the kernel SIGKILLs this worker
4387                    // immediately if the parent dies during the remaining
4388                    // init (close fd loops, signal handler install, start-
4389                    // pipe wait, worker_main). Without PR_SET_PDEATHSIG,
4390                    // a parent crash between fork and start leaves workers
4391                    // reparented to init and spinning indefinitely —
4392                    // they'd outlive the test run, consume the cgroup's
4393                    // CPU, and block the next scenario's cgroup teardown
4394                    // with EBUSY. SIGKILL is the only safe choice: it
4395                    // cannot be masked and runs before any of this child's
4396                    // destructors execute (good — those destructors still
4397                    // reference the parent's guard). prctl is NOT listed
4398                    // as async-signal-safe by signal-safety(7); safe to
4399                    // call here because this is a single-threaded
4400                    // post-fork child before any signal handlers are
4401                    // installed, so no interleaving can observe partial
4402                    // state.
4403                    unsafe {
4404                        libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL);
4405                    }
4406                    // Fork-race close: if the parent died between fork()
4407                    // return and the prctl above, this child was already
4408                    // reparented (typically to pid 1) before PDEATHSIG
4409                    // was armed — the death signal is keyed on the CURRENT
4410                    // parent, not the parent-at-fork-time, so the signal
4411                    // will never fire. getppid() == 1 means we are already
4412                    // orphaned; exit now instead of running the full
4413                    // worker loop only to leak into init. Using `_exit`
4414                    // (async-signal-safe) rather than `exit` so Rust
4415                    // destructors that reference the parent's now-dead
4416                    // guard don't run on the fork stack.
4417                    //
4418                    // PR_SET_CHILD_SUBREAPER exception: when an ancestor
4419                    // of the ktstr process has called
4420                    // `prctl(PR_SET_CHILD_SUBREAPER, 1)` (systemd user
4421                    // scopes, some container runtimes, certain CI
4422                    // harnesses), orphaned descendants reparent to the
4423                    // nearest live subreaper rather than pid 1. In that
4424                    // case `getppid() == 1` is false after an orphan-
4425                    // race even though the original parent is dead —
4426                    // the check is a best-effort fast-path for the
4427                    // common "pid 1 catches orphans" case and does NOT
4428                    // fire under subreaper ancestry. PDEATHSIG still
4429                    // fires correctly in that scenario (the signal is
4430                    // triggered when the CURRENT parent dies, and the
4431                    // subreaper then inherits us), so the guard is a
4432                    // narrowing of the leak window, not an elimination.
4433                    //
4434                    // Guest-init exception: inside a ktstr guest VM the
4435                    // test driver IS pid 1 (it runs as /init), so every
4436                    // worker forked by a scenario legitimately has
4437                    // `getppid() == 1` even though the parent is alive
4438                    // and well. Firing the orphan guard there would kill
4439                    // every worker on startup and produce sentinel
4440                    // "0 cpus, 0 iterations" reports. `ktstr_guest_init`
4441                    // sets `KTSTR_GUEST_INIT=1` before dispatch; that
4442                    // variable is inherited by every descendant process,
4443                    // so its presence is a reliable signal that pid 1 is
4444                    // the legitimate parent. Host-side workloads leave
4445                    // the variable unset and retain the orphan detection.
4446                    if std::env::var_os(crate::KTSTR_GUEST_INIT_ENV).is_none()
4447                        && unsafe { libc::getppid() } == 1
4448                    {
4449                        exit_child_success();
4450                    }
4451                    // Make this worker its own process-group leader so
4452                    // any descendants it spawns inherit `pgid == worker_pid`.
4453                    // `stop_and_collect` / Drop issue `killpg(worker_pid,
4454                    // SIGKILL)` alongside the direct `kill` — without a
4455                    // private pgid, descendants forked by a
4456                    // [`WorkType::Custom`] body (or any future workload
4457                    // that spawns helpers) stay in the parent Rust
4458                    // process's pgid, survive the worker's SIGKILL, and
4459                    // orphan onto init. PR_SET_PDEATHSIG handles the
4460                    // "parent crashes" case but is per-task and cleared
4461                    // on fork, so grandchildren don't inherit it — the
4462                    // pgid route is the only safe reach for them when
4463                    // teardown is explicit. Failure is silently ignored:
4464                    // the only reachable failure mode for setpgid(0, 0)
4465                    // in a just-forked child is EPERM from an earlier
4466                    // setsid (we never call it), so a return of -1 here
4467                    // means the kernel invariant changed and the reach
4468                    // degrades to "leader only" — same as the pre-batch
4469                    // behavior. Async-signal-safe per signal-safety(7).
4470                    unsafe {
4471                        libc::setpgid(0, 0);
4472                    }
4473                    // Install signal handler BEFORE unblocking
4474                    // SIGUSR1: the parent blocked SIGUSR1 across
4475                    // fork so this child inherits a blocked mask;
4476                    // any SIGUSR1 sent by `stop_and_collect` while
4477                    // the mask is in effect queues until we
4478                    // unblock. Once the handler is registered we
4479                    // restore the original (pre-block) mask via
4480                    // `pthread_sigmask(SIG_SETMASK, …)`, which
4481                    // delivers any pending SIGUSR1 directly into
4482                    // `sigusr1_handler` and sets `STOP` cleanly —
4483                    // the worker proceeds into the start-wait
4484                    // poll() with handler armed and any pending
4485                    // signal already consumed. Without this two-
4486                    // step, SIGUSR1 would fire under its default
4487                    // disposition (terminate) during the window
4488                    // between `fork()` and `signal()`.
4489                    STOP.store(false, Ordering::Relaxed);
4490                    // libc::signal returns SIG_ERR (cast as
4491                    // sighandler_t) on failure with errno set.
4492                    // pthread_sigmask returns 0 on success, errno
4493                    // value on failure. Both failures here are
4494                    // visible defects: signal() fail means SIGUSR1
4495                    // keeps default disposition (terminate) and
4496                    // stop_and_collect cannot stop the worker
4497                    // gracefully — it falls through to the
4498                    // killpg/SIGKILL escalation; pthread_sigmask()
4499                    // fail means the SIGUSR1 we blocked across
4500                    // fork stays blocked, so the parent's stop
4501                    // SIGUSR1 queues forever in the child and the
4502                    // child runs the full work loop until the
4503                    // killpg escalation.
4504                    //
4505                    // tracing is unsafe in a post-fork child
4506                    // before _exit (the parent's tracing
4507                    // subscriber may be locked elsewhere); use
4508                    // eprintln! which is fork-safe per
4509                    // signal-safety(7) (write(2)).
4510                    let sig_prev = unsafe {
4511                        libc::signal(
4512                            libc::SIGUSR1,
4513                            sigusr1_handler as *const () as libc::sighandler_t,
4514                        )
4515                    };
4516                    if sig_prev == libc::SIG_ERR {
4517                        let errno = std::io::Error::last_os_error();
4518                        eprintln!(
4519                            "ktstr: signal(SIGUSR1) install failed in worker child: {errno}; graceful stop unavailable, killpg escalation will reap"
4520                        );
4521                    }
4522                    let psm_unblock_rc = unsafe {
4523                        libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut())
4524                    };
4525                    if psm_unblock_rc != 0 {
4526                        eprintln!(
4527                            "ktstr: pthread_sigmask(SIG_SETMASK) unblock failed in worker child: rc={psm_unblock_rc}; SIGUSR1 stays blocked, killpg escalation will reap"
4528                        );
4529                    }
4530                    // Close unused pipe ends
4531                    unsafe {
4532                        libc::close(report_fds[0]);
4533                        libc::close(start_fds[1]);
4534                    }
4535                    // Close pipe ends belonging to other workers
4536                    // in this pair, AND every pipe fd that belongs
4537                    // to any other pair anywhere in the workload —
4538                    // including pairs owned by other groups, since
4539                    // every pre-fork allocation lives in
4540                    // `guard.pipe_pairs` regardless of which group
4541                    // declared it. The fork inherits the parent's
4542                    // entire fd table; without this sweep, a
4543                    // composed-group worker would hold open every
4544                    // primary-group pipe fd for its lifetime,
4545                    // producing fd leaks and (for chain-shaped
4546                    // workloads) keeping reader-side blocks live
4547                    // when the writer-side closes.
4548                    if needs_pipes {
4549                        let pair_idx = pipe_pair_base + i / 2;
4550                        let (ref ab, ref ba) = guard.pipe_pairs[pair_idx];
4551                        if i % 2 == 0 {
4552                            // Worker A keeps ba[0] (read) and ab[1] (write).
4553                            // Close ab[0] and ba[1].
4554                            unsafe {
4555                                libc::close(ab[0]);
4556                                libc::close(ba[1]);
4557                            }
4558                        } else {
4559                            // Worker B keeps ab[0] (read) and ba[1] (write).
4560                            // Close ab[1] and ba[0].
4561                            unsafe {
4562                                libc::close(ab[1]);
4563                                libc::close(ba[0]);
4564                            }
4565                        }
4566                        // Close all pipe fds from other pairs (any group).
4567                        for (j, (ab2, ba2)) in guard.pipe_pairs.iter().enumerate() {
4568                            if j != pair_idx {
4569                                unsafe {
4570                                    libc::close(ab2[0]);
4571                                    libc::close(ab2[1]);
4572                                    libc::close(ba2[0]);
4573                                    libc::close(ba2[1]);
4574                                }
4575                            }
4576                        }
4577                    } else {
4578                        // Worker doesn't own any pipe pair, but
4579                        // other groups' pipe pairs are still in the
4580                        // child's fd table — close them all.
4581                        for (ab2, ba2) in guard.pipe_pairs.iter() {
4582                            unsafe {
4583                                libc::close(ab2[0]);
4584                                libc::close(ab2[1]);
4585                                libc::close(ba2[0]);
4586                                libc::close(ba2[1]);
4587                            }
4588                        }
4589                    }
4590                    if let Some(depth) = chain_depth
4591                        && depth > 0
4592                    {
4593                        let chain_idx = chain_pipes_base + i / depth;
4594                        let stage = i % depth;
4595                        let prev_stage = (stage + depth - 1) % depth;
4596                        // Close every fd in the chain that this
4597                        // stage does not own. Owned fds (kept open):
4598                        //   - chain[prev_stage][0]: read end of the
4599                        //     pipe predecessor writes to.
4600                        //   - chain[stage][1]: write end of the
4601                        //     pipe successor reads from.
4602                        // Everything else is the inverse end of an
4603                        // owned pipe or fully unrelated.
4604                        for (s, pipe) in guard.chain_pipes[chain_idx].iter().enumerate() {
4605                            // Always close the write end of the
4606                            // predecessor's pipe (we only need its
4607                            // read end).
4608                            if s == prev_stage {
4609                                unsafe {
4610                                    libc::close(pipe[1]);
4611                                }
4612                            // Always close the read end of our own
4613                            // pipe (we only need its write end).
4614                            } else if s == stage {
4615                                unsafe {
4616                                    libc::close(pipe[0]);
4617                                }
4618                            // Pipes belonging to neither this stage
4619                            // nor its predecessor: close both ends.
4620                            } else {
4621                                unsafe {
4622                                    libc::close(pipe[0]);
4623                                    libc::close(pipe[1]);
4624                                }
4625                            }
4626                        }
4627                        // Close every fd from other chains (any group).
4628                        for (cj, chain) in guard.chain_pipes.iter().enumerate() {
4629                            if cj != chain_idx {
4630                                for pipe in chain {
4631                                    unsafe {
4632                                        libc::close(pipe[0]);
4633                                        libc::close(pipe[1]);
4634                                    }
4635                                }
4636                            }
4637                        }
4638                    } else {
4639                        // This group has no chain pipes, but other
4640                        // groups may. Close every chain-pipe fd
4641                        // inherited via fork — leaving a primary
4642                        // group's chain pipe open in a composed
4643                        // worker would prevent the chain from ever
4644                        // observing EOF on its read ends.
4645                        for chain in guard.chain_pipes.iter() {
4646                            for pipe in chain {
4647                                unsafe {
4648                                    libc::close(pipe[0]);
4649                                    libc::close(pipe[1]);
4650                                }
4651                            }
4652                        }
4653                    }
4654                    // Layered defense against child-side unwinding
4655                    // reaching the forked-from-parent drops:
4656                    //
4657                    // 1. No-op panic hook — the default hook prints a
4658                    //    multi-line backtrace to stderr, which is a
4659                    //    shared fd with the parent post-fork. A panic
4660                    //    in the child would interleave garbled output
4661                    //    with the parent's tracing log and confuse
4662                    //    downstream parsers. Install a silent hook
4663                    //    before catch_unwind ONLY under
4664                    //    `panic = "unwind"` — under
4665                    //    `panic = "abort"` (release / nextest
4666                    //    profile) the silent hook would suppress the
4667                    //    panic message before SIGABRT fires, leaving
4668                    //    operators with zero diagnostic from a
4669                    //    crashed worker. Letting the default hook
4670                    //    write a backtrace to stderr is the lesser
4671                    //    evil under abort: the message may interleave
4672                    //    with parent tracing output but at least
4673                    //    surfaces the panic site. catch_unwind itself
4674                    //    is a no-op under abort (the closure is
4675                    //    invoked normally and the abort terminates
4676                    //    the child before any Err return), so gating
4677                    //    only the hook installation — not the
4678                    //    catch_unwind call — keeps the unwind-build
4679                    //    fast-path unchanged while restoring
4680                    //    release-build observability.
4681                    //
4682                    // 2. `_exit(0|1)` after the closure (success or
4683                    //    catch_unwind Err) — the child never returns
4684                    //    to a frame whose `SpawnGuard` Drop could
4685                    //    run. `_exit(2)` bypasses Rust's stack-unwind
4686                    //    drops and the static-destructor table both,
4687                    //    so the parent-owned `SpawnGuard` whose
4688                    //    storage was duplicated by `fork()` cannot
4689                    //    SIGKILL its siblings (fratricide) from the
4690                    //    child path.
4691                    //
4692                    // 3. `panic::catch_unwind` — catches any panic
4693                    //    before it escapes this arm. Belt-and-braces
4694                    //    against (a) additional Drops on this
4695                    //    frame's stack (e.g. future refactors that
4696                    //    add more RAII) and (b) alloc/OOM panics
4697                    //    during worker_main / postcard encode.
4698                    //
4699                    //    Caveat: catch_unwind is a no-op under
4700                    //    `panic = "abort"`, which ktstr's Cargo.toml
4701                    //    DOES set in `[profile.release]`. In release
4702                    //    builds a panic inside this closure aborts
4703                    //    the child immediately (SIGABRT); the
4704                    //    `catch_unwind` call compiles but never
4705                    //    returns `Err`, and neither the
4706                    //    `f.write_all(&bytes)` nor the `_exit(1)`
4707                    //    below runs on the panic path. The parent's
4708                    //    `stop_and_collect` therefore observes a
4709                    //    missing WorkerReport and fills in a
4710                    //    sentinel — that sentinel fallback IS the
4711                    //    release-build correctness mechanism. Under
4712                    //    abort the silent hook is NOT installed (see
4713                    //    item 1) so the default panic handler writes
4714                    //    the panic location and message to stderr
4715                    //    just before SIGABRT, giving operators the
4716                    //    diagnostic they would otherwise have lost.
4717                    //    Defense (2) still applies unchanged: abort
4718                    //    skips Drops (matching the `_exit` path's
4719                    //    no-Drop guarantee). Dev/test builds (cargo
4720                    //    test, cargo nextest run — dev profile
4721                    //    inherits default unwind semantics) still
4722                    //    get a real `catch_unwind` Err → `_exit(1)`
4723                    //    fast-path with a silent hook so the
4724                    //    backtrace doesn't pollute test output.
4725                    //
4726                    //    Global-state safety under unwind, scoped
4727                    //    to `worker_main`'s reachable code path —
4728                    //    the `fork()` child's observable set. Two
4729                    //    items: `STOP: AtomicBool` and
4730                    //    `STATIC_HOST_INFO: OnceLock<_>`. Neither
4731                    //    of them carries a Drop whose body touches
4732                    //    the inherited MAP_SHARED regions or the
4733                    //    parent-owned pipe fds. Under a
4734                    //    hypothetical unwind that escaped
4735                    //    `catch_unwind` (a double-panic that
4736                    //    bypasses the landing pad), the only
4737                    //    fork-child Drops that actually matter are
4738                    //    the parent-owned `SpawnGuard` (covered by
4739                    //    the `_exit` no-Drop guarantee above) and
4740                    //    the child-local `wake_latencies_ns` /
4741                    //    `migrations` `Vec<T>` (per-process heap,
4742                    //    no cross-process impact). `STATIC_HOST_INFO`'s
4743                    //    inner Drop frees a handful of
4744                    //    `Option<String>`s and is safe on either
4745                    //    side of fork. Crate-wide statics outside
4746                    //    this set (fetch, probe, vmm, …) are out
4747                    //    of scope — this audit pins only what the
4748                    //    fork-child can reach from `worker_main`.
4749                    //
4750                    // 4. `_exit(1)` on catch_unwind Err, `_exit(0)`
4751                    //    on Ok — bypasses Rust's global static
4752                    //    destructors that a plain `return` would
4753                    //    run.
4754                    //
4755                    // `AssertUnwindSafe` is justified: the child
4756                    // unconditionally _exits after this block, so no
4757                    // post-unwind invariant can be observed.
4758                    #[cfg(panic = "unwind")]
4759                    {
4760                        let _ = std::panic::take_hook();
4761                        std::panic::set_hook(Box::new(|_| {}));
4762                    }
4763                    let child_result =
4764                        std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
4765                            // Wait for parent to move us to cgroup before starting work.
4766                            // Use poll() with a 30s timeout — signal-safe after fork,
4767                            // prevents hanging forever if the parent stalls.
4768                            let mut pfd = libc::pollfd {
4769                                fd: start_fds[0],
4770                                events: libc::POLLIN,
4771                                revents: 0,
4772                            };
4773                            let ret = unsafe { libc::poll(&mut pfd, 1, 30_000) };
4774                            if ret <= 0 {
4775                                exit_child_error();
4776                            }
4777                            let mut buf = [0u8; 1];
4778                            let mut f = unsafe { std::fs::File::from_raw_fd(start_fds[0]) };
4779                            let _ = f.read_exact(&mut buf);
4780                            drop(f);
4781                            // Do NOT reset STOP here. STOP is initialized to
4782                            // false at the pre-handshake site above (after the
4783                            // signal-handler install) and a SIGUSR1 delivered
4784                            // between the mask unblock and this point reflects
4785                            // a legitimate parent-issued stop request — the
4786                            // explicit-start path in `stop_and_collect` skips
4787                            // the readiness barrier and races
4788                            // `kill(pid, SIGUSR1)` against the worker's start-
4789                            // byte read here, so any STOP=true observed at
4790                            // this point must propagate into the work loop.
4791                            // Clobbering it would lose the stop and produce a
4792                            // worker that loops until the killpg/SIGKILL
4793                            // escalation.
4794                            // Publish a single "ready" byte on the report pipe
4795                            // BEFORE entering `worker_main`. The parent's
4796                            // auto-start barrier in `stop_and_collect` polls
4797                            // every worker's report fd for POLLIN with a
4798                            // bounded deadline and consumes this byte as the
4799                            // explicit signal that the worker has finished
4800                            // its post-fork init (cgroup placement, SIGUSR1
4801                            // unblock) and is about to enter the
4802                            // work loop. Replaces the prior 500 ms blind
4803                            // sleep — under host CPU contention, real worker
4804                            // start latency can exceed that sleep, dropping
4805                            // a worker into its loop AFTER `stop_and_collect`
4806                            // has already signalled stop, which surfaces as
4807                            // a starvation false-positive.
4808                            //
4809                            // Done as a raw `libc::write` (not `File::write`)
4810                            // to keep the report-fd's ownership inside its
4811                            // existing `std::fs::File::from_raw_fd` block
4812                            // below — wrapping it here would close the fd
4813                            // when the local goes out of scope, before the
4814                            // post-`worker_main` write_all path. A 1-byte
4815                            // write to a freshly-grown 8 MiB pipe with no
4816                            // reader contention completes synchronously
4817                            // without blocking; signal-safe after fork.
4818                            //
4819                            // The byte is `b'r'`. The parent's collect path
4820                            // strips a leading `b'r'` before
4821                            // `postcard::from_bytes` so the
4822                            // explicit-start call site (which skips the
4823                            // barrier) still parses correctly. A zero return or short write
4824                            // is treated as best-effort: if the kernel
4825                            // refuses the write (EFAULT cannot occur with a
4826                            // stack pointer; EBADF cannot occur on a fresh
4827                            // fd; EPIPE only fires if the reader closed,
4828                            // which the parent does not do mid-spawn), the
4829                            // parent's barrier-deadline path falls back to
4830                            // the same "skip this worker" branch as a
4831                            // genuinely-dead worker.
4832                            let ready_byte: u8 = b'r';
4833                            unsafe {
4834                                libc::write(
4835                                    report_fds[1],
4836                                    &ready_byte as *const u8 as *const libc::c_void,
4837                                    1,
4838                                );
4839                            }
4840                            // Now run. Fork-mode workers thread the global
4841                            // STOP through `worker_main` — the SIGUSR1 handler
4842                            // is process-wide, so flipping `STOP` from
4843                            // `sigusr1_handler` is what reaches the loop's
4844                            // `stop.load(Relaxed)` checks.
4845                            let report = worker_main(
4846                                affinity,
4847                                group.work_type.clone(),
4848                                group.sched_policy,
4849                                group.mem_policy.clone(),
4850                                group.mpol_flags,
4851                                group.nice,
4852                                group.comm.as_deref(),
4853                                group.uid,
4854                                group.gid,
4855                                group.numa_node,
4856                                worker_pipe_fds,
4857                                worker_futex,
4858                                group.park_after,
4859                                // Fork worker: the real signal fd (or `-1`
4860                                // when the group did not opt in). The child
4861                                // holds its own fd-table copy of the parent's
4862                                // eventfd, inherited across `fork()`.
4863                                first_iter_signal_fd,
4864                                iter_slot,
4865                                phase_epoch_ptr,
4866                                &STOP,
4867                                group.group_idx,
4868                            );
4869                            let bytes = postcard::to_stdvec(&report).unwrap_or_default();
4870                            let mut f = unsafe { std::fs::File::from_raw_fd(report_fds[1]) };
4871                            if let Err(e) = f.write_all(&bytes) {
4872                                // tracing is unsafe in a post-fork child
4873                                // (the global subscriber may be holding a
4874                                // lock taken in another thread of the
4875                                // parent process before fork); eprintln
4876                                // goes straight to stderr without locking
4877                                // a tracing subscriber. The parent
4878                                // observes a partial / empty pipe payload
4879                                // and emits a sentinel WorkerReport with
4880                                // exit_info populated, but without this
4881                                // log line the underlying I/O error
4882                                // (EPIPE if the parent closed the read
4883                                // end early, EFBIG / ENOSPC on a backing-
4884                                // file pipe, EIO on a transient kernel
4885                                // failure) is invisible.
4886                                eprintln!("ktstr: worker report write_all failed: {e}");
4887                            }
4888                            drop(f);
4889                        }));
4890                    if child_result.is_ok() {
4891                        exit_child_success();
4892                    } else {
4893                        exit_child_error();
4894                    }
4895                }
4896                child_pid => {
4897                    // Parent: restore the pre-fork signal mask
4898                    // (the block was transient — only the child
4899                    // inherits the blocked SIGUSR1, and that
4900                    // child unblocks immediately after installing
4901                    // its handler). Without this restore, SIGUSR1
4902                    // would stay blocked in the calling thread for
4903                    // the lifetime of the workload, swallowing any
4904                    // signal directed at the parent process.
4905                    let psm_parent_restore_rc = unsafe {
4906                        libc::pthread_sigmask(libc::SIG_SETMASK, &old_mask, std::ptr::null_mut())
4907                    };
4908                    if psm_parent_restore_rc != 0 {
4909                        tracing::warn!(
4910                            rc = psm_parent_restore_rc,
4911                            "pthread_sigmask(SIG_SETMASK) failed restoring mask in parent post-fork; SIGUSR1 stays blocked in this thread for the lifetime of the workload"
4912                        );
4913                    }
4914                    // Close unused pipe ends.
4915                    unsafe {
4916                        libc::close(report_fds[1]);
4917                        libc::close(start_fds[0]);
4918                    }
4919                    // child_pid is positive by the -1 arm above, so
4920                    // fits in pid_t directly — store as pid_t so
4921                    // every downstream libc call avoids the u32→i32
4922                    // sign-cast wraparound bug.
4923                    guard.children.push(ForkedChild {
4924                        pid: child_pid,
4925                        report_fd: report_fds[0],
4926                        start_fd: start_fds[1],
4927                        kind: ForkedChildKind::Worker {
4928                            group_idx: group.group_idx,
4929                        },
4930                    });
4931                }
4932            }
4933        }
4934
4935        Ok(())
4936    }
4937
4938    /// Kernel TIDs of all worker tasks, in spawn order.
4939    ///
4940    /// Returned as `libc::pid_t` — the kernel's native type — so
4941    /// callers feed them directly into `kill`, `waitpid`,
4942    /// `Pid::from_raw`, and `sched_setaffinity` writes without any
4943    /// sign-cast at the libc boundary.
4944    ///
4945    /// # WARNING — `cgroup.procs` for `CloneMode::Thread`
4946    ///
4947    /// **For `CloneMode::Thread`, passing these TIDs to a
4948    /// `cgroup.procs` write migrates the ENTIRE test-runner process
4949    /// into that cgroup**: cgroup.procs writes are tgid-scoped, and
4950    /// every Thread worker shares the test runner's tgid. The first
4951    /// such write moves the test harness, every parent thread, and
4952    /// every sibling worker into the destination cgroup; subsequent
4953    /// writes are no-ops because they all point at the same tgid.
4954    /// Use cgroup v2 threaded-mode cgroups with `cgroup.threads`
4955    /// for per-thread placement. `CloneMode::Fork` is the right
4956    /// choice when each worker needs its own cgroup.
4957    ///
4958    /// # Per-mode interpretation
4959    ///
4960    /// - [`CloneMode::Fork`]: each entry is the worker's pid
4961    ///   (== tgid == kernel tid because the worker is its own
4962    ///   thread-group leader). Safe to feed into `cgroup.procs`.
4963    /// - [`CloneMode::Thread`]: each entry is the worker's
4964    ///   `gettid()` value — distinct kernel tasks inside the
4965    ///   parent's tgid. Safe for `sched_setaffinity(tid, ...)`;
4966    ///   safe for `cgroup.threads` writes under a threaded-mode
4967    ///   cgroup; **not** safe for `cgroup.procs` (see warning above).
4968    ///
4969    /// # Thread tid publish ordering
4970    ///
4971    /// Thread workers publish their `gettid()` via an
4972    /// `Arc<AtomicI32>` BEFORE the start handshake (the publish is
4973    /// the first thing the worker closure does, before blocking on
4974    /// `start_rx`). `spawn_thread_worker` blocks on a paired
4975    /// rendezvous channel until the worker reaches the publish
4976    /// point, so by the time [`WorkloadHandle::spawn`] returns,
4977    /// every thread worker's `tid` is non-zero (or spawn would have
4978    /// bailed with a "failed to publish gettid() within 2 s"
4979    /// error). Post-spawn `worker_pids()` is safe to call without
4980    /// first calling [`Self::start`]. The publish uses `Release`;
4981    /// this reader uses `Acquire`, pairing release-acquire so that
4982    /// any reader observing a non-zero tid is also guaranteed to
4983    /// observe the worker's post-publish state.
4984    ///
4985    /// # `pcomm` containers
4986    ///
4987    /// Groups configured with [`WorkSpec::pcomm`] yield a single
4988    /// container process whose tgid leader pid is what the parent
4989    /// holds; the per-thread `gettid()` values of the workers running
4990    /// inside the container are not exported across the process
4991    /// boundary. Each pcomm group contributes ONE entry to the
4992    /// returned vector — the container pid — regardless of how many
4993    /// thread workers it hosts. This pid is correct for `cgroup.procs`
4994    /// migration (the container's whole tgid moves) but is NOT
4995    /// suitable as a target for per-thread `sched_setaffinity` —
4996    /// see [`Self::set_affinity`] for the matching error path.
4997    ///
4998    /// Order: forked children (conventional workers + pcomm
4999    /// containers, in spawn order) followed by Thread-mode workers
5000    /// (in spawn order). A workload that mixes pcomm groups with
5001    /// non-pcomm Thread-mode groups produces both populated
5002    /// collections; a workload using only one dispatch path produces
5003    /// only one populated collection.
5004    pub fn worker_pids(&self) -> Vec<libc::pid_t> {
5005        let mut out = Vec::with_capacity(self.children.len() + self.threads.len());
5006        out.extend(self.children.iter().map(|c| c.pid));
5007        out.extend(self.threads.iter().map(|tw| tw.tid.load(Ordering::Acquire)));
5008        out
5009    }
5010
5011    /// Worker pids suitable for `cgroup.procs` migration.
5012    ///
5013    /// `cgroup.procs` is **tgid-scoped** in the kernel: writing a
5014    /// tid migrates the entire thread group containing that tid
5015    /// (`kernel/cgroup/cgroup.c::__cgroup_procs_write` resolves the
5016    /// passed pid to its leader via `find_lock_task_mm` /
5017    /// `cgroup_procs_write_start`). Under [`CloneMode::Thread`]
5018    /// every worker shares the test harness's tgid, so feeding
5019    /// [`Self::worker_pids`] to `cgroup.procs` would migrate the
5020    /// harness itself — catastrophic.
5021    ///
5022    /// Returns the per-worker pids when the spawn used
5023    /// [`CloneMode::Fork`] (each worker has its own tgid). Bails
5024    /// for [`CloneMode::Thread`] with an actionable diagnostic
5025    /// pointing at `cgroup.threads` (the thread-scoped sibling) as
5026    /// the right migration sink for thread workers.
5027    ///
5028    /// Callers that integrate with `cgroup.procs` writes — e.g.
5029    /// [`crate::cgroup::CgroupManager::move_tasks`] — should call
5030    /// this in place of [`Self::worker_pids`] so a misconfigured
5031    /// Thread-mode test fails at the migration step rather than
5032    /// silently moving the harness into the per-test cgroup.
5033    pub fn worker_pids_for_cgroup_procs(&self) -> Result<Vec<libc::pid_t>> {
5034        if !self.threads.is_empty() {
5035            anyhow::bail!(
5036                "WorkloadHandle::worker_pids_for_cgroup_procs: workers were \
5037                 spawned with CloneMode::Thread; their pids share the test \
5038                 harness's tgid and a `cgroup.procs` write would migrate the \
5039                 harness. Use `cgroup.threads` (thread-scoped) for Thread-mode \
5040                 workers, or switch to CloneMode::Fork."
5041            );
5042        }
5043        Ok(self.worker_pids())
5044    }
5045
5046    /// Signal all workers to start working (after they've been
5047    /// placed in cgroups, if applicable).
5048    ///
5049    /// Idempotent — subsequent calls after the first are no-ops.
5050    pub fn start(&mut self) {
5051        if self.started {
5052            return;
5053        }
5054        self.started = true;
5055        // Fork-mode: write a byte to each child's start pipe (covers
5056        // both conventional workers and pcomm containers — both wait
5057        // on the same start-byte handshake before entering their work
5058        // path).
5059        for child in &mut self.children {
5060            unsafe {
5061                libc::write(child.start_fd, b"s".as_ptr() as *const _, 1);
5062                libc::close(child.start_fd);
5063            }
5064            child.start_fd = -1;
5065        }
5066        // Thread-mode: send `()` on each worker's start_tx. The
5067        // SyncSender(0) rendezvous means each send blocks until the
5068        // worker calls recv(); if the worker has been joined or has
5069        // panicked before reaching recv, send returns Err which we
5070        // swallow (the join in stop_and_collect surfaces the real
5071        // exit). Take ownership so a future start() call (illegal
5072        // by the idempotence guard above) can't re-send.
5073        for tw in &mut self.threads {
5074            if let Some(tx) = tw.start_tx.take() {
5075                let _ = tx.send(());
5076            }
5077        }
5078    }
5079
5080    /// Set CPU affinity for worker at `idx`.
5081    ///
5082    /// For [`CloneMode::Fork`] the per-worker pid addresses a
5083    /// distinct kernel task. For [`CloneMode::Thread`] the worker's
5084    /// `gettid()` is what `sched_setaffinity(tid, ...)` accepts;
5085    /// this method reads the tid from the worker's
5086    /// `Arc<AtomicI32>` (with `Acquire` ordering, paired with the
5087    /// `Release` publish on the worker thread). Returns an error
5088    /// if the thread has not yet published its tid — call
5089    /// [`start()`](Self::start) first so the worker reaches its
5090    /// `gettid()` publish before reading.
5091    ///
5092    /// Bails for `ForkedChildKind::PcommContainer` entries: the
5093    /// container's tgid leader pid is the only kernel handle the
5094    /// parent holds for that group, but `sched_setaffinity` against
5095    /// that pid pins only the leader thread (a placeholder thread
5096    /// that never enters the work loop), not the worker threads
5097    /// running the workload. The container's worker tids are not
5098    /// published across the process boundary. Bake the affinity into
5099    /// [`WorkSpec::affinity`] at spawn time instead — `worker_main`
5100    /// applies it per-thread inside the container.
5101    ///
5102    /// Index space: `[0, children.len())` addresses forked children
5103    /// (conventional workers and pcomm containers), and
5104    /// `[children.len(), children.len() + threads.len())` addresses
5105    /// Thread-mode workers, matching the ordering of
5106    /// [`Self::worker_pids`].
5107    pub fn set_affinity(&self, idx: usize, cpus: &BTreeSet<usize>) -> Result<()> {
5108        let pid = if idx < self.children.len() {
5109            let child = &self.children[idx];
5110            if let ForkedChildKind::PcommContainer { groups } = &child.kind {
5111                let total: usize = groups.iter().map(|(_, n)| n).sum();
5112                anyhow::bail!(
5113                    "set_affinity: child {idx} is a pcomm container \
5114                     hosting {total} thread workers; per-thread \
5115                     tids are not exported across the process boundary. \
5116                     Set affinity via WorkSpec::affinity at spawn time \
5117                     so worker_main applies it inside the container."
5118                );
5119            }
5120            child.pid
5121        } else {
5122            let thread_idx = idx - self.children.len();
5123            let tid = self.threads[thread_idx].tid.load(Ordering::Acquire);
5124            if tid == 0 {
5125                anyhow::bail!(
5126                    "set_affinity: thread worker {thread_idx} has not yet \
5127                     published gettid() (call start() first)"
5128                );
5129            }
5130            tid
5131        };
5132        set_thread_affinity(pid, cpus)
5133    }
5134
5135    /// Read all workers' current iteration counts from shared memory.
5136    ///
5137    /// Each element is the monotonically increasing iteration count for
5138    /// that worker, read with Relaxed ordering. Returns an empty vec
5139    /// if no workers were spawned.
5140    ///
5141    /// # Ordering rationale — why Relaxed is sound
5142    ///
5143    /// Every producer (the worker-side store at the
5144    /// `worker_main` publish sites) writes its slot with Relaxed
5145    /// ordering, and this reader loads with Relaxed too. No
5146    /// happens-before edge is needed because no host-side consumer
5147    /// pairs the iteration count with OTHER shared state: the
5148    /// parent samples these counters to answer "is this worker
5149    /// still making progress?" and feeds deltas into gap
5150    /// detection, not into any data-dependent follow-up read from
5151    /// a different shared memory location. A stale value on one
5152    /// sample is self-correcting — the next snapshot picks up the
5153    /// newer count without any cross-field invariant to break.
5154    ///
5155    /// The per-slot single-producer / multi-sampler shape is
5156    /// inherently non-tearing on every supported target
5157    /// (AtomicU64 is architecture-primitive on x86_64 and aarch64
5158    /// LSE with 8-byte alignment enforced by the type). The only
5159    /// question is ordering, and the audit above concludes Relaxed
5160    /// is load-bearingly correct — promoting either side to
5161    /// Acquire/Release would add a barrier with no corresponding
5162    /// paired operation to synchronise with.
5163    pub fn snapshot_iterations(&self) -> Vec<u64> {
5164        if self.iter_counters.is_null() || self.iter_counter_len == 0 {
5165            return Vec::new();
5166        }
5167        (0..self.iter_counter_len)
5168            .map(|i| {
5169                // SAFETY: alignment + atomic-only-access invariant
5170                // established at the iter_counters mmap site in
5171                // `WorkloadHandle::spawn` and carried by the
5172                // `*mut AtomicU64` type. Relaxed ordering: see the
5173                // rationale in the outer doc comment.
5174                unsafe { &*self.iter_counters.add(i) }.load(Ordering::Relaxed)
5175            })
5176            .collect()
5177    }
5178
5179    /// Block until at least `n` workers have signalled their first counted
5180    /// iteration, or `deadline` passes. Returns `true` on the former,
5181    /// `false` on timeout. The evented counterpart to polling
5182    /// [`Self::snapshot_iterations`]: workers `write(2)` a `1` into the
5183    /// shared first-iteration eventfd (see
5184    /// [`WorkloadConfig::signal_first_iteration`]) and this drains the
5185    /// counter — which sums those `1`s — accumulating across reads until it
5186    /// reaches `n`.
5187    ///
5188    /// Caller contract: only meaningful when the workload set
5189    /// `signal_first_iteration` (and thus spawned fork workers with the
5190    /// eventfd). Without the eventfd there is nothing to wait on, so this
5191    /// returns `false` immediately (a `debug_assert` catches a misuse in
5192    /// debug builds). `n == 0` returns `true` vacuously.
5193    pub(crate) fn wait_first_iteration_all(&self, deadline: Instant, n: usize) -> bool {
5194        let Some(signal_fd) = self.first_iter_signal.as_ref() else {
5195            debug_assert!(
5196                false,
5197                "wait_first_iteration_all called on a handle with no first-iteration \
5198                 eventfd (workload did not set signal_first_iteration, or dispatch \
5199                 was not fork)",
5200            );
5201            return false;
5202        };
5203        use std::os::unix::io::AsRawFd;
5204        poll_eventfd_reaches(signal_fd.as_raw_fd(), deadline, n as u64)
5205    }
5206
5207    /// Broadcast the current scenario phase to this handle's workers.
5208    /// The scenario engine calls this at each step boundary — the
5209    /// 1-indexed phase (Step k → k + 1) at StepStart, `u32::MAX` (the
5210    /// inter-step gap sentinel) at StepEnd — so a backdrop worker
5211    /// drains a per-phase `PhaseSlice` on each transition. No-op when
5212    /// the handle has no `phase_epoch` region. Release store pairs with
5213    /// the worker's Relaxed load; it is telemetry, so no happens-before
5214    /// edge is required (see the `unsafe impl Send for WorkloadHandle`
5215    /// SAFETY note), but Release keeps the store promptly visible.
5216    pub(crate) fn set_phase_epoch(&self, epoch: u32) {
5217        if self.phase_epoch.is_null() {
5218            return;
5219        }
5220        // SAFETY: alignment + atomic-only-access invariant established
5221        // at the phase_epoch mmap site in `WorkloadHandle::spawn` /
5222        // `spawn_pcomm_cgroup` and carried by the `*mut AtomicU32`
5223        // type. The parent is the sole writer; workers are readers.
5224        unsafe { &*self.phase_epoch }.store(epoch, Ordering::Release);
5225    }
5226
5227    /// Stop all workers, collect their reports, and wait for exit.
5228    ///
5229    /// Auto-starts workers if [`start()`](Self::start) was not called,
5230    /// then waits on an event-driven barrier — each fork worker
5231    /// writes a single `b'r'` byte to its report pipe immediately
5232    /// after the start handshake completes, and the parent polls
5233    /// every report fd for `POLLIN` with a 5 s deadline. The
5234    /// barrier wakes the moment the slowest worker finishes its
5235    /// post-fork init, replacing the prior unconditional 500 ms
5236    /// sleep that under-waited under host CPU contention and
5237    /// over-waited on idle hosts. Thread-mode workers are pre-
5238    /// synchronised by `start()`'s `mpsc::sync_channel(0)` rendezvous,
5239    /// so the barrier is a no-op when no fork children were spawned.
5240    /// Consumes `self` -- workers cannot be restarted.
5241    ///
5242    /// Workers that fail to produce a report (died, timed out, or wrote
5243    /// corrupt data) get a zeroed-out sentinel report with `work_units: 0`.
5244    /// This ensures `assert_not_stuck` catches dead workers as starvation
5245    /// failures.
5246    ///
5247    /// # Shutdown latency
5248    ///
5249    /// Workers spend their steady-state time blocked inside a
5250    /// `futex_wait` with timeout `WORKER_STOP_POLL_NS` (~100 ms).
5251    /// The "stop signal" is a per-mode flag the worker checks on
5252    /// every futex-wait wake; the wake interval bounds shutdown
5253    /// latency.
5254    ///
5255    /// _Fork mode_ — `stop_and_collect` sends SIGUSR1 to each
5256    /// worker pid; the per-process `sigusr1_handler` flips the
5257    /// global `STOP` in that worker's CoW address space, and the
5258    /// worker observes it on the NEXT futex wake (partner-writes
5259    /// or the 100 ms timeout, whichever comes first). The signal
5260    /// handler is process-wide and reaches one worker per kill().
5261    ///
5262    /// _Thread mode_ — `stop_and_collect` calls
5263    /// `worker.stop.store(true, Relaxed)` directly on each
5264    /// worker's `Arc<AtomicBool>`. SIGUSR1 is process-wide and
5265    /// useless for per-thread stop control, so no signal is sent;
5266    /// the worker observes the flag flip on its next futex-wait
5267    /// wake at the same 100 ms cadence.
5268    ///
5269    /// Callers that budget a graceful-shutdown window should
5270    /// allow at least one `WORKER_STOP_POLL_NS` tick (~100 ms)
5271    /// between flag flip and final collect, over and above any
5272    /// report-flush / IO latency. Tighter windows can race the
5273    /// worker's pre-stop iteration and surface as a missing
5274    /// report, which is then mapped to the sentinel path above.
5275    ///
5276    /// # Exit-shape invariance
5277    ///
5278    /// Collection discriminates purely on the presence and validity of
5279    /// the worker's pipe-delivered postcard payload — **not** on
5280    /// `waitpid` exit status. Under `panic = "unwind"` (dev/test
5281    /// profile) the worker's
5282    /// `catch_unwind` arm calls `_exit(1)` so the parent sees
5283    /// `WIFEXITED=true`, `WEXITSTATUS=1`; under `panic = "abort"`
5284    /// (release profile) the worker aborts with `SIGABRT` so the parent
5285    /// sees `WIFEXITED=false`, `WTERMSIG=6`. Either way, a panicking
5286    /// worker never finishes `f.write_all(&bytes)` on the report pipe,
5287    /// so the collect poll + bounded read hands back an empty (or
5288    /// truncated) buffer, `postcard::from_bytes` fails, and the
5289    /// sentinel path fires. Partial writes from a panic between successful
5290    /// `write_all` and `_exit(0)` are not reachable — the write is the
5291    /// last non-trivial statement inside the catch_unwind closure.
5292    /// The `waitpid` call later in this function exists solely for
5293    /// reaping zombies; its return value feeds only the "still alive
5294    /// → SIGKILL escalate" branch and is never mapped to report
5295    /// state (the sentinel path DOES now read it to populate
5296    /// [`WorkerExitInfo`] on the attached diagnostic, but the
5297    /// correctness discrimination — sentinel vs real report — still
5298    /// happens purely on pipe payload presence).
5299    pub fn stop_and_collect(mut self) -> Vec<WorkerReport> {
5300        // Auto-start if not explicitly started (workers in parent cgroup)
5301        let was_started = self.started;
5302        self.start();
5303
5304        // Event-driven worker-started barrier for the auto-start path.
5305        //
5306        // ORDERING CONSTRAINT: this drain MUST run BEFORE the SIGUSR1
5307        // stop signals below. A fork worker signalled before it enters
5308        // its measurement loop reports 0 iterations and looks starved
5309        // when it was merely slow to start; waiting for each worker's
5310        // post-handshake ready byte guarantees stop lands after the
5311        // worker is counting. `drain_ready_bytes` documents the poll
5312        // mechanics and the POLLHUP-drop semantics.
5313        //
5314        // Thread-mode workers need no barrier here: `start()` above sent
5315        // on a `mpsc::sync_channel(0)` SyncSender(0) rendezvous, which
5316        // blocks the parent until the worker's matching `recv()` returns
5317        // — by the time `start()` returns every thread worker has
5318        // crossed its start handshake. Only the fork-mode pipe-based
5319        // start signal is fire-and-forget, so the barrier is gated on
5320        // `!self.children.is_empty()`.
5321        //
5322        // Budget: the 5 s deadline mirrors the collect deadline below;
5323        // worst case (no worker ever writes its byte) bounds the wait at
5324        // the same 5 s the legacy sleep + collect path budgeted for the
5325        // entire stop_and_collect.
5326        if !was_started && !self.children.is_empty() {
5327            let pending: Vec<i32> = self.children.iter().map(|c| c.report_fd).collect();
5328            drain_ready_bytes(pending, Instant::now() + Duration::from_secs(5));
5329        }
5330
5331        let mut reports = Vec::new();
5332        let children = std::mem::take(&mut self.children);
5333        let threads = std::mem::take(&mut self.threads);
5334
5335        // NOTE: the explicit-start (was_started) ready-byte drain used
5336        // to sit here, ahead of the stop signals, as a blocking 1-byte
5337        // read per child. That wedged teardown whenever a worker the
5338        // scheduler under test never ran had not yet written its byte
5339        // (guest PID-1 blocked forever → VM watchdog). It now runs
5340        // AFTER the signals below, via the bounded `drain_ready_bytes`.
5341
5342        // Signal all fork-mode children to stop via SIGUSR1; the
5343        // signal handler flips that child's process-local STOP, which
5344        // worker_main's `stop_requested` checks read. For pcomm
5345        // containers the SIGUSR1 flips the container's STOP, which
5346        // every thread inside the container observes via `&STOP`
5347        // passed to `worker_main` — one signal stops the whole
5348        // group cleanly. `pid` is `libc::pid_t`, so it flows to
5349        // `Pid::from_raw` without the u32→i32 sign-cast wraparound
5350        // that produced `kill(-1, ...)` session-wide reaps when the
5351        // old u32 pid exceeded i32::MAX.
5352        for child in &children {
5353            let _ = nix::sys::signal::kill(
5354                nix::unistd::Pid::from_raw(child.pid),
5355                nix::sys::signal::Signal::SIGUSR1,
5356            );
5357        }
5358        // Signal all thread-mode workers by flipping each worker's
5359        // per-task `stop`. SIGUSR1 is process-wide and useless for
5360        // per-thread stop; the Arc<AtomicBool> threaded through
5361        // worker_main is the only path that reaches an individual
5362        // thread without affecting siblings.
5363        for tw in &threads {
5364            tw.stop.store(true, Ordering::Relaxed);
5365        }
5366
5367        // Explicit-start (was_started) ready-byte drain. Signal-first
5368        // (above) so healthy workers begin tearing down during any drain
5369        // wait, and the drain is bounded (`drain_ready_bytes`, same 5 s
5370        // as the barrier) so a worker the scheduler under test never ran
5371        // — and thus never wrote its 'r' — cannot wedge teardown on a
5372        // blocking read. The stale byte must still be consumed: the
5373        // collect poll below routes on POLLIN, so a byte left in the
5374        // pipe would send a report-less worker down the payload-read
5375        // branch instead of the sentinel path.
5376        if was_started {
5377            let pending: Vec<i32> = children.iter().map(|c| c.report_fd).collect();
5378            drain_ready_bytes(pending, Instant::now() + Duration::from_secs(5));
5379        }
5380
5381        // Collect reports with a shared 5s deadline across all workers.
5382        // Each worker gets the remaining budget, so no-progress workers
5383        // (e.g. under degrade mode) don't serially exhaust the VM
5384        // timeout.
5385        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
5386
5387        // Bound on the post-SIGKILL worker reap (below). A worker normally
5388        // exits <<1s after SIGKILL (post-crash bypass keeps it
5389        // CFS-schedulable). The rare wedge: a sched_setaffinity flipper
5390        // stuck uninterruptible in `affine_move_task`'s `wait_for_completion`
5391        // — a migration that can't finish until the target becomes
5392        // migratable, which during the crash/bypass window can take seconds
5393        // — cannot take its pending SIGKILL until that syscall returns, so a
5394        // blocking `waitpid` would stall collect for that whole window.
5395        // The deadline is SHARED across workers: every such wedge waits out
5396        // the SAME crash/bypass window, so once one times out the rest get
5397        // the remaining (~0) budget and give up immediately rather than
5398        // summing to N×timeout. Mirrors the scheduler-side
5399        // `reap_child_bounded` (rust_init/scheduler.rs); on timeout the zombie is left
5400        // for VM reboot to reap (SIGKILL cannot make it exit faster — see
5401        // that fn's doc).
5402        const WORKER_REAP_TIMEOUT: Duration = Duration::from_secs(3);
5403        let mut worker_reap_deadline: Option<Instant> = None;
5404        // Evented, bounded reap of a SIGKILLed worker `pid`: wait up to
5405        // `timeout` for it to become a zombie (`pidfd_open` + `poll`),
5406        // then reap non-blocking. Returns false on timeout (worker still
5407        // uninterruptibly wedged — the rare case `WORKER_REAP_TIMEOUT`
5408        // documents; left for VM reboot).
5409        fn reap_pid_bounded(pid: libc::pid_t, timeout: Duration) -> bool {
5410            // Non-blocking reap of `pid` (observed StillAlive just above,
5411            // so this succeeds only once it has become a zombie).
5412            let reap = || {
5413                matches!(
5414                    nix::sys::wait::waitpid(
5415                        nix::unistd::Pid::from_raw(pid),
5416                        Some(nix::sys::wait::WaitPidFlag::WNOHANG),
5417                    ),
5418                    Ok(nix::sys::wait::WaitStatus::Exited(..))
5419                        | Ok(nix::sys::wait::WaitStatus::Signaled(..))
5420                )
5421            };
5422            match crate::sync::pidfd_poll_exited(pid, timeout) {
5423                // Readable => zombie => the WNOHANG reap is non-blocking.
5424                crate::sync::PidfdWait::Exited => {
5425                    let _ = reap();
5426                    true
5427                }
5428                // Still uninterruptibly wedged (see `WORKER_REAP_TIMEOUT`):
5429                // leave the zombie-to-be for VM reboot (SIGKILL already
5430                // pending).
5431                crate::sync::PidfdWait::TimedOut => false,
5432                // pidfd_open failed (ESRCH/gone): one non-blocking reap.
5433                crate::sync::PidfdWait::NoPidfd => reap(),
5434            }
5435        }
5436
5437        for child in children {
5438            let mut buf = Vec::new();
5439            let remaining = deadline.saturating_duration_since(std::time::Instant::now());
5440            let ms = remaining.as_millis().min(i32::MAX as u128) as i32;
5441            let mut poll_ready = false;
5442            if ms > 0 {
5443                let mut pfd = libc::pollfd {
5444                    fd: child.report_fd,
5445                    events: libc::POLLIN,
5446                    revents: 0,
5447                };
5448                let ready = unsafe { libc::poll(&mut pfd, 1, ms) };
5449                poll_ready = ready > 0;
5450            }
5451
5452            let npid = nix::unistd::Pid::from_raw(child.pid);
5453            if !poll_ready {
5454                // Deadline expired without POLLIN — the child produced
5455                // no report, so `buf` stays empty (→ sentinel path).
5456                // Kill first, then close, so no writer is left blocked
5457                // on a pipe the parent has stopped draining (the child
5458                // may have written only the 'r' ready byte but no
5459                // payload).
5460                kill_and_killpg(npid, nix::sys::signal::Signal::SIGKILL);
5461                let _ = nix::unistd::close(child.report_fd);
5462            } else {
5463                // Child responded (POLLIN) — read the payload bounded by
5464                // the SHARED collect `deadline`, then close and kill. A
5465                // worker that wrote its 'r' (or a partial payload) and
5466                // then wedged before EOF would otherwise hold a blocking
5467                // `read_to_end` hostage; polling to the deadline and
5468                // giving up yields a short/empty buf, so the postcard /
5469                // serde decode fails and the existing sentinel path
5470                // fires. Preserve read→close→kill order (the old
5471                // File-drop close is now an explicit `close`).
5472                let mut chunk = [0u8; 64 * 1024];
5473                loop {
5474                    let remaining = deadline.saturating_duration_since(std::time::Instant::now());
5475                    // ms == 0 is fine: a zero-timeout poll still drains
5476                    // data already buffered in the pipe without
5477                    // blocking, which keeps the healthy at-deadline-edge
5478                    // case intact (the child exited; payload + EOF are
5479                    // buffered and read out in one pass here).
5480                    let ms = remaining.as_millis().min(i32::MAX as u128) as i32;
5481                    let mut pfd = libc::pollfd {
5482                        fd: child.report_fd,
5483                        events: libc::POLLIN,
5484                        revents: 0,
5485                    };
5486                    // SAFETY: single owned pollfd; nfds == 1; ms >= 0.
5487                    let ret = unsafe { libc::poll(&mut pfd, 1, ms) };
5488                    if ret < 0 {
5489                        if std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted
5490                        {
5491                            continue;
5492                        }
5493                        break;
5494                    }
5495                    if ret == 0 {
5496                        // Timeout — deadline reached with no more data.
5497                        break;
5498                    }
5499                    // SAFETY: `chunk` is a valid owned buffer of
5500                    // `chunk.len()` bytes; `report_fd` is the read end
5501                    // the parent owns until the close below.
5502                    let n = unsafe {
5503                        libc::read(
5504                            child.report_fd,
5505                            chunk.as_mut_ptr() as *mut libc::c_void,
5506                            chunk.len(),
5507                        )
5508                    };
5509                    if n > 0 {
5510                        buf.extend_from_slice(&chunk[..n as usize]);
5511                        continue;
5512                    }
5513                    if n == 0 {
5514                        // EOF — writer closed; payload complete.
5515                        break;
5516                    }
5517                    // n < 0: retry on EINTR, otherwise give up.
5518                    if std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted {
5519                        continue;
5520                    }
5521                    break;
5522                }
5523                let _ = nix::unistd::close(child.report_fd);
5524                kill_and_killpg(npid, nix::sys::signal::Signal::SIGKILL);
5525            }
5526            let waited = nix::sys::wait::waitpid(npid, Some(nix::sys::wait::WaitPidFlag::WNOHANG));
5527            let still_running = matches!(waited, Ok(nix::sys::wait::WaitStatus::StillAlive));
5528            let exit_info_source: Result<nix::sys::wait::WaitStatus, nix::errno::Errno> =
5529                if still_running {
5530                    // Bounded reap (see `reap_pid_bounded` + the shared
5531                    // `worker_reap_deadline` above): replaces a blocking
5532                    // `waitpid(npid, None)` that stalled on a worker
5533                    // uninterruptibly wedged in an affine_move_task
5534                    // migration (seconds; see `WORKER_REAP_TIMEOUT`).
5535                    // exit_info stays `StillAlive` either way — the prior
5536                    // blocking reap also discarded the status here.
5537                    let rd = *worker_reap_deadline
5538                        .get_or_insert_with(|| Instant::now() + WORKER_REAP_TIMEOUT);
5539                    let remaining = rd.saturating_duration_since(Instant::now());
5540                    reap_pid_bounded(child.pid, remaining);
5541                    Ok(nix::sys::wait::WaitStatus::StillAlive)
5542                } else {
5543                    waited
5544                };
5545
5546            // Strip a leading `b'r'` ready byte if it is still present.
5547            // Children write this byte unconditionally on every success
5548            // path right after the start handshake; both collect paths
5549            // drain it up front now (`drain_ready_bytes`, called from
5550            // the auto-start barrier and the explicit-start drain). This
5551            // strip is the defensive fallback for the worker that wrote
5552            // its byte only after the bounded drain gave up at its
5553            // deadline — the byte then sits in the pipe ahead of the
5554            // payload here. Strip exactly 1 byte if present so the
5555            // per-kind decoder sees a clean payload either way.
5556            let report_slice: &[u8] = if buf.first() == Some(&b'r') {
5557                &buf[1..]
5558            } else {
5559                &buf[..]
5560            };
5561
5562            // Per-kind decoding. `Worker` carries a single postcard
5563            // `WorkerReport`. `PcommContainer` carries
5564            // a `serde_json` `Vec<WorkerReport>` — serde_json for
5565            // the pcomm container; fork-mode workers use postcard
5566            // for the single `WorkerReport`. Both
5567            // payloads ride the EOF-terminated pipe with no length
5568            // prefix; the parent's `read_to_end` provides framing.
5569            // On a decode failure, emit one sentinel per expected
5570            // report so downstream consumers (per-group filtering,
5571            // assert_not_stuck) see the correct cardinality.
5572            match child.kind {
5573                ForkedChildKind::Worker { group_idx } => {
5574                    let decoded: Result<WorkerReport, _> = postcard::from_bytes(report_slice);
5575                    if let Ok(report) = decoded {
5576                        reports.push(report);
5577                    } else {
5578                        let exit_info = classify_wait_outcome(exit_info_source);
5579                        eprintln!(
5580                            "ktstr: worker pid={} returned no report ({} bytes read, exit={exit_info:?})",
5581                            child.pid,
5582                            buf.len(),
5583                        );
5584                        reports.push(WorkerReport {
5585                            tid: child.pid,
5586                            group_idx,
5587                            exit_info: Some(exit_info),
5588                            ..WorkerReport::default()
5589                        });
5590                    }
5591                }
5592                ForkedChildKind::PcommContainer { groups } => {
5593                    let total_workers: usize = groups.iter().map(|(_, n)| n).sum();
5594                    let decoded: Result<Vec<WorkerReport>, _> =
5595                        serde_json::from_slice(report_slice);
5596                    match decoded {
5597                        Ok(mut decoded_reports) if decoded_reports.len() == total_workers => {
5598                            reports.append(&mut decoded_reports);
5599                        }
5600                        Ok(mut decoded_reports) => {
5601                            // Cardinality mismatch — surface the
5602                            // partial set we did get and pad with
5603                            // sentinels so the total report count
5604                            // still equals `total_workers`. A short
5605                            // payload typically signals the
5606                            // thread-group leader died mid-encode
5607                            // (panic in `serde_json::to_vec`, OOM
5608                            // during Vec growth, or a write_all
5609                            // truncated by SIGKILL escalation).
5610                            // Sentinels are tagged with the right
5611                            // per-group `group_idx` using the
5612                            // `groups` layout: groups[0] consumed
5613                            // the first groups[0].1 slots,
5614                            // groups[1] the next groups[1].1, etc.
5615                            // We emit sentinels for the trailing
5616                            // missing slots, computing each
5617                            // sentinel's group_idx by walking the
5618                            // layout from `got` forward.
5619                            let exit_info = classify_wait_outcome(exit_info_source);
5620                            eprintln!(
5621                                "ktstr: pcomm thread-group leader pid={} returned {} of {} reports ({} bytes read, exit={exit_info:?})",
5622                                child.pid,
5623                                decoded_reports.len(),
5624                                total_workers,
5625                                buf.len(),
5626                            );
5627                            // Surplus reports (decoded > total_workers)
5628                            // must not leak into the parent's report
5629                            // stream — downstream cardinality assertions
5630                            // (per-group filtering, assert_not_stuck)
5631                            // assume exactly `total_workers` entries.
5632                            // Truncate to the layout's total before the
5633                            // `got..total_workers` loop runs (which is
5634                            // a no-op for the surplus case after this).
5635                            decoded_reports.truncate(total_workers);
5636                            let got = decoded_reports.len();
5637                            for r in decoded_reports {
5638                                reports.push(r);
5639                            }
5640                            // Compute group_idx for each missing
5641                            // slot by walking the layout. `slot` is
5642                            // a 0-based global index from `got` to
5643                            // `total_workers - 1`; we accumulate
5644                            // per-group counts to find which group
5645                            // owns each slot.
5646                            for slot in got..total_workers {
5647                                let mut acc = 0usize;
5648                                let mut g_idx = 0usize;
5649                                for &(gi, n) in &groups {
5650                                    if slot < acc + n {
5651                                        g_idx = gi;
5652                                        break;
5653                                    }
5654                                    acc += n;
5655                                }
5656                                reports.push(WorkerReport {
5657                                    tid: child.pid,
5658                                    group_idx: g_idx,
5659                                    exit_info: Some(exit_info.clone()),
5660                                    ..WorkerReport::default()
5661                                });
5662                            }
5663                        }
5664                        Err(_) => {
5665                            let exit_info = classify_wait_outcome(exit_info_source);
5666                            eprintln!(
5667                                "ktstr: pcomm thread-group leader pid={} returned no decodable report ({} bytes read, exit={exit_info:?})",
5668                                child.pid,
5669                                buf.len(),
5670                            );
5671                            // Total decode failure — emit one
5672                            // sentinel per slot, tagging each with
5673                            // the correct group_idx.
5674                            for &(gi, n) in &groups {
5675                                for _ in 0..n {
5676                                    reports.push(WorkerReport {
5677                                        tid: child.pid,
5678                                        group_idx: gi,
5679                                        exit_info: Some(exit_info.clone()),
5680                                        ..WorkerReport::default()
5681                                    });
5682                                }
5683                            }
5684                        }
5685                    }
5686                }
5687            }
5688        }
5689
5690        // Thread-mode collection: join each worker's JoinHandle
5691        // (with the [`THREAD_JOIN_TIMEOUT`] budget) and adopt the
5692        // returned [`WorkerReport`]. Per-worker `stop` was flipped
5693        // above; the worker observes it in worker_main's
5694        // `stop.load(Relaxed)` checks (max ~100ms latency from the
5695        // FUTEX_WAIT_TIMEOUT poll cadence). Three outcomes:
5696        //
5697        //   1. Ok(report): join returned the worker's WorkerReport.
5698        //      Push as-is.
5699        //   2. Err(payload): the thread panicked. Build a sentinel
5700        //      report and attach
5701        //      `exit_info: Some(WorkerExitInfo::Panicked(msg))`
5702        //      where `msg` comes from `extract_panic_payload`.
5703        //   3. Timeout (5s elapsed without is_finished): emit a
5704        //      tracing::warn and push a sentinel with
5705        //      `exit_info: Some(WorkerExitInfo::TimedOut)` —
5706        //      `worker_main` should have observed the per-worker
5707        //      `stop` within 100ms, so a 5s no-show signals a
5708        //      genuinely stuck worker (deadlock, infinite spin,
5709        //      blocking syscall the runtime can't interrupt).
5710        //      stop_and_collect does NOT process::exit on timeout —
5711        //      the orphan thread keeps running until the test
5712        //      harness exits, but any subsequent worker uses a
5713        //      fresh per-worker `stop` so the orphan can't pollute
5714        //      later runs.
5715        for mut tw in threads {
5716            // Drop start_tx (idempotent — `start()` may have already
5717            // taken it). If start() ran first, `start_tx` is
5718            // already `None` and the take is a no-op; if the caller
5719            // skipped start() entirely, dropping start_tx here
5720            // signals the worker via `Disconnected` so it exits
5721            // cleanly without the rendezvous send.
5722            tw.start_tx.take();
5723            let tid = tw.tid.load(Ordering::Acquire);
5724            if let Some(j) = tw.join.take() {
5725                match join_thread_with_timeout(j, &tw.exit_evt, THREAD_JOIN_TIMEOUT) {
5726                    Some(Ok(report)) => reports.push(report),
5727                    Some(Err(payload)) => {
5728                        let msg = extract_panic_payload(payload);
5729                        eprintln!("ktstr: thread worker tid={tid} panicked: {msg}");
5730                        reports.push(WorkerReport {
5731                            tid,
5732                            completed: false,
5733                            exit_info: Some(WorkerExitInfo::Panicked(msg)),
5734                            ..WorkerReport::default()
5735                        });
5736                    }
5737                    None => {
5738                        tracing::warn!(
5739                            tid,
5740                            timeout_secs = THREAD_JOIN_TIMEOUT.as_secs(),
5741                            "thread worker did not join within timeout — leaking the \
5742                             thread; sentinel report attached with TimedOut exit_info"
5743                        );
5744                        reports.push(WorkerReport {
5745                            tid,
5746                            completed: false,
5747                            exit_info: Some(WorkerExitInfo::TimedOut),
5748                            ..WorkerReport::default()
5749                        });
5750                    }
5751                }
5752            }
5753        }
5754
5755        reports
5756    }
5757
5758    /// Deliver the terminal stop to every live worker without
5759    /// reaping. Signal-only: NO `waitpid`, NO fd close, NO consume of
5760    /// `self`, no blocking of any kind. Reaping, the fd/region
5761    /// cleanup, and (for `stop_and_collect`) report decoding all
5762    /// belong to the subsequent [`Self::stop_and_collect`] or
5763    /// [`Drop`] — doing any of them here would double-reap. This only
5764    /// pre-delivers the terminal signal so that reap finds workers
5765    /// already exiting.
5766    ///
5767    /// # Why (fork mode — the case this exists for)
5768    ///
5769    /// When the sched_ext scheduler under test errors out, the kernel
5770    /// disables sched_ext and its tasks fall back to the builtin
5771    /// scheduler, where CPU-bound workers keep spinning under
5772    /// contention. The normal reap in `stop_and_collect` SIGKILLs one
5773    /// worker then *blocks* in `waitpid` for it while the rest have
5774    /// had only the cooperative SIGUSR1 and keep spinning, so each
5775    /// dying worker must win a CPU slice against every still-runnable
5776    /// sibling before it can run its exit path — the serial reap
5777    /// becomes scheduling-gated and teardown time scales with worker
5778    /// count. `kill_and_killpg(SIGKILL)` here is delivered, not
5779    /// cooperative: once every worker carries a pending SIGKILL none
5780    /// re-enters userspace, so no worker competes for the CPU while
5781    /// the reap drains them, and each killed worker's report-pipe
5782    /// write end closes so the reap's `poll` returns `POLLHUP`
5783    /// immediately instead of waiting out its deadline. One
5784    /// `kill_and_killpg` per `ForkedChild` covers both a plain worker
5785    /// and a pcomm container: SIGKILL on any thread is fatal to the
5786    /// whole thread group, and `killpg` additionally reaches
5787    /// descendants — every forked child is its own process-group
5788    /// leader via `setpgid(0, 0)`, so pgid == child pid (mirrors the
5789    /// `killpg` in [`Drop`]).
5790    ///
5791    /// # Thread mode
5792    ///
5793    /// Thread-mode workers share the harness tgid and cannot be
5794    /// signalled individually, so there is NO SIGKILL fast path for
5795    /// them. This flips each worker's cooperative `stop` flag — the
5796    /// same store `stop_and_collect` and [`Drop`] already do, just
5797    /// earlier — and the worker still only observes it on its next
5798    /// futex-wait wake (`WORKER_STOP_POLL_NS`, ~100 ms). It is
5799    /// therefore NOT a teardown speedup for thread mode; it is kept
5800    /// for symmetry and is cheap and harmless (an idempotent relaxed
5801    /// store). The win is fork-mode only, which is where the
5802    /// SIGKILL-vs-cooperative distinction exists.
5803    ///
5804    /// # Frozen cgroups
5805    ///
5806    /// A cgroup-frozen worker is not a special case here. The freezer
5807    /// parks a userspace task in `TASK_INTERRUPTIBLE` (the jobctl
5808    /// freezer trap), so a fatal signal wakes it and it exits on its
5809    /// next `get_signal` — SIGKILL kills a frozen worker with no
5810    /// unfreeze required. (`collect_step` / `collect_backdrop` still
5811    /// unfreeze each cgroup as their first teardown step for reasons of
5812    /// their own; this method does not depend on that.)
5813    pub(crate) fn sigkill_workers(&self) {
5814        for child in &self.children {
5815            kill_and_killpg(
5816                nix::unistd::Pid::from_raw(child.pid),
5817                nix::sys::signal::Signal::SIGKILL,
5818            );
5819        }
5820        for tw in &self.threads {
5821            tw.stop.store(true, Ordering::Relaxed);
5822        }
5823    }
5824}
5825
5826/// Bounded drain of the per-worker "ready byte". Each fork worker
5827/// writes a single `b'r'` byte to its report pipe immediately after
5828/// the start-pipe handshake completes (see the matching write inside
5829/// `worker_main`'s catch_unwind closure). This polls every still-
5830/// pending fd for `POLLIN` with the remaining budget until `deadline`,
5831/// consuming exactly one byte per worker, and returns once every fd
5832/// has produced its byte / hung up or the deadline expires.
5833///
5834/// The raw `libc::read` does NOT take ownership of any fd — the caller
5835/// still owns them via the `children` Vec and reads the payload tail on
5836/// its own deadline.
5837///
5838/// Per-fd outcomes:
5839///   - `POLLIN`, `read` == 1: ready byte consumed; drop from pending.
5840///   - `POLLIN`, `read` == 0: EOF — writer closed without sending the
5841///     byte (worker died pre-write). Drop; the collect path reads 0
5842///     bytes and the sentinel-report branch fires.
5843///   - `POLLIN`, `read` < 0, EINTR: keep pending, re-poll next cycle.
5844///   - `POLLIN`, `read` < 0, other: warn and drop (treat as ready).
5845///   - `POLLHUP` / `POLLERR` / `POLLNVAL`: worker closed the write end
5846///     without sending the byte (panic during post-fork init, or
5847///     kernel-killed before the write). Drop; sentinel path fires.
5848///   - EINTR on `poll`: re-poll with the remaining budget.
5849///   - hard `poll` failure (EFAULT impossible with an owned Vec;
5850///     ENOMEM extreme): warn and bail; the collect path's per-fd poll
5851///     handles each worker individually.
5852///
5853/// Bounding is the point: a worker the scheduler under test never ran
5854/// never writes its byte, so a blocking read here would wedge teardown
5855/// (guest PID-1 blocks forever → host watchdog). The deadline lets a
5856/// never-scheduled worker fall through to the collect sentinel path
5857/// instead.
5858fn drain_ready_bytes(mut pending: Vec<i32>, deadline: Instant) {
5859    while !pending.is_empty() {
5860        let remaining = deadline.saturating_duration_since(Instant::now());
5861        if remaining.is_zero() {
5862            // Deadline expired with workers still pending. Stop
5863            // waiting; the collect path treats any worker whose pipe
5864            // never produced data as a sentinel report.
5865            break;
5866        }
5867        let ms = remaining.as_millis().min(i32::MAX as u128) as i32;
5868        let mut pfds: Vec<libc::pollfd> = pending
5869            .iter()
5870            .map(|&fd| libc::pollfd {
5871                fd,
5872                events: libc::POLLIN,
5873                revents: 0,
5874            })
5875            .collect();
5876        // SAFETY: `pfds` is a non-empty owned Vec; nfds matches its
5877        // length; `ms >= 0` since the remaining-zero branch bails
5878        // first. Return codes are interpreted below.
5879        let ret = unsafe { libc::poll(pfds.as_mut_ptr(), pfds.len() as libc::nfds_t, ms) };
5880        if ret < 0 {
5881            let err = std::io::Error::last_os_error();
5882            if err.kind() == std::io::ErrorKind::Interrupted {
5883                // EINTR — re-poll with the remaining budget. The
5884                // deadline guard bounds total wait regardless of EINTR
5885                // frequency.
5886                continue;
5887            }
5888            tracing::warn!(
5889                %err,
5890                pending = pending.len(),
5891                "drain_ready_bytes: poll failed; falling through to per-worker collect"
5892            );
5893            break;
5894        }
5895        // ret == 0 means the per-iteration timeout fired without any fd
5896        // ready — measured against `remaining`, so the next iteration's
5897        // saturating_duration_since is zero and the top-of-loop guard
5898        // exits. Don't break here: a future cycle stays useful if the
5899        // clock jumped backward, at the cost of one bail iteration.
5900        if ret > 0 {
5901            pending.retain(|&fd| {
5902                // Find this fd's pollfd entry. Linear scan is fine:
5903                // typical worker counts are <100 and a HashMap costs
5904                // more in setup than the scan saves.
5905                let pfd = pfds.iter().find(|p| p.fd == fd);
5906                let revents = pfd.map(|p| p.revents).unwrap_or(0);
5907                if revents & libc::POLLIN != 0 {
5908                    let mut byte: u8 = 0;
5909                    // SAFETY: `&mut byte` is a valid 1-byte buffer; `fd`
5910                    // is the report read end the caller owns until
5911                    // collect drains it. 0 / -1 returns handled below.
5912                    let n = unsafe { libc::read(fd, &mut byte as *mut u8 as *mut libc::c_void, 1) };
5913                    if n >= 0 {
5914                        // n == 1: byte consumed. n == 0: EOF pre-write.
5915                        // Either way drop from pending.
5916                        return false;
5917                    }
5918                    let err = std::io::Error::last_os_error();
5919                    if err.kind() == std::io::ErrorKind::Interrupted {
5920                        return true;
5921                    }
5922                    tracing::warn!(
5923                        %err,
5924                        fd,
5925                        "drain_ready_bytes: byte read failed; treating worker as ready"
5926                    );
5927                    return false;
5928                }
5929                if revents & (libc::POLLHUP | libc::POLLERR | libc::POLLNVAL) != 0 {
5930                    // Worker closed the write end without sending the
5931                    // byte. Drop; the sentinel-report branch fires.
5932                    return false;
5933                }
5934                // No POLLIN, no hangup — keep waiting.
5935                true
5936            });
5937        }
5938    }
5939}
5940
5941impl Drop for WorkloadHandle {
5942    fn drop(&mut self) {
5943        use nix::sys::signal::{Signal, kill};
5944        use nix::sys::wait::waitpid;
5945        use nix::unistd::{Pid, close};
5946
5947        // Forked children (conventional workers AND pcomm
5948        // containers). `pid` is `libc::pid_t` — stored as i32 so
5949        // `Pid::from_raw` receives the kernel's native representation
5950        // directly, not the sign-cast of a u32 that could alias
5951        // negative values (including -1, i.e. every process in the
5952        // session).
5953        for child in &self.children {
5954            let pid = child.pid;
5955            let nix_pid = Pid::from_raw(pid);
5956            // killpg first: reach descendants the child may have
5957            // forked (Custom workloads, ForkExit caught mid-fork).
5958            // pgid == child pid because every forked child (worker
5959            // or pcomm container) calls `setpgid(0, 0)` at fork time.
5960            // ESRCH (group gone / no members) is expected and not a
5961            // warning-worthy failure; swallow it to keep the log
5962            // clean when the common no-descendants case drops.
5963            if let Err(e) = nix::sys::signal::killpg(nix_pid, Signal::SIGKILL)
5964                && e != nix::errno::Errno::ESRCH
5965            {
5966                tracing::warn!(pid, %e, "killpg failed in WorkloadHandle::drop");
5967            }
5968            if let Err(e) = kill(nix_pid, Signal::SIGKILL) {
5969                tracing::warn!(pid, %e, "kill failed in WorkloadHandle::drop");
5970            }
5971            if let Err(e) = waitpid(nix_pid, None) {
5972                tracing::warn!(pid, %e, "waitpid failed in WorkloadHandle::drop");
5973            }
5974            for fd in [child.report_fd, child.start_fd] {
5975                if fd >= 0
5976                    && let Err(e) = close(fd)
5977                {
5978                    tracing::warn!(fd, %e, "close failed in WorkloadHandle::drop");
5979                }
5980            }
5981        }
5982        // Thread-mode workers: flip stop, drop start_tx (in case
5983        // worker hasn't yet recv'd), join with the same 5s budget
5984        // `stop_and_collect` uses. Threads share the parent's
5985        // address space — there is no `kill` equivalent and no
5986        // MAP_SHARED ownership to give back. Drop still applies
5987        // the timeout so a stuck worker doesn't pin
5988        // `WorkloadHandle::drop` indefinitely; on timeout we log
5989        // the leak via `tracing::warn!` and proceed.
5990        let threads = std::mem::take(&mut self.threads);
5991        for mut tw in threads {
5992            tw.stop.store(true, Ordering::Relaxed);
5993            tw.start_tx.take();
5994            if let Some(j) = tw.join.take() {
5995                let tid = tw.tid.load(Ordering::Acquire);
5996                match join_thread_with_timeout(j, &tw.exit_evt, THREAD_JOIN_TIMEOUT) {
5997                    Some(Ok(_)) => {}
5998                    Some(Err(e)) => {
5999                        let payload = extract_panic_payload(e);
6000                        tracing::warn!(
6001                            tid,
6002                            payload,
6003                            "thread worker panicked in WorkloadHandle::drop"
6004                        );
6005                    }
6006                    None => {
6007                        tracing::warn!(
6008                            tid,
6009                            timeout_secs = THREAD_JOIN_TIMEOUT.as_secs(),
6010                            "thread worker failed to join within timeout in \
6011                             WorkloadHandle::drop — leaking the thread"
6012                        );
6013                    }
6014                }
6015            }
6016        }
6017        // Close inter-worker pipe pairs and chain pipes AFTER worker
6018        // shutdown. Ordering matters for Thread mode: every worker
6019        // thread shares the parent's fd table, so closing a pipe fd
6020        // before its using thread joins would surface to that thread
6021        // as `EBADF` on the next read/write/poll syscall. The
6022        // children-reap loop above and the threads-join loop above
6023        // both block until their worker is reaped or joined; only
6024        // then do these closes run, which is when the workers are
6025        // guaranteed to no longer touch their fds. Fork mode is
6026        // unaffected either way: each child held its own fd-table
6027        // copy via `fork()`, so this close is a no-op for the
6028        // child's view (its own copy was closed by the post-fork
6029        // close-other-fds block in spawn_group).
6030        //
6031        // Errors from `close` are logged via `tracing::warn!` rather
6032        // than swallowed — `EBADF` here would indicate a double-close
6033        // (an aliased ownership bug) and is more diagnostic than the
6034        // SpawnGuard early-bail path's silent close. SpawnGuard's
6035        // Drop swallows EBADF deliberately because mid-spawn the
6036        // guard may share fd ownership with already-closed
6037        // half-allocated state; the handle on the other hand has
6038        // sole ownership at this point.
6039        for (ab, ba) in &self.pipe_pairs {
6040            for fd in [ab[0], ab[1], ba[0], ba[1]] {
6041                if let Err(e) = close(fd) {
6042                    tracing::warn!(fd, %e, "close failed for pipe_pair fd in WorkloadHandle::drop");
6043                }
6044            }
6045        }
6046        for chain in &self.chain_pipes {
6047            for pipe in chain {
6048                for fd in [pipe[0], pipe[1]] {
6049                    if let Err(e) = close(fd) {
6050                        tracing::warn!(fd, %e, "close failed for chain_pipe fd in WorkloadHandle::drop");
6051                    }
6052                }
6053            }
6054        }
6055        for (&ptr, &size) in self.futex_ptrs.iter().zip(self.futex_region_sizes.iter()) {
6056            unsafe {
6057                libc::munmap(ptr as *mut libc::c_void, size);
6058            }
6059        }
6060        if !self.iter_counters.is_null() && self.iter_counter_len > 0 {
6061            unsafe {
6062                libc::munmap(
6063                    self.iter_counters as *mut libc::c_void,
6064                    self.iter_counter_len * std::mem::size_of::<u64>(),
6065                );
6066            }
6067        }
6068        if !self.phase_epoch.is_null() && self.phase_epoch_bytes > 0 {
6069            unsafe {
6070                libc::munmap(
6071                    self.phase_epoch as *mut libc::c_void,
6072                    self.phase_epoch_bytes,
6073                );
6074            }
6075        }
6076    }
6077}
6078
6079/// SIGUSR1 handler installed in the fork-mode child post-fork. Flips
6080/// the per-process global [`STOP`] so `worker_main`'s outer loop
6081/// exits at the next `stop_requested` check.
6082pub(super) extern "C" fn sigusr1_handler(_: libc::c_int) {
6083    STOP.store(true, Ordering::Relaxed);
6084}
6085
6086/// Render an actionable hint for a failed
6087/// `mmap(MAP_SHARED | MAP_ANONYMOUS)` call based on the observed
6088/// `errno`. Shared between the futex-region mmap and the
6089/// iter_counters mmap in [`WorkloadHandle::spawn`] so the two
6090/// sites emit identical hint text per errno — a drift would mean
6091/// two related failures produce inconsistent remediation advice.
6092///
6093/// Takes `Option<i32>` (the output of `std::io::Error::raw_os_error`)
6094/// so an unrecognised errno folds cleanly through the `_ => ""`
6095/// arm without forcing callers to `unwrap`.
6096///
6097/// The leading space on every non-empty arm lets callers format
6098/// as `"...failed: {errno}{hint};"` without having to add a
6099/// conditional separator — an empty hint disappears cleanly.
6100pub(super) fn mmap_shared_anon_errno_hint(errno: Option<i32>) -> &'static str {
6101    match errno {
6102        Some(libc::ENOMEM) => {
6103            " (ENOMEM: host is out of memory \
6104             or /proc/sys/vm/max_map_count is too low — \
6105             check `sysctl vm.max_map_count` and `free -h`)"
6106        }
6107        Some(libc::EPERM) => {
6108            " (EPERM: MAP_SHARED|MAP_ANONYMOUS \
6109             rejected by the kernel — check memory cgroup \
6110             limits and container seccomp policy)"
6111        }
6112        Some(libc::EINVAL) => {
6113            " (EINVAL: invalid length or \
6114             flag combination — verify num_workers > 0 so the \
6115             region size is non-zero, and that the total size \
6116             does not overflow usize)"
6117        }
6118        _ => "",
6119    }
6120}
6121
6122#[cfg(test)]
6123mod testing;
6124#[cfg(test)]
6125mod tests_composed;
6126#[cfg(test)]
6127mod tests_fan_out;
6128#[cfg(test)]
6129mod tests_futex;
6130#[cfg(test)]
6131mod tests_grandchild;
6132#[cfg(test)]
6133mod tests_idle_churn;
6134#[cfg(test)]
6135mod tests_integration;
6136#[cfg(test)]
6137mod tests_lifecycle;
6138#[cfg(test)]
6139mod tests_mempolicy;
6140#[cfg(test)]
6141mod tests_misc;
6142#[cfg(test)]
6143mod tests_pcomm;
6144#[cfg(test)]
6145mod tests_sched_policy;
6146#[cfg(test)]
6147mod tests_spawn_guard;
6148#[cfg(test)]
6149mod tests_thread_mode;
6150#[cfg(test)]
6151mod tests_wake_chain;
6152
6153/// Direct tests for the bounded ready-byte drain that fix 1a extracted
6154/// from the auto-start barrier. A full fake-child `stop_and_collect`
6155/// test would need a hand-rolled fork worker that never writes its
6156/// byte; exercising `drain_ready_bytes` against raw pipe fds covers the
6157/// three outcomes that matter — never-writes (deadline bound), one byte
6158/// consumed exactly, and POLLHUP drop — without that scaffolding.
6159#[cfg(test)]
6160mod tests_drain_bound {
6161    use super::drain_ready_bytes;
6162    use std::time::{Duration, Instant};
6163
6164    // Blocking pipe; returns (read_fd, write_fd). Both fds must be
6165    // closed by the caller.
6166    fn make_pipe() -> (i32, i32) {
6167        let mut fds = [0i32; 2];
6168        // SAFETY: `fds` is a valid 2-element buffer.
6169        let rc = unsafe { libc::pipe(fds.as_mut_ptr()) };
6170        assert_eq!(rc, 0, "pipe() failed: {}", std::io::Error::last_os_error());
6171        (fds[0], fds[1])
6172    }
6173
6174    // A worker the scheduler never ran leaves its write end open with
6175    // no data: the drain must block until the deadline (never wedge on
6176    // the blocking read the old code used) and then return.
6177    #[test]
6178    fn writer_never_writes_returns_at_deadline() {
6179        let (r, w) = make_pipe();
6180        let budget = Duration::from_millis(300);
6181        let start = Instant::now();
6182        drain_ready_bytes(vec![r], Instant::now() + budget);
6183        let elapsed = start.elapsed();
6184        assert!(
6185            elapsed >= Duration::from_millis(250),
6186            "returned before the deadline: {elapsed:?}"
6187        );
6188        assert!(
6189            elapsed < Duration::from_secs(3),
6190            "exceeded the deadline bound: {elapsed:?}"
6191        );
6192        // SAFETY: both fds are still open and owned here.
6193        unsafe {
6194            libc::close(r);
6195            libc::close(w);
6196        }
6197    }
6198
6199    // The ready byte is consumed exactly once: a following payload byte
6200    // must remain in the pipe for the collect read.
6201    #[test]
6202    fn ready_byte_consumed_exactly_one() {
6203        let (r, w) = make_pipe();
6204        let payload = b"rX";
6205        // SAFETY: valid buffer/len; `w` is open.
6206        let n = unsafe { libc::write(w, payload.as_ptr() as *const libc::c_void, payload.len()) };
6207        assert_eq!(n, 2, "short write");
6208        let start = Instant::now();
6209        drain_ready_bytes(vec![r], Instant::now() + Duration::from_secs(5));
6210        assert!(
6211            start.elapsed() < Duration::from_secs(1),
6212            "did not return promptly after the byte arrived: {:?}",
6213            start.elapsed()
6214        );
6215        let mut buf = [0u8; 8];
6216        // SAFETY: valid buffer/len; `r` is open.
6217        let got = unsafe { libc::read(r, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) };
6218        assert_eq!(got, 1, "exactly one byte should remain after the drain");
6219        assert_eq!(buf[0], b'X', "the wrong byte was consumed");
6220        // SAFETY: both fds still open and owned.
6221        unsafe {
6222            libc::close(r);
6223            libc::close(w);
6224        }
6225    }
6226
6227    // Writer closes without sending the byte: POLLHUP must drop the fd
6228    // immediately rather than waiting out the deadline.
6229    #[test]
6230    fn closed_writer_dropped_via_pollhup() {
6231        let (r, w) = make_pipe();
6232        // SAFETY: `w` is open; closing it makes `r` report POLLHUP.
6233        unsafe {
6234            libc::close(w);
6235        }
6236        let start = Instant::now();
6237        drain_ready_bytes(vec![r], Instant::now() + Duration::from_secs(5));
6238        assert!(
6239            start.elapsed() < Duration::from_secs(1),
6240            "did not drop on POLLHUP: {:?}",
6241            start.elapsed()
6242        );
6243        // SAFETY: `r` still open and owned.
6244        unsafe {
6245            libc::close(r);
6246        }
6247    }
6248}
6249
6250/// Direct tests for `poll_eventfd_reaches` (the evented first-iteration
6251/// wait's core) against a raw counter-mode eventfd — no worker spawn
6252/// needed. Covers the two contract halves the verifier probe relies on:
6253/// fewer than `n` signals times out at the deadline; exactly `n` (summed
6254/// across writes) wakes early.
6255#[cfg(test)]
6256mod tests_poll_eventfd {
6257    use super::poll_eventfd_reaches;
6258    use std::time::{Duration, Instant};
6259
6260    // Counter-mode eventfd (EFD_NONBLOCK so a drained read cannot block).
6261    fn make_eventfd() -> i32 {
6262        let fd = unsafe { libc::eventfd(0, libc::EFD_NONBLOCK | libc::EFD_CLOEXEC) };
6263        assert!(
6264            fd >= 0,
6265            "eventfd() failed: {}",
6266            std::io::Error::last_os_error()
6267        );
6268        fd
6269    }
6270
6271    fn signal(fd: i32, times: u64) {
6272        for _ in 0..times {
6273            let one: u64 = 1;
6274            let n = unsafe { libc::write(fd, &one as *const u64 as *const libc::c_void, 8) };
6275            assert_eq!(
6276                n,
6277                8,
6278                "eventfd write short: {}",
6279                std::io::Error::last_os_error()
6280            );
6281        }
6282    }
6283
6284    /// Fewer than `n` signals: the wait must time out at the deadline and
6285    /// return `false` (a never-run worker leaves the count short).
6286    #[test]
6287    fn under_count_times_out_false() {
6288        let fd = make_eventfd();
6289        signal(fd, 2); // k = 2 < n = 4
6290        let start = Instant::now();
6291        let ok = poll_eventfd_reaches(fd, Instant::now() + Duration::from_millis(200), 4);
6292        let elapsed = start.elapsed();
6293        assert!(!ok, "under-count must not report all-signalled");
6294        assert!(
6295            elapsed >= Duration::from_millis(150),
6296            "returned before the deadline: {elapsed:?}",
6297        );
6298        assert!(
6299            elapsed < Duration::from_secs(3),
6300            "exceeded the deadline bound: {elapsed:?}"
6301        );
6302        unsafe {
6303            libc::close(fd);
6304        }
6305    }
6306
6307    /// Exactly `n` signals (accumulated in the counter): the wait wakes
6308    /// early with `true`, well before a generous deadline.
6309    #[test]
6310    fn full_count_returns_true_early() {
6311        let fd = make_eventfd();
6312        signal(fd, 4); // n = 4
6313        let start = Instant::now();
6314        let ok = poll_eventfd_reaches(fd, Instant::now() + Duration::from_secs(5), 4);
6315        assert!(ok, "n signals must report all-signalled");
6316        assert!(
6317            start.elapsed() < Duration::from_secs(1),
6318            "did not wake promptly on the count: {:?}",
6319            start.elapsed(),
6320        );
6321        unsafe {
6322            libc::close(fd);
6323        }
6324    }
6325
6326    /// The count sums across SEPARATE reads: signals that arrive after the
6327    /// first drained read still accumulate toward `n`.
6328    #[test]
6329    fn count_accumulates_across_reads() {
6330        let fd = make_eventfd();
6331        signal(fd, 3);
6332        // First: 3 < 4, short deadline → false, draining the counter to 0.
6333        assert!(!poll_eventfd_reaches(
6334            fd,
6335            Instant::now() + Duration::from_millis(120),
6336            4
6337        ));
6338        // A single further signal added to the fresh (drained) counter is
6339        // only 1, so a second call still needs the FULL 4 — the accumulator
6340        // is per-call, matching one probe run. Confirm 4 fresh signals wake.
6341        signal(fd, 4);
6342        assert!(poll_eventfd_reaches(
6343            fd,
6344            Instant::now() + Duration::from_secs(5),
6345            4
6346        ));
6347        unsafe {
6348            libc::close(fd);
6349        }
6350    }
6351
6352    /// `need == 0` (no workers) returns `true` vacuously without polling.
6353    #[test]
6354    fn zero_need_is_vacuously_true() {
6355        let fd = make_eventfd();
6356        assert!(poll_eventfd_reaches(fd, Instant::now(), 0));
6357        unsafe {
6358            libc::close(fd);
6359        }
6360    }
6361}