ktstr/workload/schbench/
run.rs

1//! schbench's run engine, ported faithfully from `schbench.c`: the
2//! message-thread / worker-thread topology, the lockless wait-list, the
3//! handshake-driven wakeup-latency loop, and the matrix work under the
4//! per-CPU lock. The default (non-RPS) mode is the wakeup-latency +
5//! request-latency benchmark, with per-phase schedstat (run-delay) capture; the
6//! RPS-injector mode (`-R`) and its auto-RPS closed-loop rate control (`-A`)
7//! layer on the request queue + once-per-second control thread; pipe mode (`-p`)
8//! swaps the matrix work for a memory-transfer simulation over the same handshake.
9//!
10//! # Topology (`schbench.c` `message_thread` :1540, `worker_thread` :1419)
11//!
12//! `message_threads` message threads each spawn `worker_threads` worker
13//! threads in one process (schbench is single-process pthreads; schbench_rs
14//! re-expresses that with [`std::thread`], so the
15//! [`handshake`](super::handshake) futex is PRIVATE and the per-CPU locks +
16//! matrices live in one address space). A worker loops: block until its
17//! message thread wakes it (measuring wakeup latency), then think-sleep +
18//! matrix work under the per-CPU lock (measuring request latency). The message
19//! thread batch-wakes all waiting workers (`run_msg_thread` :1166 /
20//! `xlist_wake_all` :969).
21//!
22//! # Fidelity
23//!
24//! - Clock: `CLOCK_MONOTONIC` (ruling), not schbench's `gettimeofday`
25//!   wall-clock. Monotonic is freeze-robust (a host-side VM pause cannot make
26//!   a delta go negative) and is the correct source for a latency delta. The
27//!   measured quantity -- elapsed nanoseconds between two reads -- is identical
28//!   to what schbench measures; only the clock id differs.
29//! - The lockless wait-list is a Treiber stack ([`TreiberStack`]) over an
30//!   intrusive next-pointer, matching schbench's `xlist_add`/`xlist_splice`
31//!   cmpxchg list (`schbench.c:866-896`): userspace CAS only, no lock/syscall
32//!   on the hot path, so the syscall profile stays futex-dominated like
33//!   schbench's. (schbench hand-duplicates this for its thread list and its
34//!   request list; schbench_rs shares one generic implementation.)
35//! - do_work timing: the matrix multiply compiles to the SAME scalar inner loop
36//!   as schbench's `-O2` `do_some_math` -- a `u64` multiply-accumulate
37//!   (`imul` with a fused memory operand, then `add`), NOT SIMD. It does not
38//!   auto-vectorize: there is no vector `u64` multiply on the build target
39//!   (`x86-64-v3`; `vpmullq` is AVX-512) and the serial accumulator reduction
40//!   blocks SLP -- exactly as for schbench's `-O2` build (its Makefile
41//!   `CFLAGS = -Wall -O2`, no `-march`, likewise scalar; verified by disassembly
42//!   of both). A debug build (opt-level 0) emits far more overhead (no register
43//!   allocation, unrolling, or memory-operand fusion), so request latency runs
44//!   higher than reference schbench's. ABSOLUTE side-by-side fidelity vs the
45//!   reference (the `ktstr-schbench-validate` comparison) therefore needs a
46//!   release build. The in-ktstr A/B comparison (perf-delta / per-phase claims)
47//!   is build-invariant -- both sides use the same build -- so it is unaffected.
48
49use core::cell::UnsafeCell;
50use core::ptr;
51use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, AtomicU64, AtomicUsize, Ordering};
52use std::fs::File;
53use std::io::{Read, Seek, SeekFrom};
54
55use super::percpu_lock::PerCpuLocks;
56use super::plat::{Percentiles, PlatStats};
57
58/// Read `CLOCK_MONOTONIC` as nanoseconds (ruling: monotonic, not wall-clock).
59/// Self-contained so the schbench modules build into the standalone validation
60/// binary without reaching into `worker`'s `pub(super)` clock wrapper.
61fn monotonic_nanos() -> u64 {
62    // SAFETY: `clock_gettime` writes a `timespec` through the out-pointer and
63    // reads nothing else; CLOCK_MONOTONIC is always available on Linux.
64    let mut ts: libc::timespec = unsafe { core::mem::zeroed() };
65    let rc = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut ts) };
66    assert_eq!(rc, 0, "clock_gettime(CLOCK_MONOTONIC) failed");
67    (ts.tv_sec as u64) * 1_000_000_000 + ts.tv_nsec as u64
68}
69
70/// A node that can be linked into a [`TreiberStack`] via an intrusive
71/// next-pointer it owns.
72trait Linked: Sized {
73    fn next_link(&self) -> &AtomicPtr<Self>;
74}
75
76/// Lockless LIFO stack via an intrusive next-pointer, the shared form of
77/// schbench's `xlist_add`/`xlist_splice` (`schbench.c:866-896`). Nodes are
78/// referenced by raw pointer and must outlive every concurrent operation; the
79/// caller owns their storage and lifetimes.
80struct TreiberStack<T: Linked> {
81    head: AtomicPtr<T>,
82}
83
84impl<T: Linked> TreiberStack<T> {
85    fn new() -> Self {
86        Self {
87            head: AtomicPtr::new(ptr::null_mut()),
88        }
89    }
90
91    /// Push `node` onto the stack. Faithful port of `xlist_add`
92    /// (`schbench.c:866`): set the node's link to the current head, then
93    /// CAS-publish it, retrying on contention. The caller must not push a node
94    /// that is already on the stack (its link would be overwritten while
95    /// reachable) -- schbench upholds this (a worker re-queues only after being
96    /// spliced off and woken).
97    fn add(&self, node: *mut T) {
98        // SAFETY: `node` is a valid pointer to a `T` that outlives all list
99        // operations (caller contract).
100        let link = unsafe { (*node).next_link() };
101        loop {
102            let old = self.head.load(Ordering::Acquire);
103            link.store(old, Ordering::Relaxed);
104            if self
105                .head
106                .compare_exchange(old, node, Ordering::AcqRel, Ordering::Acquire)
107                .is_ok()
108            {
109                return;
110            }
111        }
112    }
113
114    /// Atomically take the whole stack, leaving it empty, returning the head
115    /// (null if empty). Faithful port of `xlist_splice` (`schbench.c:884`): an
116    /// atomic swap to null (the single-op equivalent of schbench's CAS loop).
117    /// The returned chain is walked via [`Linked::next_link`] in LIFO order
118    /// (schbench's thread-list splice does not reverse; wake order is
119    /// irrelevant).
120    fn splice(&self) -> *mut T {
121        self.head.swap(ptr::null_mut(), Ordering::AcqRel)
122    }
123
124    /// Atomically take the whole stack and return the chain in FIFO (insertion)
125    /// order -- the reverse of [`splice`](Self::splice)'s LIFO. Faithful port of
126    /// `request_splice` (`schbench.c:920-940`): swap to null, then reverse the
127    /// spliced chain so the consumer services the oldest queued request first
128    /// (request order matters, unlike the thread wait-list). Returns null if
129    /// empty. Single-consumer: only the owning worker drains its own queue.
130    fn splice_reversed(&self) -> *mut T {
131        let mut cur = self.head.swap(ptr::null_mut(), Ordering::AcqRel);
132        let mut rev: *mut T = ptr::null_mut();
133        while !cur.is_null() {
134            // SAFETY: `cur` is a node that was published on the stack and whose
135            // storage the caller owns; the single consumer reverses the chain it
136            // just took exclusive ownership of, so no other thread touches these
137            // links concurrently.
138            let link = unsafe { (*cur).next_link() };
139            let next = link.load(Ordering::Acquire);
140            link.store(rev, Ordering::Relaxed);
141            rev = cur;
142            cur = next;
143        }
144        rev
145    }
146}
147
148/// A queued work request in RPS-injector mode (`schbench.c` `struct request`,
149/// `:757`). The RPS thread heap-allocates one (`Box`) per request and links it
150/// onto the target worker's queue; the worker drains its queue and frees each --
151/// matching schbench's per-request malloc/free (part of the syscall profile the
152/// standalone validation compares). schbench's `start_time` field is set
153/// (`:951`) but never read, so it is omitted: the wakeup latency is measured
154/// from the worker's `wake_time` (stamped by the RPS thread at enqueue, and by
155/// the worker itself before it blocks -- see [`rps_wait`]) and the request
156/// latency from a local work-start -- neither reads `start_time`.
157/// Omitting it also drops schbench's per-request `gettimeofday` for that dead
158/// field (`:951`) -- one fewer clock syscall per request, consistent with the
159/// engine's `CLOCK_MONOTONIC`-vs-`gettimeofday` clock divergence.
160struct Request {
161    /// Intrusive link for the per-worker request [`TreiberStack`].
162    next: AtomicPtr<Request>,
163}
164
165impl Request {
166    fn new() -> Self {
167        Self {
168            next: AtomicPtr::new(ptr::null_mut()),
169        }
170    }
171}
172
173impl Linked for Request {
174    fn next_link(&self) -> &AtomicPtr<Self> {
175        &self.next
176    }
177}
178
179/// Declarative config for the [`Schbench`](crate::workload::WorkType::Schbench)
180/// workload. Construct via [`SchbenchConfig::default`] (schbench's own
181/// defaults) plus the chainable setters, e.g.
182/// `SchbenchConfig::default().message_threads(2).worker_threads(4)`. Derives
183/// Clone/Debug/PartialEq/Eq/Hash/serde; the builder shape follows
184/// [`WorkloadConfig`](crate::workload::WorkloadConfig), but `Eq`+`Hash` (which
185/// `WorkloadConfig` and `WorkSpec` omit because of their transitive `f64`) are
186/// available here since every field is integer/bool -- the ktstr f64-free
187/// leaf-config convention.
188///
189/// # schbench(8) CLI parity
190///
191/// This port re-expresses schbench's default (matrix-work) mode natively, so
192/// its tunables are config fields and topology rather than CLI flags. The
193/// mapping to schbench's option table (`schbench.c:138-187`):
194///
195/// | schbench flag | ktstr |
196/// |---|---|
197/// | `-m` message-threads | `message_threads` |
198/// | `-t` threads | `worker_threads` (workers per message thread; `0` = `ceil(cpuset_cpus / message_threads)`, see below) |
199/// | `-F` cache_footprint | `cache_footprint_kib` |
200/// | `-n` operations | `operations` |
201/// | `-s` sleep_usec | `sleep_usec` |
202/// | `-L` no-locking | `skip_locking` |
203/// | `-R` rps | `requests_per_sec` |
204/// | `-A` auto-rps | `auto_rps` |
205/// | `--split` (long-only) | `split_percent` (`None` = no split, all-private) |
206/// | `-p` pipe (also `--pipe`) | `pipe_transfer_bytes` (`0` = off; memory-transfer mode, no matrix work) |
207/// | `-r` runtime | in-VM: the scenario engine's run window (the engine runs until `stop`); host-side: the `run_secs` argument to [`run_standalone`](crate::workload::run_standalone) |
208///
209/// ## Set by ktstr topology, not a flag
210///
211/// - `-t` default: with `worker_threads = 0`, ktstr matches schbench's `-t`
212///   0-default -- it divides the CPU count across the message threads,
213///   `ceil(cpus / message_threads)` per thread (`schbench.c:1849-1852`), so the
214///   total worker count stays near the CPU count. ktstr scopes "cpus" to the
215///   allocated guest cpuset (the worker's `sched_getaffinity` mask, set by the
216///   scenario's topology / `CgroupDef`) rather than schbench's `get_nprocs`, so
217///   the total is ≈ the cpuset's CPU count. An explicit non-zero `worker_threads`
218///   is workers-per-message-thread in both.
219/// - `-M` (message-cpus) / `-W` (worker-cpus) thread pinning: ktstr places
220///   threads through its affinity / cpuset layer, so there is deliberately no
221///   per-thread-pin knob.
222///
223/// ## Observability flags -> the metric API
224///
225/// schbench's `-w` (warmuptime), `-i` (intervaltime), `-z` (zerotime), `-j`
226/// (json), and `-J` (jobname) shape its streaming stderr/JSON report. ktstr's
227/// numbers flow through the metric API instead -- per-phase attribution and the
228/// sidecar -- so these have no flag equivalent. `ktstr-schbench-validate`
229/// reproduces schbench's stderr-table shape for a side-by-side comparison.
230///
231/// ## Split mode (`--split`)
232///
233/// `Some(p)` partitions `cache_footprint_kib` into a per-thread private matrix
234/// (`p`%) and ONE process-global shared matrix (`100-p`%) that every worker
235/// multiplies into concurrently, reproducing schbench's cross-core
236/// shared-working-set cache contention (`schbench.c:1390-1404`, `:1858-1863`).
237/// ktstr models the shared matrix with `AtomicU64` `Relaxed` accesses. Like
238/// schbench's emitted code, the shared kernel keeps the running sum in a register
239/// and STORES it to each shared C cell on every inner (`k`) iteration -- C is
240/// write-only in the loop (A and B are loaded each `k`, C is never reloaded), and
241/// that per-k store is what generates the contention. Both gcc and clang keep the
242/// per-k store: `do_some_math` reads `m1`/`m2`/`m3` as offsets into one base
243/// pointer, so neither can prove the `m3` store doesn't alias the next `k`'s
244/// `m1`/`m2` loads. On x86-64 a `Relaxed` load/store lowers to a plain `MOV` (no
245/// `LOCK`), so the contention is identical to schbench's plain shared-memory race
246/// -- but sound (atomics, no data race), with zero unsafe. `None` (default) is
247/// the legacy all-private single matrix.
248///
249/// ## Pipe mode (`-p`)
250///
251/// `pipe_transfer_bytes > 0` REPLACES the matrix workload with schbench's
252/// memory-transfer simulation (`schbench.c:177`, `pipe_test`). It rides the
253/// message-handshake path: the message thread memsets each woken worker's
254/// per-thread page to `1` (`schbench.c:980-981`) and the worker memsets its own
255/// page to `2` before blocking (`schbench.c:1003-1004`), `pipe_transfer_bytes`
256/// bytes each per handshake cycle (clamped to 1 MiB, `PIPE_TRANSFER_BUFFER`).
257/// `do_work` and the think-sleep are skipped (`schbench.c:1448`), so the only
258/// per-cycle work is the wakeup handshake + the two memsets; the report is the
259/// PER-WORKER memory-transfer throughput (`avg worker transfer` = the aggregate
260/// rate divided by the worker count, `schbench.c:1697,1942-1943,1979`) alongside
261/// the wakeup-latency table, not request latency.
262///
263/// ktstr does NOT compose `-p` with `-R`: in pipe mode it always runs the
264/// message-handshake waker (so BOTH pipe memsets fire — a full transfer) and never
265/// starts the RPS injector. schbench instead COMPOSES them, half-broken: it has no
266/// precedence (`-R` alone picks the waker, `schbench.c:1594`), so `-p -R` runs the
267/// RPS injector while the worker-side memset still fires unconditionally
268/// (`schbench.c:1003-1004`) but the waker-side memset — which lives only in
269/// `xlist_wake_all` (`schbench.c:980-981`) — does not, yielding a degenerate
270/// half-pipe. ktstr's full pipe is the more faithful `-p` behavior; the realistic
271/// use is `-p` without `-R`. schbench also zeroes warmuptime in pipe mode
272/// (`schbench.c:296`); ktstr has no warmuptime concept, so that is a no-op here.
273///
274/// ## Modes not ported
275///
276/// - `-C` (calibrate): a tuning aid that times schbench's own work loop and
277///   forces `-L` (`schbench.c:166`, `:389`). Intentionally out of scope -- ktstr
278///   measures through the metric path.
279#[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
280#[serde(rename_all = "snake_case")]
281pub struct SchbenchConfig {
282    /// Number of message threads (`schbench.c` `-m`, default 1).
283    pub message_threads: usize,
284    /// Worker threads per message thread (`schbench.c` `-t`). 0 resolves to
285    /// `ceil(cpuset_cpus / message_threads)` -- the CPU count of the allocated
286    /// guest cpuset (the worker's `sched_getaffinity` mask, per ruling) divided
287    /// across the message threads, matching schbench's 0-default
288    /// (`schbench.c:1849-1852`) scoped to the cpuset rather than `get_nprocs`.
289    /// See `resolve_worker_count` and the CLI-parity section above.
290    pub worker_threads: usize,
291    /// Per-worker matrix cache footprint in KiB (`schbench.c` `-F`, default
292    /// 256); sets the matrix dimension.
293    pub cache_footprint_kib: usize,
294    /// Matrix multiplications per work cycle (`schbench.c` `-n`, default 5).
295    pub operations: usize,
296    /// Think-time sleep before the matrix work, microseconds (`schbench.c`
297    /// `-s`, default 100); simulates networking. 0 disables.
298    pub sleep_usec: u64,
299    /// Skip the per-CPU lock around the matrix work (`schbench.c` `-L`,
300    /// default false: locking on).
301    pub skip_locking: bool,
302    /// Fixed request rate, requests/second (`schbench.c` `-R`, default 0 = off).
303    /// 0 selects the default message-handshake mode (each worker is woken by its
304    /// message thread); non-zero switches to the RPS-injector mode, where a
305    /// dedicated thread enqueues `requests_per_sec` requests/second round-robin
306    /// across the workers (`schbench.c` `run_rps_thread`, `:1258`).
307    pub requests_per_sec: usize,
308    /// Auto-RPS target CPU-busy percentage (`schbench.c` `-A`, default 0 = off).
309    /// Non-zero turns on closed-loop rate control: a once-per-second control
310    /// thread grows/shrinks the live request rate toward this host-busy% target
311    /// (`schbench.c` `auto_scale_rps`, `:1180`). Setting it seeds the rate to 10
312    /// when `requests_per_sec` is 0 (`schbench.c:286`), so auto-RPS starts low
313    /// and climbs.
314    pub auto_rps: usize,
315    /// Percent of the cache footprint that is PRIVATE per worker thread
316    /// (`schbench.c` `--split`, long-only, 0-100). `None` = no split:
317    /// schbench's legacy all-private single matrix (`schbench.c:1405-1408`,
318    /// `:1879-1880`). `Some(p)` partitions `cache_footprint_kib` into a
319    /// per-thread private matrix (`p`%) and ONE process-global shared matrix
320    /// (`100-p`%) that every worker multiplies into concurrently, reproducing
321    /// schbench's cross-core shared-working-set cache contention
322    /// (`schbench.c:1390-1404`, `:1858-1863`). `Some(0)` = all shared,
323    /// `Some(100)` = all private (same matrix sizes as `None`, but routed
324    /// through the split branch, matching schbench's `split_specified` path).
325    /// Out-of-range `Some(p > 100)` panics when the engine consumes it
326    /// (`schbench.c:362-365` exits on the same); the builder also debug-asserts
327    /// the bound.
328    pub split_percent: Option<usize>,
329    /// Pipe-mode transfer size in bytes (`schbench.c` `-p`/`--pipe`, default 0 =
330    /// off, clamped to 1 MiB `PIPE_TRANSFER_BUFFER`). Non-zero REPLACES the
331    /// matrix workload with schbench's memory-transfer simulation: the message
332    /// thread memsets each woken worker's per-thread page to `1` and the worker
333    /// memsets its own page to `2` (`schbench.c:980-981`/`:1003-1004`),
334    /// `pipe_transfer_bytes` bytes each per cycle, while `do_work` + the
335    /// think-sleep are skipped (`schbench.c:1448`). Reports PER-WORKER
336    /// memory-transfer throughput (`avg worker transfer`) rather than request
337    /// latency.
338    pub pipe_transfer_bytes: usize,
339}
340
341/// schbench's `PIPE_TRANSFER_BUFFER` (`schbench.c:41`): the per-thread pipe page
342/// is 1 MiB; a larger `pipe_transfer_bytes` is clamped to it (schbench clamps and
343/// warns, `schbench.c:291-294`).
344pub(crate) const PIPE_TRANSFER_BUFFER: usize = 1024 * 1024;
345
346/// Cap on a single RPS-injector pacing sleep (ns). `run_rps_thread` runs
347/// synchronously on the message thread that `run` joins on shutdown, so an
348/// uncapped paced sleep would delay teardown by up to a full inter-arrival
349/// interval — at a degenerate low `requests_per_sec` (interval = 1s / rps) that
350/// can exceed the scenario cleanup/watchdog budget. Capping each sleep at this
351/// quantum re-checks `stop` at least this often, bounding shutdown latency. At
352/// realistic rates the interval is well under this, so a single uncapped wait
353/// results — the cap only engages at pathological low rates. (Schbench-local; the
354/// sibling taobench engine has its own equivalent, both with self-contained
355/// `monotonic_nanos`.)
356const STOP_POLL_QUANTUM_NS: u64 = 50_000_000;
357
358/// schbench's pipe-mode (`-p`) throughput summary — the `avg worker transfer`
359/// line (`schbench.c:1979-1982`): the per-worker transfer rate as ops/sec plus
360/// the pretty-scaled bytes/sec. Built by [`pipe_transfer_report`].
361#[derive(Debug, Clone, Copy, PartialEq)]
362pub struct PipeTransferReport {
363    /// Completed transfer cycles per second, PER WORKER (`loop_count / Σ worker
364    /// runtimes`, `schbench.c:1942-1943`).
365    pub ops_per_sec: f64,
366    /// Bytes transferred per second PER WORKER, scaled into [`unit`](Self::unit)
367    /// (÷1024 per step, `pretty_size`).
368    pub scaled: f64,
369    /// The unit `scaled` is expressed in: `B`/`KB`/`MB`/`GB`/`TB`/`PB`/`EB`
370    /// (`schbench.c:1606`).
371    pub unit: &'static str,
372}
373
374/// Derive the pipe-mode `avg worker transfer` line from a run's aggregate
375/// `achieved_rps` (completed cycles/sec over the true elapsed window), the
376/// requested `pipe_transfer_bytes`, and the resolved `nr_workers`. The figure is
377/// PER WORKER: schbench divides by `loop_runtime` = Σ each worker's runtime
378/// (`schbench.c:1697` sums `worker->runtime`; `:1942-1943`/`:1979` divide by it),
379/// and `Σ worker runtimes ≈ nr_workers * elapsed`, so the per-worker rate is the
380/// aggregate `achieved_rps / nr_workers` — the label is literally "avg WORKER
381/// transfer". (Dividing the aggregate by wall-clock alone would over-report by
382/// `nr_workers`×.) The transfer size is CLAMPED to `PIPE_TRANSFER_BUFFER` first —
383/// the engine moves only the clamped size per cycle (`run` applies the same
384/// `.min()`), matching schbench's parse-time clamp (`schbench.c:291-294`) — so the
385/// throughput reflects the bytes ACTUALLY moved. Scaling is schbench's
386/// `pretty_size` (`schbench.c:1606`). `nr_workers` is floored at 1 (no division by
387/// zero).
388pub fn pipe_transfer_report(
389    achieved_rps: f64,
390    pipe_transfer_bytes: usize,
391    nr_workers: usize,
392) -> PipeTransferReport {
393    let n = nr_workers.max(1) as f64;
394    let bytes = pipe_transfer_bytes.min(PIPE_TRANSFER_BUFFER);
395    // PER-WORKER: schbench's loops_per_sec/mb_per_sec divide the aggregate by
396    // Σ worker runtimes (≈ nr_workers * elapsed), i.e. `achieved_rps / nr_workers`.
397    let ops_per_sec = achieved_rps / n;
398    let bytes_per_sec = achieved_rps * bytes as f64 / n;
399    let (scaled, unit) = pretty_size(bytes_per_sec);
400    PipeTransferReport {
401        ops_per_sec,
402        scaled,
403        unit,
404    }
405}
406
407/// schbench's `pretty_size` (`schbench.c:1606-1620`): scale a byte count by 1024
408/// into B/KB/MB/GB/TB/PB/EB. Stops at the last unit (matches schbench's
409/// `units[divs + 1] == NULL` break — never overflows the table).
410fn pretty_size(mut number: f64) -> (f64, &'static str) {
411    const UNITS: [&str; 7] = ["B", "KB", "MB", "GB", "TB", "PB", "EB"];
412    let mut divs = 0;
413    while number >= 1024.0 && divs + 1 < UNITS.len() {
414        divs += 1;
415        number /= 1024.0;
416    }
417    (number, UNITS[divs])
418}
419
420impl Default for SchbenchConfig {
421    fn default() -> Self {
422        // schbench defaults (schbench.c option table + globals).
423        Self {
424            message_threads: 1,
425            worker_threads: 0,
426            cache_footprint_kib: 256,
427            operations: 5,
428            sleep_usec: 100,
429            skip_locking: false,
430            requests_per_sec: 0,
431            auto_rps: 0,
432            split_percent: None,
433            pipe_transfer_bytes: 0,
434        }
435    }
436}
437
438impl SchbenchConfig {
439    /// Set the number of message threads (schbench `-m`).
440    #[must_use = "builder methods consume self; bind the result"]
441    pub fn message_threads(mut self, n: usize) -> Self {
442        self.message_threads = n;
443        self
444    }
445    /// Set worker threads per message thread (schbench `-t`); 0 = one per
446    /// allocated CPU.
447    #[must_use = "builder methods consume self; bind the result"]
448    pub fn worker_threads(mut self, n: usize) -> Self {
449        self.worker_threads = n;
450        self
451    }
452    /// Set the per-worker matrix cache footprint in KiB (schbench `-F`).
453    #[must_use = "builder methods consume self; bind the result"]
454    pub fn cache_footprint_kib(mut self, kib: usize) -> Self {
455        self.cache_footprint_kib = kib;
456        self
457    }
458    /// Set the matrix multiplications per work cycle (schbench `-n`).
459    #[must_use = "builder methods consume self; bind the result"]
460    pub fn operations(mut self, n: usize) -> Self {
461        self.operations = n;
462        self
463    }
464    /// Set the think-time sleep in microseconds (schbench `-s`); 0 disables.
465    #[must_use = "builder methods consume self; bind the result"]
466    pub fn sleep_usec(mut self, usec: u64) -> Self {
467        self.sleep_usec = usec;
468        self
469    }
470    /// Skip the per-CPU lock around the matrix work (schbench `-L`).
471    #[must_use = "builder methods consume self; bind the result"]
472    pub fn skip_locking(mut self, skip: bool) -> Self {
473        self.skip_locking = skip;
474        self
475    }
476    /// Set the fixed request rate in requests/second (schbench `-R`); 0 selects
477    /// the default message-handshake mode, non-zero the RPS-injector mode.
478    #[must_use = "builder methods consume self; bind the result"]
479    pub fn requests_per_sec(mut self, rps: usize) -> Self {
480        self.requests_per_sec = rps;
481        self
482    }
483    /// Set the auto-RPS target host-busy percentage (schbench `-A`); 0 disables
484    /// auto-scaling. Non-zero seeds the rate to 10 when `requests_per_sec` is 0.
485    #[must_use = "builder methods consume self; bind the result"]
486    pub fn auto_rps(mut self, target_pct: usize) -> Self {
487        self.auto_rps = target_pct;
488        self
489    }
490    /// Set the private/shared cache-footprint split percentage (schbench
491    /// `--split`); `None` (default) = no split, all-private single matrix.
492    /// `Some(p)` requires `p <= 100`: the builder debug-asserts it for early
493    /// feedback, and the engine hard-panics on an out-of-range value at the
494    /// consumption boundary in `run` -- the analog of schbench exiting on a bad
495    /// `--split` (`schbench.c:362-365`).
496    #[must_use = "builder methods consume self; bind the result"]
497    pub fn split_percent(mut self, percent: Option<usize>) -> Self {
498        debug_assert!(
499            percent.is_none_or(|p| p <= 100),
500            "split_percent must be 0..=100"
501        );
502        self.split_percent = percent;
503        self
504    }
505    /// Set the pipe-mode transfer size in bytes (schbench `-p`/`--pipe`); 0
506    /// (default) = off (the matrix workload). Non-zero switches to schbench's
507    /// memory-transfer simulation; values above 1 MiB (`PIPE_TRANSFER_BUFFER`)
508    /// are clamped when the engine consumes it (schbench clamps the same,
509    /// `schbench.c:291-294`).
510    #[must_use = "builder methods consume self; bind the result"]
511    pub fn pipe_transfer_bytes(mut self, bytes: usize) -> Self {
512        self.pipe_transfer_bytes = bytes;
513        self
514    }
515
516    /// Matrix dimension from the cache footprint, identical to schbench
517    /// (`schbench.c:1880`: `sqrt(cache_footprint_kb * 1024 / 3 /
518    /// sizeof(unsigned long))`) and to ktstr's `FanOutCompute` precompute. Zero
519    /// `operations` or `cache_footprint_kib` yields a 0 dimension (no matrix
520    /// work).
521    pub(crate) fn matrix_size(&self) -> usize {
522        if self.operations > 0 && self.cache_footprint_kib > 0 {
523            ((self.cache_footprint_kib * 1024 / 3 / core::mem::size_of::<u64>()) as f64).sqrt()
524                as usize
525        } else {
526            0
527        }
528    }
529
530    /// Shared-matrix dimension for `--split` (`schbench.c:1859,1862`:
531    /// `sqrt(cache_footprint_kb*(100-split)/100 * 1024/3/sizeof(ulong))`).
532    /// `None` split (or 0 `operations`/`cache_footprint_kib`) => 0 (no shared
533    /// matrix); `Some(100)` => 0 (all private, no shared work).
534    pub(crate) fn shared_matrix_size(&self) -> usize {
535        match self.split_percent {
536            Some(p) if self.operations > 0 && self.cache_footprint_kib > 0 => {
537                let shared_kb = self.cache_footprint_kib * (100 - p) / 100;
538                ((shared_kb * 1024 / 3 / core::mem::size_of::<u64>()) as f64).sqrt() as usize
539            }
540            _ => 0,
541        }
542    }
543
544    /// Private (per-worker) matrix dimension. `Some(p)` (`schbench.c:1860,1863`):
545    /// `sqrt(cache_footprint_kb*split/100 * 1024/3/sizeof(ulong))`. `None`
546    /// (legacy): the full-footprint [`matrix_size`](Self::matrix_size)
547    /// (`schbench.c:1880`, the all-private single matrix).
548    pub(crate) fn private_matrix_size(&self) -> usize {
549        match self.split_percent {
550            Some(p) if self.operations > 0 && self.cache_footprint_kib > 0 => {
551                let private_kb = self.cache_footprint_kib * p / 100;
552                ((private_kb * 1024 / 3 / core::mem::size_of::<u64>()) as f64).sqrt() as usize
553            }
554            _ => self.matrix_size(),
555        }
556    }
557
558    /// The effective TOTAL request rate after schbench's startup seed: the user's
559    /// `requests_per_sec`, or 10 when auto-RPS is on and no fixed rate was given
560    /// (`schbench.c:286`: auto-RPS starts at 10 and climbs toward its target).
561    /// Both the mode gate ([`rps_per_message_thread`](Self::rps_per_message_thread))
562    /// and the live auto-scaled injection rate seed from this.
563    pub(crate) fn normalized_total_rps(&self) -> usize {
564        if self.auto_rps != 0 && self.requests_per_sec == 0 {
565            10
566        } else {
567            self.requests_per_sec
568        }
569    }
570
571    /// Per-message-thread request rate driving the RPS-vs-default MODE GATE: the
572    /// normalized total ([`normalized_total_rps`](Self::normalized_total_rps))
573    /// divided across the message threads, each of which runs its own RPS thread.
574    /// schbench divides once at startup (`requests_per_sec /= message_threads`,
575    /// `schbench.c:1899`) BEFORE the per-thread mode gate (`if (requests_per_sec)`,
576    /// `schbench.c:1594`), so the gate sees the divided value: a total below the
577    /// message-thread count rounds to 0 and the thread runs the DEFAULT
578    /// message-handshake mode (not a zero-rate RPS). The gate is decided ONCE from
579    /// this seeded value; the actual injection rate is the live auto-scaled total
580    /// (which may climb above the seed). Integer division drops the remainder,
581    /// matching schbench -- so auto-RPS with more than 10 message threads (seed
582    /// 10 / m = 0) degenerates to default mode, exactly as schbench does.
583    pub(crate) fn rps_per_message_thread(&self) -> usize {
584        self.normalized_total_rps() / self.message_threads.max(1)
585    }
586
587    /// Effective per-message-thread worker count for [`run`]. A non-zero
588    /// `worker_threads` is workers-per-message-thread as-is; the `0` default
589    /// mirrors schbench's `-t` 0-default (`schbench.c:1849-1852`:
590    /// `worker_threads = (num_cpus + message_threads - 1) / message_threads`)
591    /// but over the allocated guest cpuset (`allowed_count` from
592    /// [`resolve_cpu_topology`], per the ktstr cpuset ruling) instead of
593    /// `get_nprocs`: it divides the cpuset CPU count across the message threads
594    /// (ceil), so the TOTAL worker count (`message_threads * this`) stays ≈ the
595    /// cpuset CPU count -- matching schbench's total ≈ `num_cpus`.
596    /// `message_threads` floors at 1 to avoid a zero divisor; the divide is a
597    /// no-op at the default `message_threads = 1`.
598    pub(crate) fn resolve_worker_count(&self, allowed_count: usize) -> usize {
599        if self.worker_threads != 0 {
600            return self.worker_threads;
601        }
602        allowed_count.div_ceil(self.message_threads.max(1))
603    }
604}
605
606/// Per-thread state, the Rust counterpart of schbench's `struct thread_data`
607/// (`schbench.c:766`). Shared across threads by raw pointer for the lockless
608/// wait-list, so cross-thread access is restricted to the atomic fields
609/// (`next`, `futex`, `wake_time`); the histogram fields are owned solely by the
610/// worker thread the `ThreadData` belongs to.
611pub(crate) struct ThreadData {
612    /// Treiber-stack link for the message thread's wait-list
613    /// (schbench `thread_data->next`, `:776`). Null when not queued.
614    next: AtomicPtr<ThreadData>,
615    /// Wake handshake futex (`schbench.c` `thread_data->futex`, `:791`).
616    futex: super::handshake::Handshake,
617    /// Monotonic-ns timestamp the waker stamps just before posting, so the
618    /// woken thread can measure scheduler wakeup latency (schbench `wake_time`,
619    /// `:788`, stamped in `xlist_wake_all` `:984`).
620    wake_time: AtomicU64,
621    /// Wakeup-latency histogram (`schbench.c` `wakeup_stats`, `:794`).
622    /// Owner-thread-only; see the `Sync` impl SAFETY note.
623    wakeup_stats: UnsafeCell<PlatStats>,
624    /// Request (work-cycle) latency histogram (`schbench.c` `request_stats`,
625    /// `:795`). Owner-thread-only.
626    request_stats: UnsafeCell<PlatStats>,
627    /// Mean per-schedule run-queue wait (ns), read from `/proc/<tid>/schedstat`
628    /// at thread exit (schbench's `read_sched_delay`, `:1118`). Owner-only.
629    /// Feeds the WHOLE-RUN `SchbenchResult` (mean-of-means, matching real
630    /// schbench for the side-by-side validation); the per-phase delay comes from
631    /// [`Self::phase_snapshots`] instead.
632    sched_delay_ns: UnsafeCell<u64>,
633    /// Per-phase snapshots this thread accumulated via drain-on-change against
634    /// the shared `phase_epoch` (one [`PhaseSnapshot`] per phase the thread did
635    /// work in, plus a final in-flight phase at exit). Owner-thread-only,
636    /// drained by the main thread after join — same happens-before as the
637    /// histogram cells. Empty when run non-phasic (`phase_epoch == None`) until
638    /// the single end-of-run drain.
639    phase_snapshots: UnsafeCell<Vec<PhaseSnapshot>>,
640    /// RPS-injector mode: this worker's pending request queue. The RPS thread
641    /// pushes heap-allocated [`Request`]s (round-robin across workers); the
642    /// owning worker drains via `splice_reversed` (FIFO) and frees each. Unused
643    /// in the default message-handshake mode (`requests_per_sec == 0`).
644    requests: TreiberStack<Request>,
645    /// RPS-injector backpressure counter (`schbench.c` `thread_data->pending`,
646    /// `:779`): incremented by the RPS thread per enqueue, reset to 0 by the
647    /// owning worker at each splice. The RPS thread stops enqueuing to a worker
648    /// whose `pending` exceeds the batch cap (`:1284`).
649    pending: AtomicU64,
650    /// Pipe-mode (`-p`) transfer size in bytes (0 = not pipe mode); the length of
651    /// [`Self::pipe_page`] and the per-cycle memset. Plain `Copy`, set once at
652    /// construction, read-only thereafter.
653    pipe_bytes: usize,
654    /// Pipe-mode per-thread transfer page (`schbench.c` `thread_data->pipe_page`,
655    /// `:801`). The worker memsets it to `2` before blocking and the waker memsets
656    /// it to `1` when waking the worker; the bytes are never read back (the point
657    /// is the memory-transfer cost). `UnsafeCell` because the WAKER writes the
658    /// worker's page (cross-thread) -- see the `Sync` SAFETY note for the
659    /// wait-list/futex ordering that keeps it race-free. Empty unless pipe mode.
660    pipe_page: UnsafeCell<Box<[u8]>>,
661}
662
663/// One thread's latency + run-queue-delay accumulation over a single phase
664/// (the window between two `phase_epoch` transitions). Built on the owning
665/// thread at each drain-on-change boundary; the histograms are `take`n from the
666/// live cells (snapshot-and-reset) and the run-delay is a RAW
667/// `/proc/<tid>/schedstat` `(run_delay, pcount)` DELTA over the phase, so the
668/// host re-derives the per-phase mean as `Σrun_delay / Σpcount` (the
669/// sample-weighted pooled mean — a deliberate divergence from schbench's
670/// whole-run mean-of-means, correct for a heterogeneous per-phase thread set).
671/// Message threads leave `wakeup`/`request` empty and `loop_count` 0 (they
672/// carry only run-delay); workers fill all fields.
673struct PhaseSnapshot {
674    /// The `phase_epoch` value active while this snapshot's samples were
675    /// recorded (NOT the value at drain time). `0` = BASELINE, `u32::MAX` =
676    /// inter-step gap; both are emitted as-is and discarded host-side.
677    epoch: u32,
678    wakeup: PlatStats,
679    request: PlatStats,
680    /// `/proc/<tid>/schedstat` field 2 (run_delay ns) delta over this phase.
681    run_delay_ns: u64,
682    /// `/proc/<tid>/schedstat` field 3 (pcount = timeslices) delta over this
683    /// phase. The host guards `pcount == 0` → metric absent (never a div-by-zero
684    /// 0), matching [`mean_sched_delay`].
685    pcount: u64,
686    /// Completed work cycles this worker ran in this phase (0 for the message
687    /// thread).
688    loop_count: u64,
689    /// `CLOCK_THREAD_CPUTIME_ID` delta over this phase — the CPU time this
690    /// WORKER thread actually received (0 for the message thread: the
691    /// CPU-throughput denominator is Σ worker CPU, the CPU-clock analog of
692    /// schbench's Σ worker runtimes, `schbench.c:1697`). Feeds the
693    /// CPU-second-denominated `schbench_loops_per_cpu_sec` throughput.
694    cpu_ns: u64,
695}
696
697/// Per-phase, cross-thread aggregate for one `phase_epoch`: the wakeup + request
698/// histograms merged across every worker that ran in the phase, plus the
699/// run-delay raw pairs split by thread class (message vs worker). The per-phase
700/// wire carrier — rides inside [`crate::workload::PhaseSlice`] guest→host. All
701/// fields are integer, so the host re-pools across workers/cgroups by
702/// [`PlatStats::combine`] (histogram add) + integer sums; percentiles are
703/// re-derived from the merged histogram, NEVER averaged.
704///
705/// ESTIMATOR NOTE: the per-phase run-delay mean the host derives from
706/// `*_run_delay_ns / *_pcount` is SAMPLE-WEIGHTED (`Σrun_delay / Σpcount`), a
707/// DIFFERENT estimator from the whole-run [`SchbenchResult`]'s `sched_delay_*`,
708/// which keeps schbench's mean-of-per-thread-means (`collect_sched_delay`) for
709/// schbench parity. They measure different things by design — never cross-compare a
710/// per-phase value against a whole-run threshold (or vice-versa). `pcount == 0`
711/// for a class means that class was never scheduled in the phase → the host
712/// emits the metric as ABSENT, not `0`.
713#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
714pub(crate) struct SchbenchPhaseStats {
715    /// Wakeup-latency histogram merged across the phase's workers.
716    pub(crate) wakeup: PlatStats,
717    /// Request-latency histogram merged across the phase's workers.
718    pub(crate) request: PlatStats,
719    /// Per-phase achieved-RPS distribution: the control thread's per-second
720    /// samples attributed to this phase's epoch (the whole-run rps is unchanged;
721    /// this is purely additive). A 1s sample straddling an epoch boundary lands
722    /// wholly in the sample-time epoch (bounded fuzz, <=1 sample/transition, like
723    /// the msg-thread late-drain). Empty for a phase shorter than the ~1s control
724    /// cadence -- the host gates on `sample_count() > 0` so a sub-second phase
725    /// reads ABSENT, never rps=0. A rate; re-derived from the merged histogram.
726    ///
727    /// DENOMINATION: these per-second samples stay WALL-denominated (an
728    /// exception to the CPU-second throughput policy) — each sample is
729    /// schbench's own per-tick `rps_stats` reading (Δloops over a ~1s wall
730    /// window, upstream parity), and a per-tick CPU denominator would need
731    /// cross-thread cumulative-CPU reads from the control thread every
732    /// second, changing the sampling the upstream comparison validates. The
733    /// CPU-denominated throughput lives in `worker_cpu_ns` / the
734    /// `schbench_loops_per_cpu_sec` metric instead.
735    pub(crate) rps: PlatStats,
736    /// Σ message-thread `run_delay` (ns) over the phase.
737    pub(crate) msg_run_delay_ns: u64,
738    /// Σ message-thread `pcount` over the phase (the mean's denominator).
739    pub(crate) msg_pcount: u64,
740    /// Σ worker `run_delay` (ns) over the phase.
741    pub(crate) worker_run_delay_ns: u64,
742    /// Σ worker `pcount` over the phase (the mean's denominator).
743    pub(crate) worker_pcount: u64,
744    /// Σ completed work cycles across the phase's workers.
745    pub(crate) loop_count: u64,
746    /// Σ worker-thread `CLOCK_THREAD_CPUTIME_ID` deltas over the phase (ns) —
747    /// the CPU time the phase's workers actually received, the denominator of
748    /// the CPU-second throughput `schbench_loops_per_cpu_sec` (message-thread
749    /// CPU is excluded, mirroring schbench's Σ worker runtimes). CPU-clock
750    /// denominated so the throughput measures the workload, not host/guest
751    /// scheduling delay; the wall picture reconstructs as `cpu_rate / D` with
752    /// the sidecar's `host_dilation`. `#[serde(default)]`: pre-field sidecars
753    /// deserialize with 0 (the metric then reads absent).
754    #[serde(default)]
755    pub(crate) worker_cpu_ns: u64,
756}
757
758impl SchbenchPhaseStats {
759    /// Merge `other` into `self`: combine the wakeup + request histograms
760    /// (bucket-count addition, [`PlatStats::combine`]) and integer-add the
761    /// run-delay raw pairs + loop_count. Associative AND commutative (combine +
762    /// `saturating_add` both are), so pooling is order-independent — the SAME
763    /// operation whether pooling per-engine across message threads (in [`run`])
764    /// or per-cgroup across workers host-side
765    /// (`crate::assert::PhaseCgroupStats::merge`). Percentiles are NEVER
766    /// merged; the merged histogram is re-derived to percentiles by the reader.
767    pub(crate) fn merge(&mut self, other: &SchbenchPhaseStats) {
768        self.wakeup.combine(&other.wakeup);
769        self.request.combine(&other.request);
770        self.rps.combine(&other.rps);
771        self.msg_run_delay_ns = self.msg_run_delay_ns.saturating_add(other.msg_run_delay_ns);
772        self.msg_pcount = self.msg_pcount.saturating_add(other.msg_pcount);
773        self.worker_run_delay_ns = self
774            .worker_run_delay_ns
775            .saturating_add(other.worker_run_delay_ns);
776        self.worker_pcount = self.worker_pcount.saturating_add(other.worker_pcount);
777        self.loop_count = self.loop_count.saturating_add(other.loop_count);
778        self.worker_cpu_ns = self.worker_cpu_ns.saturating_add(other.worker_cpu_ns);
779    }
780}
781
782impl Linked for ThreadData {
783    fn next_link(&self) -> &AtomicPtr<Self> {
784        &self.next
785    }
786}
787
788// SAFETY: `ThreadData` is shared across threads only via the lockless wait-list
789// and (in RPS mode) the per-worker request queue, whose operations touch
790// exclusively the atomic fields (`next`, `futex`, `wake_time`, `requests`,
791// `pending`) -- all internally synchronized. `requests` is single-producer /
792// single-consumer: the RPS thread is the only pusher to a given worker's queue
793// (round-robin assignment) and the owning worker is the only drainer, with the
794// push's Release CAS / drain's Acquire swap establishing happens-before;
795// `pending` is plain atomic counting. The `UnsafeCell` fields (`wakeup_stats`,
796// `request_stats`, `sched_delay_ns`, `phase_snapshots`) are written and read
797// ONLY by the single worker thread that owns this `ThreadData`, and only the
798// main thread reads/drains them after all workers have joined (a happens-before
799// via the join). The per-phase drain (`worker_loop` / `run_msg_thread`) `take`s
800// and pushes into the owning thread's own cells, never another's. The `pipe_page`
801// cell (pipe mode) is the one cross-thread `UnsafeCell`: the owning worker memsets
802// it (to 2) and the message thread memsets it (to 1). They never overlap, via two
803// happens-before edges:
804//   - SAME cycle (worker fill-2 vs waker fill-1): the worker's fill-2 is SEQUENCED
805//     BEFORE its `wait_list.add` (Release CAS), and the waker's `splice` (Acquire
806//     swap) observes that add, giving fill-2 -> add -> splice -> waker fill-1. The
807//     load-bearing fact is "fill-2 precedes add", which holds UNCONDITIONALLY --
808//     including the no-block shutdown path (stop set, worker skips the futex wait):
809//     the worker still added itself after fill-2, so a waker that then splices and
810//     fills it is ordered after the fill-2, and the worker's exit path never
811//     touches the page again. Blocking is NOT what makes it safe.
812//   - NEXT cycle (worker's fill-2 vs the waker's prior fill-1): the futex
813//     post/consume CAS (SeqCst, `Handshake`) orders waker fill-1 -> post -> the
814//     worker's consume -> its next fill-2; the worker cannot reach its next fill-2
815//     until it consumes the token the waker published after fill-1.
816// So the two threads never touch `pipe_page` concurrently. No two threads ever
817// touch a cell concurrently, so sharing `&ThreadData` across threads is sound.
818unsafe impl Sync for ThreadData {}
819
820impl ThreadData {
821    fn new(pipe_bytes: usize) -> Self {
822        Self {
823            next: AtomicPtr::new(ptr::null_mut()),
824            futex: super::handshake::Handshake::new(),
825            wake_time: AtomicU64::new(0),
826            wakeup_stats: UnsafeCell::new(PlatStats::default()),
827            request_stats: UnsafeCell::new(PlatStats::default()),
828            sched_delay_ns: UnsafeCell::new(0),
829            phase_snapshots: UnsafeCell::new(Vec::new()),
830            requests: TreiberStack::new(),
831            pending: AtomicU64::new(0),
832            pipe_bytes,
833            pipe_page: UnsafeCell::new(vec![0u8; pipe_bytes].into_boxed_slice()),
834        }
835    }
836
837    /// Pipe-mode transfer: fill this thread's `pipe_page` (`pipe_bytes` bytes) with
838    /// `val`. The page is never read back, so the fill would be DCE-eligible. The
839    /// trailing `black_box(page.as_ptr())` ESCAPES THE ADDRESS (not a loaded value):
840    /// the optimizer must then assume the opaque call may read through the pointer,
841    /// so every byte of the fill must be in place at that point -- the fill is
842    /// retained. This is stronger than `black_box(value)`, which only keeps a value
843    /// live and (per the optimization-resistance note in `worker::worker_main`) does
844    /// NOT by itself stop a backing STORE from being elided -- which is why the
845    /// cache-pressure workloads there use `write_volatile`. Here the faithful
846    /// schbench op is a memset (`schbench.c:980-981`/`:1003-1004`) and `slice::fill`
847    /// lowers to one; a `write_volatile` byte loop would not. The memory traffic IS
848    /// the workload.
849    ///
850    /// SAFETY: the caller guarantees no concurrent access to `pipe_page`. The only
851    /// callers are the owning worker (before it adds itself to the wait-list) and
852    /// the message thread (after it splices the worker off the wait-list); the
853    /// load-bearing edge is fill-2-before-add (holding even on the no-block
854    /// shutdown path), so the add/splice + futex handshake order these with no
855    /// overlap (see the `Sync` SAFETY note).
856    unsafe fn pipe_fill(&self, val: u8) {
857        // SAFETY: exclusive access per the caller contract; the box lives for the
858        // `ThreadData`'s lifetime.
859        let page = unsafe { &mut *self.pipe_page.get() };
860        page.fill(val);
861        std::hint::black_box(page.as_ptr());
862    }
863}
864
865/// schbench's per-request think-time sleep ("simulated networking",
866/// `schbench.c:1461`). This is the workload's defined behavior, not a
867/// synchronization wait: `usleep` maps to `clock_nanosleep`, matching
868/// schbench's syscall profile.
869fn think_sleep(usec: u64) {
870    std::thread::sleep(std::time::Duration::from_micros(usec));
871}
872
873/// schbench's `--split` op partition (`schbench.c:1391-1392`): of `operations`
874/// total, `p`% run on the private matrix and the rest on the shared matrix.
875/// Returns `(ops_shared, ops_private)`. Extracted so the integer-division split
876/// is pinned by a unit test rather than re-implemented inline in `do_work`.
877fn ops_split(operations: usize, p: usize) -> (usize, usize) {
878    let ops_private = (operations * p) / 100;
879    let ops_shared = operations - ops_private;
880    (ops_shared, ops_private)
881}
882
883/// One work cycle's matrix multiplications under the per-CPU lock. Faithful to
884/// schbench's `do_work` (`schbench.c:1379-1413`): take the current CPU's mutex
885/// (unless `skip_locking`), run `operations` matrix multiplies, unlock. With
886/// `split_percent` set, the ops are partitioned (`schbench.c:1390-1404`):
887/// `ops_shared` run on the ONE process-global shared matrix (every worker
888/// contends), then `ops_private` on this worker's private buffer; `None` is the
889/// legacy all-private single matrix (`:1406-1408`). The guard drops at the end,
890/// holding the lock across all operations exactly as schbench does (`:1387`/
891/// `:1411`).
892#[allow(clippy::too_many_arguments)]
893fn do_work(
894    private_buf: &mut [u64],
895    private_matrix_size: usize,
896    shared_buf: &[core::sync::atomic::AtomicU64],
897    shared_matrix_size: usize,
898    split_percent: Option<usize>,
899    operations: usize,
900    locks: Option<&PerCpuLocks>,
901    work_units: &mut u64,
902) {
903    let _guard = locks.map(|l| l.lock_this_cpu());
904    match split_percent {
905        Some(p) => {
906            let (ops_shared, ops_private) = ops_split(operations, p);
907            // schbench.c:1395-1398: shared ops first, on the global matrix.
908            if shared_matrix_size > 0 && ops_shared > 0 {
909                for _ in 0..ops_shared {
910                    crate::workload::worker::matrix_multiply_shared(
911                        shared_buf,
912                        shared_matrix_size,
913                        work_units,
914                    );
915                }
916            }
917            // schbench.c:1401-1404: private ops second, on this worker's buffer.
918            if private_matrix_size > 0 && ops_private > 0 {
919                for _ in 0..ops_private {
920                    crate::workload::worker::matrix_multiply(
921                        private_buf,
922                        private_matrix_size,
923                        work_units,
924                    );
925                }
926            }
927        }
928        None => {
929            // schbench.c:1406-1408: legacy all-private single matrix.
930            for _ in 0..operations {
931                if private_matrix_size > 0 {
932                    crate::workload::worker::matrix_multiply(
933                        private_buf,
934                        private_matrix_size,
935                        work_units,
936                    );
937                }
938            }
939        }
940    }
941}
942
943/// Worker side of one wakeup cycle. Faithful to schbench's `msg_and_wait`
944/// (`schbench.c:997`, default branch): stamp our `wake_time`, push ourselves
945/// onto the message thread's wait-list, wake the message thread, then block
946/// until it wakes us back; record the wakeup (scheduler) latency. The `!stop`
947/// guard mirrors schbench's `if (!stopping)` (`schbench.c:1030`) so we do not
948/// block during shutdown.
949fn msg_and_wait(
950    td: &ThreadData,
951    msg_td: &ThreadData,
952    wait_list: &TreiberStack<ThreadData>,
953    stop: &AtomicBool,
954) {
955    // Our futex is BLOCKED here (consumed by the prior wait, or fresh).
956    // Pipe mode: write our transfer page to 2 (schbench memsets td->pipe_page
957    // before blocking, schbench.c:1003-1004). Done before we enqueue on the
958    // wait-list, so the waker's fill-to-1 (after it splices us off) never races --
959    // see the Sync SAFETY note.
960    if td.pipe_bytes > 0 {
961        // SAFETY: we own this page and have not yet enqueued, so the waker cannot
962        // touch it concurrently.
963        unsafe { td.pipe_fill(2) };
964    }
965    td.wake_time.store(monotonic_nanos(), Ordering::Release);
966    wait_list.add(td as *const ThreadData as *mut ThreadData);
967    msg_td.futex.post();
968    if !stop.load(Ordering::Acquire) {
969        td.futex.wait_forever();
970    }
971    let now = monotonic_nanos();
972    let wake = td.wake_time.load(Ordering::Acquire);
973    // schbench buckets in microseconds (gettimeofday resolution); the monotonic
974    // clock gives ns, so divide. `if delta > 0` matches schbench (`:1036`).
975    let delta_us = now.saturating_sub(wake) / 1000;
976    if delta_us > 0 {
977        // SAFETY: only this worker thread accesses its own wakeup_stats cell.
978        unsafe { (*td.wakeup_stats.get()).add_lat(delta_us.min(u32::MAX as u64) as u32) };
979    }
980}
981
982/// Worker side of one RPS-injector cycle. Faithful to schbench's `msg_and_wait`
983/// RPS branch (`schbench.c:1007-1037`): stamp our own `wake_time`, reset
984/// `pending`, then splice this worker's request queue (FIFO). If non-empty,
985/// return the chain immediately (fast path, no wait, no wakeup sample --
986/// `:1014-1017`). If empty, block on this worker's own futex until the RPS thread
987/// posts it, record the wakeup (scheduler) latency, then splice whatever it
988/// queued (null on a spurious wake or the shutdown wake-all). Unlike
989/// [`msg_and_wait`], the worker does NOT enqueue on the message thread's
990/// wait-list or post a message thread -- the RPS thread is the sole waker.
991///
992/// `wake_time` is stamped HERE before the splice (schbench `:1008`) AND by the
993/// RPS thread at enqueue (`:1294`). On a TRUE block the RPS thread's stamp wins
994/// (it overwrites ours before posting), so `now - wake_time` is the enqueue->run
995/// scheduler latency. But on a FAST wake -- a leftover RUNNING futex token from a
996/// request the worker already drained via splice (the word is consumed by splice,
997/// not by `wait`, so it stays RUNNING) -- the RPS thread does NOT re-stamp, so our
998/// own stamp makes the delta measure THIS block decision (near-zero), not the
999/// stale enqueue time of the already-served request. Omitting it inflates the
1000/// fast-wake samples to the full inter-request gap (the wakeup-p50 bug: 543ms
1001/// vs schbench's 7µs).
1002fn rps_wait(td: &ThreadData, stop: &AtomicBool) -> *mut Request {
1003    // Stamp our own wake_time before splicing (schbench `:1008`); see the doc for
1004    // why this is what keeps fast-wake samples honest.
1005    td.wake_time.store(monotonic_nanos(), Ordering::Release);
1006    // Reset the backpressure counter before draining (schbench `td->pending = 0`,
1007    // `:1012`): the RPS thread stops enqueuing while pending exceeds the batch
1008    // cap, so clearing it re-opens this worker for the next batch.
1009    td.pending.store(0, Ordering::Release);
1010    let chain = td.requests.splice_reversed();
1011    if !chain.is_null() {
1012        return chain; // work already queued: serve it without blocking
1013    }
1014    if stop.load(Ordering::Acquire) {
1015        return ptr::null_mut();
1016    }
1017    // Empty queue: block until the RPS thread posts our futex, then `now -
1018    // wake_time` is the scheduler wakeup latency (schbench `fwait` NULL then
1019    // add_lat, `:1032`/`:1034-1037`).
1020    td.futex.wait_forever();
1021    // Shutdown wake: if stop is set, the wake came from run_rps_thread's stop-path
1022    // wake-all (`:1308-1312`), which posts every worker WITHOUT restamping
1023    // wake_time. A worker blocked between bursts when stop fires would otherwise
1024    // record `now - (its own stale block-time stamp)` -- the whole inter-burst
1025    // gap -- as a phantom wakeup, one garbage tail sample per worker. That is not
1026    // a request-driven wakeup, so skip the sample (schbench's `if (!stopping)
1027    // fwait` guard, `:1030`, encodes the same intent: shutdown is not a latency
1028    // event). Return any leftover requests for the caller to free.
1029    if stop.load(Ordering::Acquire) {
1030        return td.requests.splice_reversed();
1031    }
1032    let now = monotonic_nanos();
1033    let wake = td.wake_time.load(Ordering::Acquire);
1034    let delta_us = now.saturating_sub(wake) / 1000;
1035    if delta_us > 0 {
1036        // SAFETY: only this worker thread accesses its own wakeup_stats cell.
1037        unsafe { (*td.wakeup_stats.get()).add_lat(delta_us.min(u32::MAX as u64) as u32) };
1038    }
1039    // Splice whatever the RPS thread enqueued before posting us (null if the post
1040    // was a spurious wake with nothing queued).
1041    td.requests.splice_reversed()
1042}
1043
1044/// Free a chain of [`Request`]s spliced off a worker's queue but not processed
1045/// (the shutdown path, and the post-join cleanup of any requests the RPS thread
1046/// enqueued after a worker's last splice). Each node was `Box::into_raw`'d by the
1047/// RPS thread; the draining/cleaning thread owns them exclusively after the
1048/// splice, so each is freed exactly once.
1049fn free_request_chain(mut req: *mut Request) {
1050    while !req.is_null() {
1051        // SAFETY: `req` is a node taken via `splice_reversed` (exclusive
1052        // ownership); freed exactly once.
1053        let next = unsafe { (*req).next.load(Ordering::Acquire) };
1054        drop(unsafe { Box::from_raw(req) });
1055        req = next;
1056    }
1057}
1058
1059/// Wake every worker on the wait-list, stamping each one's `wake_time` so it
1060/// can measure scheduler latency. Faithful to schbench's `xlist_wake_all`
1061/// (`schbench.c:969`): splice the whole list, read the clock ONCE, stamp every
1062/// worker with that single time, then post each. The single clock read is what
1063/// detects the scheduler preempting the waker mid-batch (`schbench.c:961-964`).
1064/// Pipe mode (`-p`) diverges, matching schbench (`schbench.c:980-984`): it memsets
1065/// each worker's transfer page to `1` and re-reads the clock PER worker (so the
1066/// memset cost is not charged to that worker's wakeup latency).
1067fn wake_all(wait_list: &TreiberStack<ThreadData>) {
1068    let mut cur = wait_list.splice();
1069    let now = monotonic_nanos();
1070    while !cur.is_null() {
1071        // SAFETY: `cur` is a worker `ThreadData` alive for the whole run; only
1072        // its atomic fields (next / wake_time / futex) are touched here.
1073        let td = unsafe { &*cur };
1074        let next = td.next.load(Ordering::Acquire);
1075        td.next.store(ptr::null_mut(), Ordering::Relaxed);
1076        if td.pipe_bytes > 0 {
1077            // Pipe mode: write this woken worker's transfer page to 1 (schbench
1078            // memsets list->pipe_page in the wake loop, schbench.c:980-981). We
1079            // spliced it off the wait-list and it is still blocked (our `post`
1080            // below wakes it), so no concurrent access -- see the Sync SAFETY note.
1081            // SAFETY: `td` is off the wait-list and blocked here.
1082            unsafe { td.pipe_fill(1) };
1083            // schbench re-reads the clock per worker in pipe mode (after the
1084            // memset, schbench.c:982) so the fill is not charged to the worker's
1085            // wakeup latency; non-pipe uses the single batched `now`.
1086            td.wake_time.store(monotonic_nanos(), Ordering::Release);
1087        } else {
1088            td.wake_time.store(now, Ordering::Release);
1089        }
1090        td.futex.post();
1091        cur = next;
1092    }
1093}
1094
1095/// The message thread's loop. Faithful to schbench's `run_msg_thread`
1096/// (`schbench.c:1166`): batch-wake all waiting workers, then block until a
1097/// worker posts us back. On stop, drain once more (to wake any worker that
1098/// queued during the final wake) and exit.
1099///
1100/// `phase_epoch` drives the per-phase drain exactly as in [`worker_loop`], but
1101/// the message thread records only per-phase RUN-DELAY (its `wakeup`/`request`
1102/// histograms stay empty, taken empty by [`drain_phase`]). It polls the epoch
1103/// after each wake cycle; a phase in which no worker ever posts it leaves that
1104/// phase's run-delay drained late at the next wake (bounded boundary fuzz,
1105/// matching the worker drain).
1106fn run_msg_thread(
1107    msg_td: &ThreadData,
1108    wait_list: &TreiberStack<ThreadData>,
1109    stop: &AtomicBool,
1110    phase_epoch: Option<&AtomicU32>,
1111) {
1112    let tid = gettid_self();
1113    let mut cur_epoch = phase_epoch.map_or(0, |e| e.load(Ordering::Relaxed));
1114    let mut phase_ss_start = read_schedstat_raw(tid);
1115    loop {
1116        // msg_td.futex is BLOCKED here (consumed by the prior wait, or fresh).
1117        wake_all(wait_list);
1118        if stop.load(Ordering::Acquire) {
1119            wake_all(wait_list);
1120            break;
1121        }
1122        msg_td.futex.wait_forever();
1123        if let Some(pe) = phase_epoch {
1124            let new_epoch = pe.load(Ordering::Relaxed);
1125            if new_epoch != cur_epoch {
1126                let ss_end = read_schedstat_raw(tid);
1127                // SAFETY: this is the thread that owns msg_td (the coordinator
1128                // thread of run_one_message_thread).
1129                unsafe { drain_phase(msg_td, cur_epoch, phase_ss_start, ss_end, 0, 0) };
1130                cur_epoch = new_epoch;
1131                phase_ss_start = ss_end;
1132            }
1133        }
1134    }
1135    // Final drain: close the still-open phase (the sole snapshot when non-phasic).
1136    let ss_end = read_schedstat_raw(tid);
1137    // SAFETY: this is the thread that owns msg_td.
1138    unsafe { drain_phase(msg_td, cur_epoch, phase_ss_start, ss_end, 0, 0) };
1139    // Record the message thread's whole-run mean run-queue wait (`schbench.c:1664`),
1140    // reusing the cumulative `ss_end` pair just read — one /proc read, so the mean
1141    // and the final-phase delta derive from one consistent snapshot.
1142    // SAFETY: owner-only access to msg_td's sched_delay_ns cell.
1143    unsafe { *msg_td.sched_delay_ns.get() = mean_sched_delay(ss_end) };
1144}
1145
1146/// The RPS-injector dispatcher loop, replacing [`run_msg_thread`] when
1147/// `requests_per_sec != 0`. Each second it enqueues `requests_per_sec` requests
1148/// round-robin across this group's `workers`, skipping a worker whose `pending`
1149/// exceeds the batch cap (backpressure, `schbench.c:1284-1290`); on stop, it
1150/// wakes every worker and exits. Runs on the message thread's calling thread, so
1151/// — like [`run_msg_thread`] — it records the dispatcher thread's per-phase +
1152/// whole-run run-queue wait into `msg_td` (the "message thread delay" in RPS
1153/// mode, `schbench.c:1664-1670`).
1154///
1155/// DIVERGENCE from schbench's `run_rps_thread` (`schbench.c:1258-1314`),
1156/// intentional: schbench FRONT-LOADS the second (bursts all N enqueues, then
1157/// `usleep`s to fill the rest of the second) and relies on its workers
1158/// interleaving with the burst to keep each queue under the cap. That interleave
1159/// is scheduling-dependent; ktstr's burst instead front-ran the workers (the
1160/// first splice grabbed a large batch the worker drained to completion before
1161/// re-splicing), so `pending` stayed pegged and the injector dropped the
1162/// remainder every second -- a ~BATCH/worker supply cap that under-delivered the
1163/// requested rate (e.g. ~257/s at a 400/s target). ktstr instead PACES the
1164/// enqueues evenly across the second (one every `1s/N`), which forces the
1165/// injector to interleave with the workers deterministically: `pending` drains
1166/// between enqueues and the full rate is delivered up to worker capacity. Above
1167/// capacity the pacing collapses to a flat-out loop and the `BATCH` backpressure
1168/// bounds the queue, matching schbench's bounded-backlog overload behavior. The
1169/// pacing sleep and the bounded backpressure usleep are workload-defined `-R`
1170/// cadence (like the per-request think-sleep), not synchronization waits. Uses
1171/// `CLOCK_MONOTONIC` (ruling), not schbench's `gettimeofday`.
1172///
1173/// Verified by `ktstr-schbench-validate` (release, standalone), not a nextest
1174/// unit test: the under-delivery signal only appears when worker capacity exceeds
1175/// the front-load cap (~2*BATCH/s), which requires a release build (debug matrix
1176/// work caps capacity well below that) on dedicated CPU (nextest's parallel
1177/// execution starves the workers, collapsing both the paced and a hypothetical
1178/// reverted injector to the same capacity-limited rate). The fixed `-R` axis in
1179/// the schbench validation comparison is the regression evidence.
1180fn run_rps_thread(
1181    workers: &[ThreadData],
1182    msg_td: &ThreadData,
1183    stop: &AtomicBool,
1184    phase_epoch: Option<&AtomicU32>,
1185    live_rate: &AtomicUsize,
1186) {
1187    let tid = gettid_self();
1188    let mut cur_epoch = phase_epoch.map_or(0, |e| e.load(Ordering::Relaxed));
1189    let mut phase_ss_start = read_schedstat_raw(tid);
1190    // schbench's per-worker queue-depth cap before the injector backs off
1191    // (`schbench.c:1267`).
1192    const BATCH: u64 = 128;
1193    const ONE_SEC_NS: u64 = 1_000_000_000;
1194    let worker_count = workers.len().max(1);
1195    let mut cur_tid: usize = 0;
1196    while !stop.load(Ordering::Acquire) {
1197        // Re-read the live per-message-thread rate each second. `live_rate` is
1198        // already the per-thread value (schbench's post-`/= message_threads`
1199        // global `requests_per_sec`, `:1899`), injected directly with no further
1200        // division (`schbench.c:1290`). Auto-RPS retargets it from the control
1201        // thread; a fixed `-R` leaves it constant.
1202        let requests_per_sec = live_rate.load(Ordering::Relaxed);
1203        let start = monotonic_nanos();
1204        // PACE the enqueues evenly across the second rather than front-loading
1205        // them. schbench bursts all N then fills the rest of the second
1206        // (`schbench.c:1300-1306`), relying on its workers interleaving with the
1207        // burst to keep each worker's queue under the `BATCH` cap. ktstr's burst
1208        // front-ran the workers: the first splice grabbed a large batch the worker
1209        // processed to completion before re-splicing, so `pending` stayed pegged
1210        // and the injector dropped the remainder every second (a steady-state
1211        // ~BATCH/worker supply cap -> only ~257/s at a 400/s target). Pacing one
1212        // enqueue every `1s/N` forces the injector to interleave with the workers,
1213        // so `pending` drains between enqueues and the full rate is delivered up
1214        // to worker capacity. Above capacity `due` falls behind real time, the
1215        // pacing sleep collapses to zero, and the `BATCH` backpressure bounds the
1216        // queue -- matching schbench's overload behavior (a bounded backlog).
1217        let interval_ns = if requests_per_sec > 0 {
1218            ONE_SEC_NS / requests_per_sec as u64
1219        } else {
1220            ONE_SEC_NS
1221        };
1222        let mut due = start;
1223        for _ in 0..requests_per_sec {
1224            if stop.load(Ordering::Acquire) {
1225                break;
1226            }
1227            // Wait until this enqueue's scheduled time. A no-op once `due` falls
1228            // behind real time (rate above what the workers can drain), so the
1229            // loop then runs flat-out and the backpressure below bounds the queue.
1230            // `due` is re-based to `start` each second and accrues at most
1231            // N*(ONE_SEC_NS/N) ~= 1s within a pass, so saturating_add never wraps
1232            // (the saturating form documents that the per-pass re-base is the
1233            // load-bearing invariant).
1234            due = due.saturating_add(interval_ns);
1235            // Pace to this enqueue's scheduled time, but cap each sleep at
1236            // STOP_POLL_QUANTUM_NS and re-check `stop`, so a shutdown during a long
1237            // inter-arrival gap (a degenerate low rate, where the interval can
1238            // exceed the cleanup/watchdog budget) is observed within the quantum
1239            // instead of after a full interval — mirroring the taobench open-loop
1240            // pacing. A stop observed here breaks the enqueue loop BEFORE issuing
1241            // this slot's request.
1242            let mut stop_during_pace = false;
1243            loop {
1244                if stop.load(Ordering::Acquire) {
1245                    stop_during_pace = true;
1246                    break;
1247                }
1248                let now = monotonic_nanos();
1249                if now >= due {
1250                    break;
1251                }
1252                let nap = (due - now).min(STOP_POLL_QUANTUM_NS);
1253                std::thread::sleep(std::time::Duration::from_nanos(nap));
1254            }
1255            if stop_during_pace {
1256                break;
1257            }
1258            let worker = &workers[cur_tid % worker_count];
1259            cur_tid += 1;
1260            // Backpressure: don't queue more to a worker already `BATCH` deep
1261            // (`schbench.c:1284-1290`). The bounded usleep is schbench's injector
1262            // throttle (workload behavior), not a sync poll. The round-robin
1263            // cursor already advanced, so the skipped slot moves to the next
1264            // worker, matching schbench (the request for this slot is dropped).
1265            if worker.pending.load(Ordering::Acquire) > BATCH {
1266                std::thread::sleep(std::time::Duration::from_micros(100));
1267                continue;
1268            }
1269            worker.pending.fetch_add(1, Ordering::AcqRel);
1270            // schbench mallocs one request per enqueue; the worker frees it after
1271            // processing (`schbench.c:1292`/`:1476`). Box matches that profile.
1272            let req = Box::into_raw(Box::new(Request::new()));
1273            worker.requests.add(req);
1274            // Stamp the target's wake_time so it measures wakeup latency as
1275            // now - enqueue (`schbench.c:1294`), then post it.
1276            worker.wake_time.store(monotonic_nanos(), Ordering::Release);
1277            worker.futex.post();
1278        }
1279        // The pacing loop already spans ~1 second; top up only if it finished
1280        // early (rate 0, or `stop`), preserving schbench's per-second cadence. Cap
1281        // each sleep at STOP_POLL_QUANTUM_NS and re-check `stop` so a shutdown
1282        // during the top-up is observed within the quantum, not after a full
1283        // remaining second (the same teardown bound as the pacing sleep above).
1284        loop {
1285            if stop.load(Ordering::Acquire) {
1286                break;
1287            }
1288            let elapsed = monotonic_nanos().saturating_sub(start);
1289            if elapsed >= ONE_SEC_NS {
1290                break;
1291            }
1292            let nap = (ONE_SEC_NS - elapsed).min(STOP_POLL_QUANTUM_NS);
1293            std::thread::sleep(std::time::Duration::from_nanos(nap));
1294        }
1295        // Dispatcher per-phase run-delay drain (mirrors run_msg_thread): on epoch
1296        // change, finalize this thread's run-delay for the ended phase. Empty
1297        // histograms (the dispatcher records only run-delay).
1298        if let Some(pe) = phase_epoch {
1299            let new_epoch = pe.load(Ordering::Relaxed);
1300            if new_epoch != cur_epoch {
1301                let ss_end = read_schedstat_raw(tid);
1302                // SAFETY: this thread owns msg_td (the dispatcher's ThreadData).
1303                unsafe { drain_phase(msg_td, cur_epoch, phase_ss_start, ss_end, 0, 0) };
1304                cur_epoch = new_epoch;
1305                phase_ss_start = ss_end;
1306            }
1307        }
1308        if stop.load(Ordering::Acquire) {
1309            // Wake every worker so a blocked one observes stop and exits
1310            // (`schbench.c:1308-1312`).
1311            for w in workers {
1312                w.futex.post();
1313            }
1314            break;
1315        }
1316    }
1317    // Final drain + whole-run mean run-delay for the dispatcher thread, matching
1318    // run_msg_thread's exit (`schbench.c:1664`).
1319    let ss_end = read_schedstat_raw(tid);
1320    // SAFETY: this thread owns msg_td.
1321    unsafe { drain_phase(msg_td, cur_epoch, phase_ss_start, ss_end, 0, 0) };
1322    // SAFETY: owner-only access to msg_td's sched_delay_ns cell.
1323    unsafe { *msg_td.sched_delay_ns.get() = mean_sched_delay(ss_end) };
1324}
1325
1326/// The calling thread's kernel tid (`gettid`), for reading its own schedstat.
1327fn gettid_self() -> libc::pid_t {
1328    // SAFETY: gettid takes no arguments and only returns the caller's tid.
1329    unsafe { libc::syscall(libc::SYS_gettid) as libc::pid_t }
1330}
1331
1332/// Raw `(run_delay_ns, pcount)` pair from `/proc/<tid>/schedstat` (fields 2, 3),
1333/// UNDIVIDED — schbench's `read_sched_delay` source (`schbench.c:1118`) without
1334/// the division. Both consumers want the raw pair, not a pre-divided mean: the
1335/// per-phase path re-pools `Σrun_delay / Σpcount` host-side, and the whole-run
1336/// per-thread mean is [`mean_sched_delay`] of the thread's final cumulative
1337/// pair. An absent file (the thread exited) yields `(0, 0)`, like schbench's
1338/// fopen-failure path.
1339///
1340/// `CONFIG_SCHED_INFO` (selected by `CONFIG_SCHEDSTATS`, and also by
1341/// `CONFIG_TASK_DELAY_ACCT` -- ktstr.kconfig enables both) populates these
1342/// fields even under sched_ext and regardless of the `kernel.sched_schedstats`
1343/// sysctl (that sysctl gates the separate per-rq/domain counters, not
1344/// `sched_info`).
1345fn read_schedstat_raw(tid: libc::pid_t) -> (u64, u64) {
1346    match std::fs::read_to_string(format!("/proc/{tid}/schedstat")) {
1347        Ok(s) => parse_schedstat_raw(&s),
1348        // The thread may have exited; schbench's fopen-failure path also -> 0.
1349        Err(_) => (0, 0),
1350    }
1351}
1352
1353/// Parse a `/proc/<tid>/schedstat` line into the raw `(run_delay_ns, pcount)`
1354/// pair (field 2 = run_delay ns, field 3 = pcount = timeslices). The kernel's
1355/// `proc_pid_schedstat` always emits three integer fields, so a malformed
1356/// *present* line is a kernel/parse bug: panic rather than silently report 0
1357/// (matching the handshake's fail-loud stance + the no-silent-wrong-answer
1358/// rule). An absent file is the caller's concern, handled as `(0, 0)`.
1359fn parse_schedstat_raw(s: &str) -> (u64, u64) {
1360    let mut fields = s.split_whitespace();
1361    let _run = fields.next(); // field 1: sum_exec_runtime (on-CPU ns), unused
1362    let run_delay: u64 = fields
1363        .next()
1364        .and_then(|f| f.parse().ok())
1365        .expect("schedstat field 2 (run_delay) must be a present integer");
1366    let pcount: u64 = fields
1367        .next()
1368        .and_then(|f| f.parse().ok())
1369        .expect("schedstat field 3 (pcount) must be a present integer");
1370    (run_delay, pcount)
1371}
1372
1373/// Mean per-schedule run-queue wait (ns) from a raw `(run_delay_ns, pcount)`
1374/// pair: `run_delay / pcount`, schbench's `read_sched_delay` arithmetic
1375/// (`schbench.c:1146`). Guards `pcount == 0` (a never-scheduled thread, or the
1376/// `!sched_info_on()` "0 0 0" line) to 0, where schbench divides by zero. Used
1377/// for the WHOLE-RUN per-thread mean (the mean-of-means component); the
1378/// per-phase path instead keeps the raw pair and re-pools `Σrd/Σpc` host-side.
1379fn mean_sched_delay((run_delay, pcount): (u64, u64)) -> u64 {
1380    if pcount == 0 { 0 } else { run_delay / pcount }
1381}
1382
1383/// Finalize the just-ended phase on the OWNING thread: `take` the live
1384/// histograms (snapshot-and-reset, so the next phase starts clean) and push a
1385/// [`PhaseSnapshot`] tagged `epoch`, carrying the schedstat run-delay/pcount
1386/// delta (`ss_end − ss_start`) and this thread's `loop_count` for the phase.
1387/// Mirrors the worker/mod.rs:3858-3894 backdrop drain-on-change. For the
1388/// message thread the histograms are unwritten (taken empty) and `loop_count`
1389/// is 0; an empty histogram folds harmlessly host-side ([`PlatStats::combine`]).
1390///
1391/// # Safety
1392/// Must be called by the thread that owns `td` — it is the sole writer of
1393/// `td`'s `wakeup_stats` / `request_stats` / `phase_snapshots` cells (the
1394/// [`ThreadData`] `Sync` contract).
1395unsafe fn drain_phase(
1396    td: &ThreadData,
1397    epoch: u32,
1398    ss_start: (u64, u64),
1399    ss_end: (u64, u64),
1400    loop_count: u64,
1401    cpu_ns: u64,
1402) {
1403    // SAFETY: owner-only cell access, per this function's contract.
1404    unsafe {
1405        let wakeup = (*td.wakeup_stats.get()).take();
1406        let request = (*td.request_stats.get()).take();
1407        (*td.phase_snapshots.get()).push(PhaseSnapshot {
1408            epoch,
1409            wakeup,
1410            request,
1411            run_delay_ns: ss_end.0.saturating_sub(ss_start.0),
1412            pcount: ss_end.1.saturating_sub(ss_start.1),
1413            loop_count,
1414            cpu_ns,
1415        });
1416    }
1417}
1418
1419/// The per-message-thread shared context every worker borrows — bundled so
1420/// [`worker_loop`]'s signature stays within clippy's argument budget. All
1421/// fields are `Copy` shared references; every worker spawned by one
1422/// [`run_one_message_thread`] reads the same instance.
1423struct WorkerCtx<'a> {
1424    msg_td: &'a ThreadData,
1425    wait_list: &'a TreiberStack<ThreadData>,
1426    locks: Option<&'a PerCpuLocks>,
1427    config: &'a SchbenchConfig,
1428    stop: &'a AtomicBool,
1429    progress: &'a AtomicU64,
1430    phase_epoch: Option<&'a AtomicU32>,
1431    /// The ONE process-global shared matrix (`--split` mode), shared across
1432    /// every worker of every message thread; empty slice when not splitting.
1433    shared: &'a [core::sync::atomic::AtomicU64],
1434    shared_matrix_size: usize,
1435}
1436
1437/// One worker thread's loop. Faithful to schbench's `worker_thread`
1438/// (`schbench.c:1419`, default branch): block until woken (recording wakeup
1439/// latency), then think-sleep + matrix work under the per-CPU lock (recording
1440/// request latency), until stop. `ctx.progress` counts completed work cycles
1441/// across all workers (the live cycle counter / achieved request rate).
1442///
1443/// `ctx.phase_epoch` (when `Some`) is the shared per-phase generation the
1444/// scenario engine bumps at each step boundary. The worker polls it once per
1445/// cycle and, on change, drains the just-ended phase ([`drain_phase`]) —
1446/// `take`ing its own histograms + a schedstat raw delta — so per-phase
1447/// percentiles are isolated across e.g. an scx→detached-EEVDF transition. A
1448/// final drain at exit closes the last phase (and is the SOLE snapshot when
1449/// non-phasic, `cur_epoch` 0).
1450fn worker_loop(td: &ThreadData, ctx: &WorkerCtx) {
1451    // Destructure the all-`Copy` context into the same bindings the body used
1452    // before the bundle, so the loop logic below reads unchanged.
1453    let WorkerCtx {
1454        msg_td,
1455        wait_list,
1456        locks,
1457        config,
1458        stop,
1459        progress,
1460        phase_epoch,
1461        shared,
1462        shared_matrix_size,
1463    } = *ctx;
1464    let tid = gettid_self();
1465    // Per-worker PRIVATE matrix (schbench.c:1568-1574: private_matrix_size when
1466    // splitting, else the full matrix_size). The shared matrix (split mode) is
1467    // the process-global `shared` allocated once in run().
1468    let private_matrix_size = config.private_matrix_size();
1469    let mut private_buf = if private_matrix_size > 0 {
1470        vec![0u64; 3 * private_matrix_size * private_matrix_size]
1471    } else {
1472        Vec::new()
1473    };
1474    let mut work_units = 0u64;
1475    // Per-phase drain-on-change state. `cur_epoch` is the epoch the current
1476    // phase's samples belong to; `phase_ss_start` baselines the schedstat raw
1477    // pair so each phase reports its own delta.
1478    let mut cur_epoch = phase_epoch.map_or(0, |e| e.load(Ordering::Relaxed));
1479    let mut phase_ss_start = read_schedstat_raw(tid);
1480    let mut phase_loop_count = 0u64;
1481    // CLOCK_THREAD_CPUTIME_ID baseline for the per-phase worker CPU delta
1482    // (the `schbench_loops_per_cpu_sec` denominator). Re-baselined at each
1483    // drain like `phase_ss_start`.
1484    let mut phase_cpu_start = crate::workload::worker::sched::thread_cpu_time_ns();
1485    // RPS mode iff the per-thread rate is non-zero (the `-R` total covers at
1486    // least one request/sec per message thread). Below that it rounds to 0 and
1487    // the worker runs the default message-handshake mode -- matching schbench's
1488    // startup-divide-before-gate ([`SchbenchConfig::rps_per_message_thread`]).
1489    // Pipe mode (`-p`): the memory-transfer workload. It runs the message-handshake
1490    // path (where the pipe-page memsets live), NOT the RPS injector -- a ktstr
1491    // divergence; schbench composes -p+-R half-broken (see the `pipe_transfer_bytes`
1492    // field doc).
1493    let pipe_mode = config.pipe_transfer_bytes > 0;
1494    let rps_mode = !pipe_mode && config.rps_per_message_thread() != 0;
1495    while !stop.load(Ordering::Acquire) {
1496        // Acquire work. Default mode: the message-thread handshake (records
1497        // wakeup latency; no request chain -> a single work cycle). RPS mode:
1498        // splice this worker's request queue, blocking on its own futex until
1499        // the RPS thread posts; returns the spliced chain (FIFO) to drain.
1500        let mut req: *mut Request = if rps_mode {
1501            let chain = rps_wait(td, stop);
1502            if stop.load(Ordering::Acquire) {
1503                // Free any spliced-but-unprocessed requests, then exit.
1504                free_request_chain(chain);
1505                break;
1506            }
1507            if chain.is_null() {
1508                continue; // spurious wake with an empty queue
1509            }
1510            chain
1511        } else {
1512            msg_and_wait(td, msg_td, wait_list, stop);
1513            if stop.load(Ordering::Acquire) {
1514                break;
1515            }
1516            ptr::null_mut()
1517        };
1518        // Process the work: one cycle in default mode (`req` null), or one cycle
1519        // per queued request in RPS mode (freeing each). `work_start` is stamped
1520        // before the think-sleep so the request latency covers think-time +
1521        // matrix work (`schbench.c:1464-1481`).
1522        loop {
1523            let work_start = monotonic_nanos();
1524            // Pipe mode: the per-cycle transfer is the wake/block memsets
1525            // (msg_and_wait fills our page to 2; wake_all fills it to 1). The work
1526            // loop only counts cycles for the MB/s throughput; schbench likewise
1527            // skips do_work + the think-sleep in pipe mode (schbench.c:1448).
1528            if !pipe_mode {
1529                if config.sleep_usec > 0 {
1530                    think_sleep(config.sleep_usec);
1531                }
1532                do_work(
1533                    &mut private_buf,
1534                    private_matrix_size,
1535                    shared,
1536                    shared_matrix_size,
1537                    config.split_percent,
1538                    config.operations,
1539                    locks,
1540                    &mut work_units,
1541                );
1542            }
1543            let now = monotonic_nanos();
1544            let delta_us = now.saturating_sub(work_start) / 1000;
1545            if delta_us > 0 {
1546                // SAFETY: only this worker thread accesses its own request_stats cell.
1547                unsafe { (*td.request_stats.get()).add_lat(delta_us.min(u32::MAX as u64) as u32) };
1548            }
1549            progress.fetch_add(1, Ordering::Relaxed);
1550            phase_loop_count += 1;
1551            if req.is_null() {
1552                break; // default mode: a single work cycle
1553            }
1554            // RPS mode: advance to the next queued request and free this one.
1555            // SAFETY: `req` was `Box::into_raw`'d by the RPS thread and handed to
1556            // this worker exclusively via the queue splice; we own it and free it
1557            // exactly once here.
1558            let next = unsafe { (*req).next.load(Ordering::Acquire) };
1559            drop(unsafe { Box::from_raw(req) });
1560            req = next;
1561            if req.is_null() {
1562                break;
1563            }
1564        }
1565        // Drain-on-change: when the parent advances `phase_epoch`, finalize the
1566        // phase just ended (tagged with the OLD epoch the samples were recorded
1567        // under) and re-baseline. A worker blocked across a whole phase simply
1568        // contributes an empty histogram for it (no cycle ran), which folds
1569        // harmlessly.
1570        if let Some(pe) = phase_epoch {
1571            let new_epoch = pe.load(Ordering::Relaxed);
1572            if new_epoch != cur_epoch {
1573                let ss_end = read_schedstat_raw(tid);
1574                let cpu_end = crate::workload::worker::sched::thread_cpu_time_ns();
1575                // SAFETY: this is the thread that owns `td`.
1576                unsafe {
1577                    drain_phase(
1578                        td,
1579                        cur_epoch,
1580                        phase_ss_start,
1581                        ss_end,
1582                        phase_loop_count,
1583                        cpu_end.saturating_sub(phase_cpu_start),
1584                    )
1585                };
1586                cur_epoch = new_epoch;
1587                phase_ss_start = ss_end;
1588                phase_loop_count = 0;
1589                phase_cpu_start = cpu_end;
1590            }
1591        }
1592    }
1593    // Shutdown wake: the loop exits on `stop` without a final `msg_and_wait`, so
1594    // the message thread may still be parked in `wait_forever` (its only waker is
1595    // a worker post). schbench's MAIN thread handles this -- it fposts each
1596    // message thread after `stopping = 1` (`schbench.c:1832`, `:1933`) -- but our
1597    // `run()` joins passively and `msg_td` is private to `run_one_message_thread`,
1598    // so the exiting worker wakes its own message thread instead. Without this, a
1599    // worker that exits mid-`do_work` leaves the message thread parked forever and
1600    // `run()` deadlocks. Idempotent across workers: the first post the message
1601    // thread observes makes it re-check `stop` and break; later posts are no-ops.
1602    // In RPS mode this is itself a harmless no-op: the dispatcher runs
1603    // `run_rps_thread` (which never parks on `msg_td`) and observes `stop` itself.
1604    msg_td.futex.post();
1605    // Final drain: close the still-open phase. When non-phasic this is the only
1606    // snapshot (`cur_epoch` 0), so run() builds the whole-run result uniformly
1607    // from snapshots.
1608    let ss_end = read_schedstat_raw(tid);
1609    let cpu_end = crate::workload::worker::sched::thread_cpu_time_ns();
1610    // SAFETY: this is the thread that owns `td`.
1611    unsafe {
1612        drain_phase(
1613            td,
1614            cur_epoch,
1615            phase_ss_start,
1616            ss_end,
1617            phase_loop_count,
1618            cpu_end.saturating_sub(phase_cpu_start),
1619        )
1620    };
1621    // Record this worker's whole-run mean run-queue wait at exit — the
1622    // mean-of-means component for the whole-run SchbenchResult (schbench reads
1623    // each thread's schedstat for the final aggregate, `schbench.c:1664-1670`).
1624    // Reuses the cumulative `ss_end` pair just read (one /proc read; mean and
1625    // final-phase delta share one consistent snapshot).
1626    // SAFETY: owner-only access to this thread's sched_delay_ns cell.
1627    unsafe { *td.sched_delay_ns.get() = mean_sched_delay(ss_end) };
1628}
1629
1630/// Resolve the worker-thread default and the per-CPU lock-array size from the
1631/// calling thread's CPU affinity (the allocated cpuset, per ruling). Returns
1632/// `(allowed_cpu_count, lock_array_size)` where the array size is the highest
1633/// allowed CPU id + 1, so `sched_getcpu` indexes it without clamping even on a
1634/// sparse cpuset.
1635fn resolve_cpu_topology() -> (usize, usize) {
1636    // SAFETY: a zeroed cpu_set_t filled by sched_getaffinity for the calling
1637    // thread (pid 0); CPU_ISSET only reads it.
1638    unsafe {
1639        let mut set: libc::cpu_set_t = core::mem::zeroed();
1640        let rc = libc::sched_getaffinity(0, core::mem::size_of::<libc::cpu_set_t>(), &mut set);
1641        if rc != 0 {
1642            return (1, 1);
1643        }
1644        let mut count = 0usize;
1645        let mut max_id = 0usize;
1646        for cpu in 0..libc::CPU_SETSIZE as usize {
1647            if libc::CPU_ISSET(cpu, &set) {
1648                count += 1;
1649                max_id = cpu;
1650            }
1651        }
1652        (count.max(1), (max_id + 1).max(1))
1653    }
1654}
1655
1656/// Combined WHOLE-RUN results of a schbench run: the merged wakeup + request
1657/// latency percentiles and the achieved request rate (completed work
1658/// cycles/second). The histograms are the union of every phase; `sched_delay_*`
1659/// keep schbench's mean-of-means (`collect_sched_delay`) so the side-by-side
1660/// validation matches real schbench's reported number.
1661pub(crate) struct SchbenchResult {
1662    pub(crate) wakeup: Percentiles,
1663    pub(crate) request: Percentiles,
1664    /// Per-second achieved-RPS distribution (schbench's `rps_stats`), sampled
1665    /// once per second by the control thread. For auto-RPS, samples are gated
1666    /// until the target is hit (`schbench.c:1781`); for fixed/default mode every
1667    /// second is sampled.
1668    pub(crate) rps: Percentiles,
1669    pub(crate) loop_count: u64,
1670    /// Resolved total worker-thread count (`message_threads * worker_threads`).
1671    /// Divisor for schbench's PER-WORKER pipe-mode rate (`avg worker transfer`):
1672    /// schbench reports `loop_count / Σ worker runtimes` (`schbench.c:1697`,
1673    /// `:1942-1943`/`:1979`), and `Σ worker runtimes ≈ nr_workers * elapsed`, so
1674    /// the per-worker rate is `achieved_rps / nr_workers`.
1675    pub(crate) nr_workers: usize,
1676    /// Aggregate completed cycles / WALL second over the run window.
1677    /// Deliberately wall-denominated: this feeds ONLY the standalone /
1678    /// validate surface (`run_standalone` → `ktstr-schbench-validate`),
1679    /// whose output must stay byte-identical to upstream schbench's
1680    /// wall-clock report. The ktstr sidecar's throughput metric is the
1681    /// CPU-second-denominated `schbench_loops_per_cpu_sec` (from the
1682    /// per-phase `worker_cpu_ns` carrier), not this field.
1683    pub(crate) achieved_rps: f64,
1684    /// Auto-RPS final TOTAL target rate at run exit: the live per-message-thread
1685    /// rate * message_threads (schbench's `requests_per_sec * message_threads`,
1686    /// `schbench.c:1995`). For fixed `-R` and default mode this equals the seeded
1687    /// total; only auto-RPS makes it diverge as the control loop retargets.
1688    pub(crate) final_rps_goal: usize,
1689    /// Mean message-thread run-queue wait (ns), averaged across message threads
1690    /// (schbench's `message_thread_delay`, `schbench.c:1673`).
1691    pub(crate) sched_delay_msg_ns: u64,
1692    /// Mean worker-thread run-queue wait (ns), averaged across all workers
1693    /// (schbench's `worker_thread_delay`, `schbench.c:1674`).
1694    pub(crate) sched_delay_worker_ns: u64,
1695}
1696
1697/// The full result of [`run`]: the whole-run [`SchbenchResult`] plus the
1698/// per-phase aggregates keyed by `phase_epoch` (each the cross-thread merge for
1699/// one scenario step's HOLD window). When non-phasic (`phase_epoch == None`)
1700/// `phases` holds the single `(0, ..)` baseline entry the host discards; tests
1701/// read `whole_run`. The worker dispatch turns each `(epoch, SchbenchPhaseStats)`
1702/// into a `PhaseSlice`.
1703pub(crate) struct SchbenchOutcome {
1704    pub(crate) whole_run: SchbenchResult,
1705    pub(crate) phases: Vec<(u32, SchbenchPhaseStats)>,
1706}
1707
1708/// One message thread's pooled results, returned from [`run_one_message_thread`]
1709/// to [`run`]: the whole-run histograms (Σ over its workers' phase snapshots),
1710/// the run-delay components for the whole-run mean-of-means, and the per-epoch
1711/// aggregate keyed by `phase_epoch`.
1712struct MessageThreadResult {
1713    whole_wakeup: PlatStats,
1714    whole_request: PlatStats,
1715    msg_sched_delay_ns: u64,
1716    workers_sched_delay_sum: u64,
1717    phases: std::collections::BTreeMap<u32, SchbenchPhaseStats>,
1718}
1719
1720/// Run one message thread plus its workers, returning the whole-run histograms
1721/// AND the per-epoch aggregate. The message thread runs on the calling thread;
1722/// workers are scoped so they are joined before their owner-only phase snapshots
1723/// are drained.
1724#[allow(clippy::too_many_arguments)]
1725fn run_one_message_thread(
1726    worker_threads: usize,
1727    locks: Option<&PerCpuLocks>,
1728    config: &SchbenchConfig,
1729    stop: &AtomicBool,
1730    progress: &AtomicU64,
1731    phase_epoch: Option<&AtomicU32>,
1732    live_rate: &AtomicUsize,
1733    shared: &[core::sync::atomic::AtomicU64],
1734    shared_matrix_size: usize,
1735) -> MessageThreadResult {
1736    // schbench clamps -p to PIPE_TRANSFER_BUFFER (schbench.c:291-294). Workers get
1737    // the per-thread transfer page; the message thread (waker) needs none.
1738    let pipe_bytes = config.pipe_transfer_bytes.min(PIPE_TRANSFER_BUFFER);
1739    let workers: Vec<ThreadData> = (0..worker_threads)
1740        .map(|_| ThreadData::new(pipe_bytes))
1741        .collect();
1742    let msg_td = ThreadData::new(0);
1743    let wait_list = TreiberStack::new();
1744    let ctx = WorkerCtx {
1745        msg_td: &msg_td,
1746        wait_list: &wait_list,
1747        locks,
1748        config,
1749        stop,
1750        progress,
1751        phase_epoch,
1752        shared,
1753        shared_matrix_size,
1754    };
1755
1756    std::thread::scope(|inner| {
1757        for w in &workers {
1758            inner.spawn(|| worker_loop(w, &ctx));
1759        }
1760        // Dispatcher: RPS-injector mode runs the rate-driven request thread;
1761        // default mode runs the message-thread handshake. Both run on this
1762        // calling thread and record the dispatcher's run-delay into `msg_td`
1763        // (schbench `message_thread`, `:1594`).
1764        // Pipe mode (`-p`) uses the message-handshake waker (run_msg_thread does
1765        // the per-worker pipe-page fill in wake_all) and takes precedence over -R.
1766        // This DIVERGES from schbench, where -R alone picks the path
1767        // (`schbench.c:1594`): schbench's `-p -R` runs the RPS injector (no
1768        // xlist_wake_all -> no waker-side memset -> a half-pipe). ktstr runs the
1769        // full pipe; the realistic use is `-p` alone. Otherwise -R selects the
1770        // injector.
1771        if config.pipe_transfer_bytes == 0 && config.rps_per_message_thread() != 0 {
1772            run_rps_thread(&workers, &msg_td, stop, phase_epoch, live_rate);
1773        } else {
1774            run_msg_thread(&msg_td, &wait_list, stop, phase_epoch);
1775        }
1776        // Stop is set: wake every worker so a blocked one observes stop and
1777        // exits (schbench fposts each worker before joining, `:1599-1602`).
1778        for w in &workers {
1779            w.futex.post();
1780        }
1781        // The inner scope joins the workers here.
1782    });
1783
1784    // RPS mode: free any requests the RPS thread enqueued after a worker's last
1785    // splice (the worker exited on stop without draining them). After the join,
1786    // this is the sole access to each worker's queue, so no request leaks across
1787    // run() calls.
1788    if config.rps_per_message_thread() != 0 {
1789        for w in &workers {
1790            free_request_chain(w.requests.splice_reversed());
1791        }
1792    }
1793
1794    // After join (happens-before): drain each thread's owner-only phase
1795    // snapshots into the whole-run histograms + the per-epoch aggregate.
1796    let mut whole_wakeup = PlatStats::default();
1797    let mut whole_request = PlatStats::default();
1798    let mut workers_sched_delay_sum = 0u64;
1799    let mut phases: std::collections::BTreeMap<u32, SchbenchPhaseStats> =
1800        std::collections::BTreeMap::new();
1801    for w in &workers {
1802        // SAFETY: every worker has joined (inner scope ended), so this is the
1803        // sole access to their cells.
1804        unsafe {
1805            workers_sched_delay_sum =
1806                workers_sched_delay_sum.saturating_add(*w.sched_delay_ns.get());
1807            for snap in (*w.phase_snapshots.get()).drain(..) {
1808                whole_wakeup.combine(&snap.wakeup);
1809                whole_request.combine(&snap.request);
1810                let e = phases.entry(snap.epoch).or_default();
1811                e.wakeup.combine(&snap.wakeup);
1812                e.request.combine(&snap.request);
1813                e.worker_run_delay_ns = e.worker_run_delay_ns.saturating_add(snap.run_delay_ns);
1814                e.worker_pcount = e.worker_pcount.saturating_add(snap.pcount);
1815                e.loop_count = e.loop_count.saturating_add(snap.loop_count);
1816                e.worker_cpu_ns = e.worker_cpu_ns.saturating_add(snap.cpu_ns);
1817            }
1818        }
1819    }
1820    // SAFETY: run_msg_thread ran on this thread inside the scope above (before
1821    // the join), so msg_td's cells are settled and this is sole access.
1822    let msg_sched_delay = unsafe { *msg_td.sched_delay_ns.get() };
1823    // SAFETY: same — sole access to msg_td's snapshot cell after the scope.
1824    unsafe {
1825        for snap in (*msg_td.phase_snapshots.get()).drain(..) {
1826            let e = phases.entry(snap.epoch).or_default();
1827            e.msg_run_delay_ns = e.msg_run_delay_ns.saturating_add(snap.run_delay_ns);
1828            e.msg_pcount = e.msg_pcount.saturating_add(snap.pcount);
1829        }
1830    }
1831    MessageThreadResult {
1832        whole_wakeup,
1833        whole_request,
1834        msg_sched_delay_ns: msg_sched_delay,
1835        workers_sched_delay_sum,
1836        phases,
1837    }
1838}
1839
1840/// Persistent /proc/stat reader state for auto-RPS, carrying the previous
1841/// cumulative totals so each call computes a delta over the interval (schbench
1842/// keeps one fd + the prior total/idle, `schbench.c` `read_busy`, `:1046`).
1843#[derive(Default)]
1844struct ReadBusyState {
1845    fd: Option<File>,
1846    prev_total: u64,
1847    prev_idle: u64,
1848}
1849
1850/// Host-busy percentage over the interval since the last call, from the
1851/// aggregate `cpu` line of /proc/stat (schbench `read_busy`,
1852/// `schbench.c:1046-1112`). `None` means "no usable reading this tick": the first
1853/// call (baseline seed, schbench's `first_run`) OR a zero jiffy-delta (two reads
1854/// in the same tick -- schbench has no guard and divides by zero into a NaN that
1855/// spuriously trips its target-hit `else` branch; we skip the tick instead). A
1856/// real reading is `Some(0.0..=100.0)`, including a genuine fully-idle `Some(0.0)`
1857/// -- distinguished from the seed via the `prev_total` state, NOT via the value,
1858/// so an idle interval still drives a grow (the bug a `busy == 0.0` test had). In
1859/// the guest VM /proc/stat covers exactly the allocated cpuset (the guest has
1860/// only `cores=N`), so this system-wide read is correctly scoped to the
1861/// workload's CPUs. f32 to match schbench's `float` arithmetic.
1862fn read_busy(s: &mut ReadBusyState) -> Option<f32> {
1863    if s.fd.is_none() {
1864        s.fd = File::open("/proc/stat").ok();
1865    }
1866    let f = s.fd.as_mut()?;
1867    f.seek(SeekFrom::Start(0)).ok()?;
1868    let mut buf = [0u8; 512];
1869    let n = f.read(&mut buf).ok()?;
1870    let text = core::str::from_utf8(&buf[..n]).ok()?;
1871    // First line only; first token must be "cpu" (the aggregate line). Sum every
1872    // numeric field -> total; field index 3 (after "cpu") is idle -- kernel
1873    // show_stat order is user/nice/system/IDLE/iowait/irq/softirq/steal/guest/
1874    // guest_nice. `split_whitespace` collapses the "cpu  " double space, matching
1875    // schbench's `strtok_r` (`schbench.c:1082-1096`).
1876    let line = text.lines().next().unwrap_or("");
1877    let mut fields = line.split_whitespace();
1878    if fields.next() != Some("cpu") {
1879        return None;
1880    }
1881    let mut total = 0u64;
1882    let mut idle = 0u64;
1883    for (i, tok) in fields.enumerate() {
1884        let v: u64 = tok.parse().unwrap_or(0);
1885        if i == 3 {
1886            idle = v;
1887        }
1888        total += v;
1889    }
1890    // First call: seed the baseline, no busy% yet (`schbench.c:1098-1101`).
1891    if s.prev_total == 0 {
1892        s.prev_total = total;
1893        s.prev_idle = idle;
1894        return None;
1895    }
1896    let dt = total.saturating_sub(s.prev_total);
1897    let di = idle.saturating_sub(s.prev_idle);
1898    s.prev_total = total;
1899    s.prev_idle = idle;
1900    if dt == 0 {
1901        // Zero jiffy-delta: skip rather than NaN (see the doc; schbench diverges).
1902        return None;
1903    }
1904    Some(100.0 - (di as f32 / dt as f32) * 100.0)
1905}
1906
1907/// Grow or shrink the live per-message-thread request rate toward `target_pct`
1908/// host-busy, given an already-read `busy` percentage (schbench `auto_scale_rps`'s
1909/// scaling body, `schbench.c:1203-1250`). Pure: no I/O, so the scaling decision is
1910/// unit-tested with injected `busy` values. Stores the new per-thread rate into
1911/// `live_rate` -- schbench scales its global `requests_per_sec` (already per-thread
1912/// after `:1899`) and writes it back at `:1250`. Returns true on the transition
1913/// INTO the near-target band (when `target_hit` flips false->true), so the caller
1914/// resets the rps histogram (schbench memsets `rps_stats` on that transition). The
1915/// near-target test reads the delta AFTER its damping reassignment, exactly as
1916/// schbench does (`:1210`,`:1233`). f32 math + `ceil` (grow) / `floor` (shrink)
1917/// match schbench's float deltas + rounding.
1918fn scale_rps_for_busy(
1919    busy: f32,
1920    live_rate: &AtomicUsize,
1921    target_hit: &AtomicBool,
1922    target_pct: usize,
1923) -> bool {
1924    let target = target_pct as f32;
1925    let rps = live_rate.load(Ordering::Relaxed) as f32;
1926    let already_hit = target_hit.load(Ordering::Relaxed);
1927    let mut just_hit = false;
1928    let new_rate: usize = if busy < target {
1929        // Under target: grow (`schbench.c:1203-1224`).
1930        let mut delta = target / busy;
1931        if delta > 3.0 {
1932            delta = 3.0;
1933        } else if delta < 1.2 {
1934            delta = 1.0 + (delta - 1.0) / 8.0;
1935            // Threshold check AFTER the damping reassignment (`schbench.c:1209-1213`).
1936            if delta < 1.05 && !already_hit {
1937                just_hit = true;
1938            }
1939        } else if delta < 1.5 {
1940            delta = 1.0 + (delta - 1.0) / 4.0;
1941        }
1942        let t = (rps * delta).ceil();
1943        // Not enough capacity to reach the target -> hold (`schbench.c:1219-1224`).
1944        if t >= (1u64 << 31) as f32 {
1945            rps as usize
1946        } else {
1947            t as usize
1948        }
1949    } else if busy > target {
1950        // Over target: shrink (`schbench.c:1226-1242`).
1951        let mut delta = target / busy;
1952        if delta < 0.3 {
1953            delta = 0.3;
1954        } else if delta > 0.9 {
1955            delta += (1.0 - delta) / 8.0;
1956            // Threshold check AFTER the damping reassignment (`schbench.c:1232-1236`).
1957            if delta > 0.95 && !already_hit {
1958                just_hit = true;
1959            }
1960        } else if delta > 0.8 {
1961            delta += (1.0 - delta) / 4.0;
1962        }
1963        (rps * delta).floor().max(0.0) as usize
1964    } else {
1965        // Exactly on target (`schbench.c:1243-1248`).
1966        if !already_hit {
1967            just_hit = true;
1968        }
1969        rps as usize
1970    };
1971    live_rate.store(new_rate, Ordering::Relaxed);
1972    if just_hit {
1973        target_hit.store(true, Ordering::Relaxed);
1974    }
1975    just_hit
1976}
1977
1978/// Read host-busy from /proc/stat and scale the live rate toward the target
1979/// (schbench `auto_scale_rps`, `schbench.c:1180-1251`). The first read only seeds
1980/// the baseline (schbench's `first_run` early return, `:1201`); a zero-delta tick
1981/// is skipped (see [`read_busy`]). Returns [`scale_rps_for_busy`]'s just-hit flag.
1982fn auto_scale_rps(
1983    busy_state: &mut ReadBusyState,
1984    live_rate: &AtomicUsize,
1985    target_hit: &AtomicBool,
1986    target_pct: usize,
1987) -> bool {
1988    let Some(busy) = read_busy(busy_state) else {
1989        return false;
1990    };
1991    scale_rps_for_busy(busy, live_rate, target_hit, target_pct)
1992}
1993
1994/// Reset BOTH per-phase rps accumulators in LOCKSTEP at an auto-RPS target-hit.
1995/// schbench memsets `rps_stats` on the target-hit transition to discard the
1996/// pre-target ramp; the per-epoch map MUST reset with it, or the per-phase view
1997/// keeps ramp samples the whole-run view dropped and `Σ per-epoch == whole-run`
1998/// breaks (asymmetric-counter-bookkeeping).
1999fn reset_rps_accumulators(
2000    whole: &mut PlatStats,
2001    per_epoch: &mut std::collections::BTreeMap<u32, PlatStats>,
2002) {
2003    *whole = PlatStats::default();
2004    per_epoch.clear();
2005}
2006
2007/// The once-per-second control thread, schbench's main control loop
2008/// (`schbench.c:1756-1827`): sample the achieved per-second RPS into `rps_stats`
2009/// and -- when auto-RPS is on -- adjust the live rate toward the target. Returns
2010/// BOTH the whole-run `rps_stats` AND a per-epoch RPS map (the same samples
2011/// bucketed by the phase epoch observed at sample time), moved out on join. The
2012/// whole-run `rps_stats` is byte-identical to the pre-per-phase behavior -- the
2013/// per-epoch map is PURELY ADDITIVE (each sample lands in the whole-run histogram
2014/// AND exactly one epoch bucket), so `Σ per-epoch == whole-run`. `phase_epoch` is
2015/// read Relaxed (telemetry, gates no coupled memory); a 1s sample straddling an
2016/// epoch change is attributed wholly to the sample-time epoch (bounded fuzz,
2017/// mirroring the worker drain-on-change). The 1-second cadence is the workload's
2018/// defined sampling/scaling interval, not a synchronization wait; it observes
2019/// `stop` after each interval.
2020fn control_loop(
2021    progress: &AtomicU64,
2022    stop: &AtomicBool,
2023    config: &SchbenchConfig,
2024    live_rate: &AtomicUsize,
2025    target_hit: &AtomicBool,
2026    phase_epoch: Option<&AtomicU32>,
2027) -> (PlatStats, std::collections::BTreeMap<u32, PlatStats>) {
2028    let mut rps_stats = PlatStats::default();
2029    let mut per_epoch_rps: std::collections::BTreeMap<u32, PlatStats> =
2030        std::collections::BTreeMap::new();
2031    let mut last_loop = 0u64;
2032    let mut last_t = monotonic_nanos();
2033    let mut busy_state = ReadBusyState::default();
2034    while !stop.load(Ordering::Acquire) {
2035        // Sleep ~1s between samples, but in STOP_POLL_QUANTUM_NS chunks that
2036        // re-check `stop`, so a shutdown is observed within a quantum instead of
2037        // blocking run()'s teardown for a full second (the control thread is joined
2038        // by run(), so an uncapped sleep here is the dominant teardown latency —
2039        // worse than, and unconditional unlike, the RPS injector's pacing sleep).
2040        let sleep_target = monotonic_nanos().saturating_add(1_000_000_000);
2041        loop {
2042            if stop.load(Ordering::Acquire) {
2043                break;
2044            }
2045            let now = monotonic_nanos();
2046            if now >= sleep_target {
2047                break;
2048            }
2049            std::thread::sleep(std::time::Duration::from_nanos(
2050                (sleep_target - now).min(STOP_POLL_QUANTUM_NS),
2051            ));
2052        }
2053        // Sample the just-elapsed window BEFORE re-checking stop at the loop top,
2054        // so the in-progress second is recorded rather than dropped. `dt` is the
2055        // ACTUAL elapsed time (now - last_t).
2056        //
2057        // schbench samples-first then sleeps the full second (`sleep(1)` only
2058        // `if (!done)`, schbench.c:1827), so every schbench sample -- including the
2059        // boundary one taken when `done` is detected -- spans a real ~1s window
2060        // (rps over a real `delta = tvdelta(..)`, schbench.c:1774,1777). ktstr
2061        // sleeps-first/samples-last with a BOUNDED sleep, which diverges two ways:
2062        //   - On early `stop` this boundary sample spans a true PARTIAL window: for
2063        //     a substantial window (>= one quantum) the real rate over the
2064        //     sub-window the workers were active (more granular than schbench, which
2065        //     would have waited the full second); a degenerate sub-quantum window is
2066        //     dropped by `control_loop_rps_sample` (a count/dt rate is unbounded as
2067        //     dt->0). The bounded sleep is a ktstr scenario-teardown requirement
2068        //     schbench does not face.
2069        //   - The sleep-first shape omits schbench's tiny FIRST-window sample
2070        //     (schbench reads `now` at loop start before any work, schbench.c:1757,
2071        //     so its first `delta` is ~us); ktstr's first sample follows a full ~1s
2072        //     sleep.
2073        let now = monotonic_nanos();
2074        let lc = progress.load(Ordering::Relaxed);
2075        let dt = now.saturating_sub(last_t);
2076        // Achieved per-second rate = Δcompleted-cycles / Δt (schbench
2077        // `combine_message_thread_rps` over the shared loop count, schbench.c:1777),
2078        // floored at one quantum by `control_loop_rps_sample` (a ktstr teardown
2079        // guard, not in schbench — see its doc).
2080        if let Some(sample) = control_loop_rps_sample(lc.saturating_sub(last_loop), dt) {
2081            // Gate: for auto-RPS, sample only once the target is hit
2082            // (`schbench.c:1781`); fixed/default mode samples every second.
2083            if config.auto_rps == 0 || target_hit.load(Ordering::Relaxed) {
2084                rps_stats.add_lat(sample);
2085                // Per-phase parity: also bucket the sample by the epoch observed
2086                // NOW (the sample-time epoch; a straddling 1s sample lands wholly
2087                // here, bounded fuzz). 0/u32::MAX sentinel epochs route into the
2088                // same all_phases entries the host discards (the run() merge).
2089                let epoch = phase_epoch.map_or(0, |e| e.load(Ordering::Relaxed));
2090                per_epoch_rps.entry(epoch).or_default().add_lat(sample);
2091            }
2092        }
2093        last_loop = lc;
2094        last_t = now;
2095        if config.auto_rps != 0 {
2096            let just_hit = auto_scale_rps(&mut busy_state, live_rate, target_hit, config.auto_rps);
2097            if just_hit {
2098                // Discard ALL pre-target ramp samples (schbench memsets rps_stats
2099                // on the target-hit transition -- grow/shrink/on-target bands all
2100                // funnel through just_hit, `schbench.c:1212`/`:1235`/`:1247`). The
2101                // per-epoch map resets in LOCKSTEP so the per-phase view never
2102                // includes ramp samples the whole-run view discards (the
2103                // Σ-per-epoch == whole-run invariant survives the reset).
2104                reset_rps_accumulators(&mut rps_stats, &mut per_epoch_rps);
2105            }
2106        }
2107    }
2108    (rps_stats, per_epoch_rps)
2109}
2110
2111/// Achieved per-second rate sample for one control-loop window:
2112/// Δcompleted-cycles over `dt_ns`, in cycles/sec, clamped to `u32`. `None` when
2113/// the window is shorter than one `STOP_POLL_QUANTUM_NS` quantum.
2114///
2115/// The floor is a correctness guard, not a nicety. The bounded teardown sleep in
2116/// `control_loop` can return on `stop` with a sub-millisecond `dt` (the window
2117/// between a sample's `last_t = now` and the next iteration's first stop check),
2118/// and a count/dt RATE is unbounded as `dt -> 0` (a single completion in a 1us
2119/// window reads as ~1e6 cycles/sec), so an unfloored sub-quantum sample would
2120/// spike `rps_max` and the top rps percentiles -- assertable metrics. A real
2121/// partial window is always >= one quantum (the sleep loop only observes `stop`
2122/// at a quantum boundary), so the floor drops only the degenerate race window; an
2123/// admitted window bounds the sample to `<= delta_loops * 1e9 /
2124/// STOP_POLL_QUANTUM_NS` (~ the real rate, no inflation). This diverges from
2125/// schbench's `isfinite(rps)` guard (schbench.c:1782), which catches only an
2126/// exact-zero delta -- a finite-but-huge rate from a tiny window slips past it,
2127/// but schbench's full-second cadence (schbench.c:1827) never produces one.
2128fn control_loop_rps_sample(delta_loops: u64, dt_ns: u64) -> Option<u32> {
2129    if dt_ns < STOP_POLL_QUANTUM_NS {
2130        return None;
2131    }
2132    let rps = delta_loops as f64 * 1e9 / dt_ns as f64;
2133    Some((rps as u64).min(u32::MAX as u64) as u32)
2134}
2135
2136/// Run the schbench workload until `stop` is set, returning the whole-run
2137/// percentiles + achieved rate AND the per-phase aggregates. `progress` is the
2138/// live count of completed work cycles across all workers. `phase_epoch` (when
2139/// `Some`) is the scenario engine's shared per-phase generation; the engine
2140/// splits its histograms at each transition (see [`worker_loop`]).
2141pub(crate) fn run(
2142    config: &SchbenchConfig,
2143    stop: &AtomicBool,
2144    progress: &AtomicU64,
2145    phase_epoch: Option<&AtomicU32>,
2146) -> SchbenchOutcome {
2147    let (allowed_count, lock_array_size) = resolve_cpu_topology();
2148    let worker_threads = config.resolve_worker_count(allowed_count);
2149    let locks = if config.skip_locking {
2150        None
2151    } else {
2152        Some(PerCpuLocks::new(lock_array_size))
2153    };
2154    // schbench rejects a `--split` outside 0..=100 at parse and exits
2155    // (schbench.c:362-365). ktstr's analog is a loud panic here at the
2156    // consumption boundary -- before any matrix sizing or the shared Arc
2157    // allocation below -- so an out-of-range config (a `pub split_percent` set
2158    // via struct literal bypasses the builder's debug_assert) can never
2159    // underflow `100 - p` (a release `usize` wrap that would size a garbage
2160    // shared matrix and OOM the allocation) or silently mis-split.
2161    if let Some(p) = config.split_percent {
2162        assert!(
2163            p <= 100,
2164            "schbench split_percent must be in 0..=100, got {p} (schbench.c:362-365 rejects out of range)"
2165        );
2166    }
2167    // schbench.c:1871-1872: ONE process-global shared matrix for `--split`,
2168    // allocated once and shared (interior-mutable AtomicU64) by every worker
2169    // across all message threads. Empty when not splitting (`None`) or when the
2170    // shared portion rounds to 0 (split=100): the `shared_matrix_size > 0`
2171    // guard in do_work then skips the shared branch, so the empty slice is
2172    // never indexed.
2173    let shared_matrix_size = config.shared_matrix_size();
2174    let shared: std::sync::Arc<[core::sync::atomic::AtomicU64]> = if shared_matrix_size > 0 {
2175        (0..3 * shared_matrix_size * shared_matrix_size)
2176            .map(|_| core::sync::atomic::AtomicU64::new(0))
2177            .collect()
2178    } else {
2179        std::sync::Arc::from([] as [core::sync::atomic::AtomicU64; 0])
2180    };
2181
2182    let start = monotonic_nanos();
2183    let mut all_wakeup = PlatStats::default();
2184    let mut all_request = PlatStats::default();
2185    let mut total_msg_sched_delay = 0u64;
2186    let mut total_worker_sched_delay = 0u64;
2187    let mut all_phases: std::collections::BTreeMap<u32, SchbenchPhaseStats> =
2188        std::collections::BTreeMap::new();
2189
2190    // The live PER-MESSAGE-THREAD request rate the RPS injectors read each second
2191    // -- schbench's global `requests_per_sec` AFTER its startup `/= message_threads`
2192    // (`schbench.c:1899`), which is exactly the value `auto_scale_rps` scales
2193    // (`schbench.c:1250`) and each `run_rps_thread` injects directly with no
2194    // further division (`schbench.c:1290`). Seeded from `rps_per_message_thread()`
2195    // (the -R total / m, or the auto-RPS seed 10 / m). The control thread
2196    // auto-scales it when auto-RPS is on; otherwise it stays constant -- a fixed
2197    // -R then injects that seed rate unchanged. `target_hit` gates the rps_stats sampling
2198    // for auto-RPS (`schbench.c:1781`); `rps_stats` is moved out of the control
2199    // thread on join. `live_rate` is read back after the scope as the auto-RPS
2200    // "final rps goal" (`schbench.c:1995`, per-thread * m).
2201    //
2202    // Ordering: both atomics are lone scalars that gate NO coupled memory --
2203    // `live_rate` is a plain count the RPS thread loops on (the requests it
2204    // enqueues publish via the Treiber stack's own AcqRel), and `target_hit` is
2205    // touched only by the control thread. So Relaxed is correct (a 1-second-stale
2206    // read is benign within the auto-RPS cadence; schbench reads its global with
2207    // no sync at all). The final read after the scope is synchronized by the join.
2208    let live_rate = AtomicUsize::new(config.rps_per_message_thread());
2209    let target_hit = AtomicBool::new(false);
2210    let mut rps_stats = PlatStats::default();
2211
2212    std::thread::scope(|outer| {
2213        let handles: Vec<_> = (0..config.message_threads)
2214            .map(|_| {
2215                let locks = locks.as_ref();
2216                let live_rate = &live_rate;
2217                let shared = &shared;
2218                outer.spawn(move || {
2219                    run_one_message_thread(
2220                        worker_threads,
2221                        locks,
2222                        config,
2223                        stop,
2224                        progress,
2225                        phase_epoch,
2226                        live_rate,
2227                        &shared[..],
2228                        shared_matrix_size,
2229                    )
2230                })
2231            })
2232            .collect();
2233        // Control thread: schbench's main control loop -- once/sec, sample the
2234        // achieved RPS into rps_stats and (for auto-RPS) auto-scale `live_rate`.
2235        let control = outer
2236            .spawn(|| control_loop(progress, stop, config, &live_rate, &target_hit, phase_epoch));
2237        for h in handles {
2238            let mtr = h.join().expect("schbench message thread panicked");
2239            all_wakeup.combine(&mtr.whole_wakeup);
2240            all_request.combine(&mtr.whole_request);
2241            total_msg_sched_delay = total_msg_sched_delay.saturating_add(mtr.msg_sched_delay_ns);
2242            total_worker_sched_delay =
2243                total_worker_sched_delay.saturating_add(mtr.workers_sched_delay_sum);
2244            for (epoch, sps) in mtr.phases {
2245                all_phases.entry(epoch).or_default().merge(&sps);
2246            }
2247        }
2248        let (whole_rps, control_per_epoch_rps) =
2249            control.join().expect("schbench control thread panicked");
2250        rps_stats = whole_rps;
2251        // Merge the control thread's per-epoch RPS into the per-phase aggregate
2252        // EXACTLY ONCE, post-join, OUTSIDE the per-message-thread loop above (else
2253        // it would double-count by message_threads). The control thread is the
2254        // SOLE rps source; the workers contribute empty rps via mtr.phases.
2255        for (epoch, hist) in control_per_epoch_rps {
2256            all_phases.entry(epoch).or_default().rps.combine(&hist);
2257        }
2258    });
2259
2260    // The auto-RPS "final rps goal" (`schbench.c:1995`): the live per-thread rate
2261    // at exit * message_threads. Read after the scope (all threads joined, so the
2262    // control loop's last store is visible).
2263    let final_rps_goal = live_rate.load(Ordering::Relaxed) * config.message_threads;
2264    let loop_count = progress.load(Ordering::Relaxed);
2265    let elapsed_ns = monotonic_nanos().saturating_sub(start);
2266    let achieved_rps = if elapsed_ns > 0 {
2267        loop_count as f64 / (elapsed_ns as f64 / 1e9)
2268    } else {
2269        0.0
2270    };
2271    // Average the per-thread run-queue waits, matching schbench's
2272    // collect_sched_delay (`schbench.c:1673-1674`): message delay over
2273    // message_threads, worker delay over all workers. (Whole-run mean-of-means;
2274    // the per-phase path uses sample-weighted Σrd/Σpc — see SchbenchPhaseStats.)
2275    let sched_delay_msg_ns = total_msg_sched_delay / (config.message_threads.max(1) as u64);
2276    let total_workers = (config.message_threads * worker_threads).max(1) as u64;
2277    let sched_delay_worker_ns = total_worker_sched_delay / total_workers;
2278
2279    SchbenchOutcome {
2280        whole_run: SchbenchResult {
2281            wakeup: all_wakeup.percentiles(),
2282            request: all_request.percentiles(),
2283            rps: rps_stats.percentiles(),
2284            loop_count,
2285            nr_workers: total_workers as usize,
2286            achieved_rps,
2287            final_rps_goal,
2288            sched_delay_msg_ns,
2289            sched_delay_worker_ns,
2290        },
2291        phases: all_phases.into_iter().collect(),
2292    }
2293}
2294
2295#[cfg(test)]
2296mod tests {
2297    use super::*;
2298
2299    /// Lightweight node for stack stress tests (ThreadData is far heavier —
2300    /// two boxed `PlatStats` histograms plus per-thread state — too heavy to
2301    /// allocate by the thousand).
2302    struct TestNode {
2303        next: AtomicPtr<TestNode>,
2304    }
2305    impl Linked for TestNode {
2306        fn next_link(&self) -> &AtomicPtr<Self> {
2307            &self.next
2308        }
2309    }
2310    impl TestNode {
2311        fn new() -> Self {
2312            Self {
2313                next: AtomicPtr::new(ptr::null_mut()),
2314            }
2315        }
2316    }
2317
2318    #[test]
2319    fn resolve_worker_count_divides_cpuset_across_message_threads() {
2320        // Explicit non-zero worker_threads is honored as-is (per message thread).
2321        let c = SchbenchConfig::default()
2322            .worker_threads(3)
2323            .message_threads(4);
2324        assert_eq!(c.resolve_worker_count(8), 3);
2325
2326        // 0-default mirrors schbench's ceil(cpuset_cpus / message_threads)
2327        // (schbench.c:1849-1852), scoped to the cpuset count:
2328        //   m=1: 8/1 = 8 per thread, total 8 (== the cpuset count; no divide).
2329        let c = SchbenchConfig::default().message_threads(1);
2330        assert_eq!(c.resolve_worker_count(8), 8);
2331        //   m=2: 8/2 = 4 per thread, total 8.
2332        let c = SchbenchConfig::default().message_threads(2);
2333        assert_eq!(c.resolve_worker_count(8), 4);
2334        //   m=3: ceil(8/3) = 3 per thread, total 9 (≈ 8; schbench rounds up too).
2335        let c = SchbenchConfig::default().message_threads(3);
2336        assert_eq!(c.resolve_worker_count(8), 3);
2337        //   message_threads floored at 1 — no div-by-zero.
2338        let c = SchbenchConfig::default().message_threads(0);
2339        assert_eq!(c.resolve_worker_count(8), 8);
2340    }
2341
2342    #[test]
2343    fn matrix_size_matches_schbench_formula() {
2344        // sqrt(256*1024/3/8) = sqrt(10922) = 104.
2345        assert_eq!(
2346            SchbenchConfig {
2347                cache_footprint_kib: 256,
2348                operations: 5,
2349                ..Default::default()
2350            }
2351            .matrix_size(),
2352            104
2353        );
2354        // Zero operations -> no matrix work.
2355        assert_eq!(
2356            SchbenchConfig {
2357                operations: 0,
2358                ..Default::default()
2359            }
2360            .matrix_size(),
2361            0
2362        );
2363    }
2364
2365    #[test]
2366    fn split_matrix_sizes_match_schbench_formula() {
2367        // schbench.c:1858-1863: with --split=p the footprint splits into a
2368        // private matrix (p%) and a shared matrix (100-p%); each dimension is
2369        // sqrt(kib*frac/100 * 1024/3/sizeof(ulong)). None => the legacy single
2370        // all-private matrix (full footprint) and no shared matrix.
2371        let cfg = |split| SchbenchConfig {
2372            cache_footprint_kib: 256,
2373            operations: 5,
2374            split_percent: split,
2375            ..Default::default()
2376        };
2377        // None: private == the full-footprint matrix_size (104), no shared.
2378        assert_eq!(cfg(None).private_matrix_size(), 104);
2379        assert_eq!(cfg(None).shared_matrix_size(), 0);
2380        // 0%: nothing private; the whole footprint is the shared matrix.
2381        assert_eq!(cfg(Some(0)).private_matrix_size(), 0);
2382        assert_eq!(cfg(Some(0)).shared_matrix_size(), 104);
2383        // 25%: 64 KiB private (sqrt(64*1024/3/8)=52), 192 KiB shared (=90).
2384        assert_eq!(cfg(Some(25)).private_matrix_size(), 52);
2385        assert_eq!(cfg(Some(25)).shared_matrix_size(), 90);
2386        // 100%: all private (full footprint), no shared work.
2387        assert_eq!(cfg(Some(100)).private_matrix_size(), 104);
2388        assert_eq!(cfg(Some(100)).shared_matrix_size(), 0);
2389        // Zero operations -> no matrix work either way.
2390        let none_ops = SchbenchConfig {
2391            operations: 0,
2392            split_percent: Some(50),
2393            ..Default::default()
2394        };
2395        assert_eq!(none_ops.private_matrix_size(), 0);
2396        assert_eq!(none_ops.shared_matrix_size(), 0);
2397    }
2398
2399    #[test]
2400    fn ops_split_matches_schbench_integer_division() {
2401        // schbench.c:1391-1392: ops_private = operations*split/100 (integer
2402        // truncation), ops_shared = operations - ops_private. Returns
2403        // (ops_shared, ops_private).
2404        assert_eq!(ops_split(5, 0), (5, 0)); // all shared
2405        assert_eq!(ops_split(5, 25), (4, 1)); // 5*25/100 = 1 private
2406        assert_eq!(ops_split(5, 50), (3, 2)); // 5*50/100 = 2 private
2407        assert_eq!(ops_split(5, 100), (0, 5)); // all private
2408    }
2409
2410    #[test]
2411    fn engine_split_runs_shared_matrix_concurrently() {
2412        // --split with two workers: both concurrently multiply into the ONE
2413        // process-global shared AtomicU64 matrix (per-k stores to the shared C
2414        // cells -- the deliberate cache contention schbench models). Reaching the
2415        // assertion proves the shared-matrix path
2416        // neither deadlocks nor trips a data-race trap -- the atomics make the
2417        // shared race sound (vs schbench's plain shared-memory race). operations=8
2418        // @ split=50 => 4 shared + 4 private ops/cycle, so both paths run. Stop is
2419        // event-driven (spin on the progress counter, not a sleep).
2420        let config = SchbenchConfig {
2421            message_threads: 1,
2422            worker_threads: 2,
2423            cache_footprint_kib: 16,
2424            operations: 8,
2425            sleep_usec: 0,
2426            skip_locking: false,
2427            requests_per_sec: 0,
2428            auto_rps: 0,
2429            split_percent: Some(50),
2430            pipe_transfer_bytes: 0,
2431        };
2432        // Sanity: both split matrices are non-empty at this footprint, so the
2433        // test exercises shared + private work, not a degenerate 0-dim path.
2434        assert!(config.shared_matrix_size() > 0, "shared matrix present");
2435        assert!(config.private_matrix_size() > 0, "private matrix present");
2436        let stop = AtomicBool::new(false);
2437        let progress = AtomicU64::new(0);
2438        let outcome = std::thread::scope(|s| {
2439            let runner = s.spawn(|| run(&config, &stop, &progress, None));
2440            while progress.load(Ordering::Relaxed) < 50 {
2441                core::hint::spin_loop();
2442            }
2443            stop.store(true, Ordering::Release);
2444            runner.join().expect("run panicked")
2445        });
2446        assert!(
2447            outcome.whole_run.loop_count >= 50,
2448            "engine did split work: {}",
2449            outcome.whole_run.loop_count
2450        );
2451    }
2452
2453    #[test]
2454    fn pipe_fill_writes_every_byte_and_sizes_the_page() {
2455        // pipe_fill is the per-cycle memory transfer (schbench's pipe_page memset,
2456        // schbench.c:980-981/:1003-1004): it must touch EVERY byte of a
2457        // pipe_bytes-sized page. The bytes are never read back, so a dead-store
2458        // elision would silently zero the workload -- the black_box in pipe_fill
2459        // prevents that; this pins the observable effect (every byte written).
2460        let td = ThreadData::new(64);
2461        assert_eq!(td.pipe_bytes, 64);
2462        // SAFETY: single-threaded test, exclusive access to this td's page.
2463        unsafe { td.pipe_fill(2) };
2464        // SAFETY: same thread; no concurrent access.
2465        assert_eq!(
2466            unsafe { &*td.pipe_page.get() }.len(),
2467            64,
2468            "page sized to pipe_bytes"
2469        );
2470        assert!(
2471            unsafe { &*td.pipe_page.get() }.iter().all(|&b| b == 2),
2472            "worker fill (2) touches every byte"
2473        );
2474        // SAFETY: exclusive.
2475        unsafe { td.pipe_fill(1) };
2476        assert!(
2477            unsafe { &*td.pipe_page.get() }.iter().all(|&b| b == 1),
2478            "waker fill (1) overwrites every byte"
2479        );
2480        // Non-pipe ThreadData (pipe_bytes == 0) allocates an empty page: no transfer.
2481        let none = ThreadData::new(0);
2482        assert_eq!(none.pipe_bytes, 0);
2483        // SAFETY: exclusive.
2484        assert_eq!(unsafe { &*none.pipe_page.get() }.len(), 0);
2485    }
2486
2487    #[test]
2488    fn pipe_transfer_bytes_clamps_to_one_mib() {
2489        // schbench clamps -p to PIPE_TRANSFER_BUFFER (schbench.c:41,291-294); run()
2490        // applies the same `.min()` before sizing each worker's page.
2491        assert_eq!(PIPE_TRANSFER_BUFFER, 1024 * 1024);
2492        assert_eq!(
2493            (2 * PIPE_TRANSFER_BUFFER).min(PIPE_TRANSFER_BUFFER),
2494            PIPE_TRANSFER_BUFFER,
2495            "over-cap pipe size clamps to the 1 MiB buffer"
2496        );
2497        // At/under-cap values pass through unchanged.
2498        assert_eq!(4096_usize.min(PIPE_TRANSFER_BUFFER), 4096);
2499    }
2500
2501    #[test]
2502    fn pipe_transfer_report_is_per_worker_scales_and_clamps() {
2503        // PER-WORKER: ops/sec = achieved_rps / nr_workers
2504        // (schbench.c:1942-1943 divide by Σ worker runtimes ≈ nr_workers*elapsed).
2505        // 2048 aggregate cycles/sec / 2 workers = 1024 per-worker ops/sec.
2506        let r = pipe_transfer_report(2048.0, 1, 2);
2507        assert_eq!(r.ops_per_sec, 1024.0);
2508        // pretty_size (schbench.c:1606-1620): 1024 per-worker B/s -> 1.00 KB/s.
2509        assert_eq!((r.scaled, r.unit), (1.0, "KB"));
2510        // 1 worker, 1 B/cycle @ 1 cycle/s -> 1.00 B/s.
2511        let one = pipe_transfer_report(1.0, 1, 1);
2512        assert_eq!((one.ops_per_sec, one.scaled, one.unit), (1.0, 1.0, "B"));
2513        // 1 MiB/cycle @ 1 cycle/s/worker -> 1.00 MB/s.
2514        assert_eq!(pipe_transfer_report(1.0, 1 << 20, 1).unit, "MB");
2515        // CLAMP: an over-cap pipe size reports the clamped 1 MiB, not
2516        // the requested 2 MiB -- the engine moves only the clamped size per cycle
2517        // (schbench clamps at parse, schbench.c:291-294).
2518        let over = pipe_transfer_report(1.0, 2 << 20, 1);
2519        let cap = pipe_transfer_report(1.0, 1 << 20, 1);
2520        assert_eq!(
2521            (over.scaled, over.unit),
2522            (cap.scaled, cap.unit),
2523            "over-cap pipe size clamps to 1 MiB"
2524        );
2525        // EB cap: a huge byte rate never scales past the last unit (no overflow).
2526        assert_eq!(pipe_transfer_report(f64::MAX, 1 << 20, 1).unit, "EB");
2527        // nr_workers 0 -> floored at 1, never a division by zero.
2528        assert_eq!(pipe_transfer_report(100.0, 4096, 0).ops_per_sec, 100.0);
2529    }
2530
2531    #[test]
2532    fn engine_pipe_mode_transfers_without_matrix_work() {
2533        // Pipe mode (-p): the worker loop skips do_work + the think-sleep, and the
2534        // per-cycle work is the cross-thread pipe_page handshake -- the worker
2535        // memsets its page to 2 before blocking, the waker memsets it to 1 on wake
2536        // (schbench.c:980-981/:1003-1004). Two workers exercise the concurrent
2537        // waker->worker page writes. Reaching the assertions proves the handshake
2538        // runs to completion: no deadlock and no data-race trap on the cross-thread
2539        // `pipe_page` UnsafeCell (the wait-list + futex ordering keeps it race-free,
2540        // see the Sync SAFETY note). `operations`/`cache_footprint_kib` are set
2541        // LARGE on purpose: if do_work erroneously ran, every cycle would log a
2542        // multi-us request-latency sample -- the second assertion would then fail.
2543        // Stop is event-driven (spin on the progress counter, not a sleep).
2544        let config = SchbenchConfig::default()
2545            .message_threads(1)
2546            .worker_threads(2)
2547            .sleep_usec(0)
2548            .operations(50)
2549            .cache_footprint_kib(256)
2550            .pipe_transfer_bytes(4096);
2551        assert!(config.pipe_transfer_bytes > 0, "pipe mode engaged");
2552        assert!(
2553            config.matrix_size() > 0,
2554            "matrix work would be non-trivial if do_work ran (regression guard)"
2555        );
2556        let stop = AtomicBool::new(false);
2557        let progress = AtomicU64::new(0);
2558        let outcome = std::thread::scope(|s| {
2559            let runner = s.spawn(|| run(&config, &stop, &progress, None));
2560            while progress.load(Ordering::Relaxed) < 50 {
2561                core::hint::spin_loop();
2562            }
2563            stop.store(true, Ordering::Release);
2564            runner.join().expect("run panicked")
2565        });
2566        // loop_count accrues one per completed transfer cycle (schbench.c:1479),
2567        // proving the pipe handshake + memsets ran end-to-end without deadlock.
2568        assert!(
2569            outcome.whole_run.loop_count >= 50,
2570            "pipe engine transferred {} cycles",
2571            outcome.whole_run.loop_count
2572        );
2573        // Discriminating assertion: do_work is SKIPPED, so the
2574        // back-to-back work_start->now span yields sub-us request deltas that the
2575        // `delta_us > 0` filter drops -- request stats stay ~empty. If matrix work
2576        // erroneously ran at operations=50/256KiB, nearly every cycle would record a
2577        // >0 request delta and request samples would approach loop_count, failing
2578        // this. (A rare us-straddle may record a handful; `* 4 < loop_count` tolerates
2579        // that while still catching a real do_work regression.)
2580        assert!(
2581            outcome.whole_run.request.nr_samples * 4 < outcome.whole_run.loop_count,
2582            "pipe mode skips matrix work: {} request samples vs {} cycles",
2583            outcome.whole_run.request.nr_samples,
2584            outcome.whole_run.loop_count
2585        );
2586    }
2587
2588    #[test]
2589    #[should_panic(expected = "split_percent must be in 0..=100")]
2590    fn engine_panics_on_out_of_range_split() {
2591        // Regression pin for the out-of-range finding: a `pub split_percent` set
2592        // past 100 via STRUCT LITERAL (the path that bypasses the builder's
2593        // debug_assert) must fail LOUDLY at the run() consumption boundary in
2594        // every build profile -- the analog of schbench exiting on a bad
2595        // `--split` -- never silently underflow `100 - p` into a garbage-huge
2596        // shared matrix. The assert fires before any thread spawn or allocation,
2597        // so this is cheap (`stop` is pre-set just in case).
2598        let config = SchbenchConfig {
2599            message_threads: 1,
2600            worker_threads: 1,
2601            cache_footprint_kib: 16,
2602            operations: 1,
2603            sleep_usec: 0,
2604            skip_locking: false,
2605            requests_per_sec: 0,
2606            auto_rps: 0,
2607            split_percent: Some(101),
2608            pipe_transfer_bytes: 0,
2609        };
2610        let stop = AtomicBool::new(true);
2611        let progress = AtomicU64::new(0);
2612        let _ = run(&config, &stop, &progress, None);
2613    }
2614
2615    #[test]
2616    fn stack_add_splice_is_lifo() {
2617        let a = TestNode::new();
2618        let b = TestNode::new();
2619        let stack: TreiberStack<TestNode> = TreiberStack::new();
2620        assert!(stack.splice().is_null(), "empty stack splices to null");
2621        stack.add(&a as *const _ as *mut _);
2622        stack.add(&b as *const _ as *mut _);
2623        // LIFO: b (pushed last) is the head; a follows.
2624        let head = stack.splice();
2625        assert_eq!(head.cast_const(), &b as *const TestNode);
2626        // SAFETY: head -> b (alive on stack); its link -> a.
2627        let second = unsafe { (*head).next.load(Ordering::Acquire) };
2628        assert_eq!(second.cast_const(), &a as *const TestNode);
2629        // SAFETY: second -> a; its link is null.
2630        assert!(unsafe { (*second).next.load(Ordering::Acquire) }.is_null());
2631        assert!(stack.splice().is_null(), "splice emptied the stack");
2632    }
2633
2634    #[test]
2635    fn stack_concurrent_add_loses_no_nodes() {
2636        // N threads each concurrently push K distinct nodes; after all join, a
2637        // single splice must return exactly N*K nodes. A broken CAS (lost
2638        // update / ABA) would drop nodes and the count would be short. Push is
2639        // deterministic (each node pushed once), so this never hangs.
2640        const THREADS: usize = 8;
2641        const PER_THREAD: usize = 2000;
2642        let nodes: Vec<TestNode> = (0..THREADS * PER_THREAD).map(|_| TestNode::new()).collect();
2643        let stack = TreiberStack::new();
2644
2645        std::thread::scope(|s| {
2646            for chunk in nodes.chunks(PER_THREAD) {
2647                let stack = &stack;
2648                s.spawn(move || {
2649                    for n in chunk {
2650                        stack.add(n as *const TestNode as *mut TestNode);
2651                    }
2652                });
2653            }
2654        });
2655
2656        // All pushes complete (scope joined). Drain and count distinct nodes.
2657        let mut seen = std::collections::HashSet::new();
2658        let mut cur = stack.splice();
2659        while !cur.is_null() {
2660            assert!(seen.insert(cur), "node observed twice");
2661            // SAFETY: cur is one of the live `nodes`; next is its link.
2662            cur = unsafe { (*cur).next.load(Ordering::Acquire) };
2663        }
2664        assert_eq!(
2665            seen.len(),
2666            THREADS * PER_THREAD,
2667            "no node lost under contention"
2668        );
2669    }
2670
2671    #[test]
2672    fn engine_runs_and_produces_latency_samples() {
2673        // Small topology; run until ~50 work cycles complete, then stop. The
2674        // test completing proves the shutdown does not hang (a scope-join
2675        // shutdown bug would block here until the nextest timeout). Stop is
2676        // event-driven: spin on the shared progress counter, not a sleep.
2677        let config = SchbenchConfig {
2678            message_threads: 1,
2679            worker_threads: 2,
2680            cache_footprint_kib: 16,
2681            operations: 1,
2682            sleep_usec: 0,
2683            skip_locking: false,
2684            requests_per_sec: 0,
2685            auto_rps: 0,
2686            split_percent: None,
2687            pipe_transfer_bytes: 0,
2688        };
2689        let stop = AtomicBool::new(false);
2690        let progress = AtomicU64::new(0);
2691        let outcome = std::thread::scope(|s| {
2692            let runner = s.spawn(|| run(&config, &stop, &progress, None));
2693            while progress.load(Ordering::Relaxed) < 50 {
2694                core::hint::spin_loop();
2695            }
2696            stop.store(true, Ordering::Release);
2697            runner.join().expect("run panicked")
2698        });
2699        let result = &outcome.whole_run;
2700        assert!(
2701            result.loop_count >= 50,
2702            "engine did work: {}",
2703            result.loop_count
2704        );
2705        assert!(result.wakeup.nr_samples > 0, "wakeup samples recorded");
2706        assert!(result.request.nr_samples > 0, "request samples recorded");
2707        assert!(result.achieved_rps > 0.0, "positive achieved rps");
2708        // Non-phasic (phase_epoch None): the only snapshot is the baseline
2709        // epoch 0, which the host discards. No per-phase metrics are emitted.
2710        assert_eq!(
2711            outcome.phases.len(),
2712            1,
2713            "non-phasic => single baseline phase"
2714        );
2715        assert_eq!(outcome.phases[0].0, 0, "the lone phase is BASELINE epoch 0");
2716    }
2717
2718    #[test]
2719    fn engine_terminates_when_lone_worker_stops() {
2720        // Regression for the shutdown deadlock: with a SINGLE worker there is no
2721        // second worker to post the message thread, so a worker that exits on
2722        // `stop` while the message thread is parked in `wait_forever` must wake it
2723        // (the unconditional post at worker_loop's exit), or run() hangs forever.
2724        // Reaching the assertion at all proves run() returned. `sleep_usec` 0
2725        // keeps the lone worker almost always mid-`do_work` when stop fires (the
2726        // deadlock-prone window). engine_runs / engine_splits use 2 workers and
2727        // missed this -- a second worker posting the message thread masks the bug;
2728        // a hang here is the nextest timeout, exactly how the bug first surfaced.
2729        let config = SchbenchConfig {
2730            message_threads: 1,
2731            worker_threads: 1,
2732            cache_footprint_kib: 256,
2733            operations: 5,
2734            sleep_usec: 0,
2735            skip_locking: false,
2736            requests_per_sec: 0,
2737            auto_rps: 0,
2738            split_percent: None,
2739            pipe_transfer_bytes: 0,
2740        };
2741        let stop = AtomicBool::new(false);
2742        let progress = AtomicU64::new(0);
2743        let outcome = std::thread::scope(|s| {
2744            let runner = s.spawn(|| run(&config, &stop, &progress, None));
2745            while progress.load(Ordering::Relaxed) < 10 {
2746                core::hint::spin_loop();
2747            }
2748            stop.store(true, Ordering::Release);
2749            // Deadlocks here on regression: the lone worker exits without waking
2750            // the parked message thread, so this join never returns.
2751            runner.join().expect("run panicked")
2752        });
2753        assert!(
2754            outcome.whole_run.loop_count >= 10,
2755            "engine did work and returned: {}",
2756            outcome.whole_run.loop_count
2757        );
2758    }
2759
2760    #[test]
2761    fn engine_rps_mode_injects_drains_and_terminates() {
2762        // RPS-injector mode (requests_per_sec != 0): a dedicated thread enqueues
2763        // requests round-robin across the workers, which splice + drain their
2764        // queues instead of the message-thread handshake. Verify the RPS path
2765        // runs (loop_count + request samples) and terminates cleanly -- run()
2766        // returns (no deadlock: the RPS thread is the waker, the worker the
2767        // waitee, and shutdown wakes the workers via run_one_message_thread's
2768        // worker-wake + the RPS thread's stop-time fpost-all; a hang is the
2769        // nextest timeout) with no request leak (the post-join free).
2770        let config = SchbenchConfig {
2771            message_threads: 1,
2772            worker_threads: 2,
2773            cache_footprint_kib: 16,
2774            operations: 1,
2775            sleep_usec: 0,
2776            skip_locking: false,
2777            requests_per_sec: 10_000,
2778            auto_rps: 0,
2779            split_percent: None,
2780            pipe_transfer_bytes: 0,
2781        };
2782        let stop = AtomicBool::new(false);
2783        let progress = AtomicU64::new(0);
2784        let outcome = std::thread::scope(|s| {
2785            let runner = s.spawn(|| run(&config, &stop, &progress, None));
2786            while progress.load(Ordering::Relaxed) < 50 {
2787                core::hint::spin_loop();
2788            }
2789            stop.store(true, Ordering::Release);
2790            // Deadlocks here on regression: a worker parked in rps_wait that the
2791            // shutdown path fails to wake, or the RPS thread failing to observe
2792            // stop, would hang this join.
2793            runner.join().expect("run panicked")
2794        });
2795        let result = &outcome.whole_run;
2796        assert!(
2797            result.loop_count >= 50,
2798            "RPS workers serviced requests: {}",
2799            result.loop_count
2800        );
2801        assert!(
2802            result.request.nr_samples > 0,
2803            "RPS request-latency samples recorded"
2804        );
2805    }
2806
2807    #[test]
2808    fn rps_injector_pacing_sleep_is_bounded_for_prompt_shutdown() {
2809        // requests_per_sec=1, message_threads=1 => per-thread inter-arrival
2810        // interval ~1s. A stop set while the injector is in that paced sleep must be
2811        // observed within STOP_POLL_QUANTUM_NS (50 ms), so run()'s message-thread
2812        // join completes well under one interval. An unbounded pacing sleep would
2813        // block the join ~1s. 500 ms is 10x the quantum (robust to scheduling
2814        // jitter) and half the interval (cleanly distinguishes bounded from
2815        // unbounded). The 20 ms warmup reaches the long paced sleep before stop (at
2816        // 1 req/s a progress-spin could outlast the test).
2817        let config = SchbenchConfig {
2818            message_threads: 1,
2819            worker_threads: 1,
2820            cache_footprint_kib: 16,
2821            operations: 1,
2822            sleep_usec: 0,
2823            skip_locking: false,
2824            requests_per_sec: 1, // interval = 1s / 1 = 1s
2825            auto_rps: 0,
2826            split_percent: None,
2827            pipe_transfer_bytes: 0,
2828        };
2829        let stop = AtomicBool::new(false);
2830        let progress = AtomicU64::new(0);
2831        let start = std::time::Instant::now();
2832        std::thread::scope(|s| {
2833            let runner = s.spawn(|| run(&config, &stop, &progress, None));
2834            std::thread::sleep(std::time::Duration::from_millis(20));
2835            stop.store(true, Ordering::Release);
2836            let _ = runner.join().expect("run panicked");
2837        });
2838        let elapsed = start.elapsed();
2839        assert!(
2840            elapsed < std::time::Duration::from_millis(500),
2841            "RPS injector joined in {elapsed:?}; the paced sleep must be bounded \
2842             (interval ~1s — an unbounded sleep would block the join that long)",
2843        );
2844    }
2845
2846    #[test]
2847    fn engine_rps_below_message_threads_falls_to_default() {
2848        // Regression: when the -R total is below the message-thread count,
2849        // the per-thread rate (rps_per_message_thread) rounds to 0, and the
2850        // engine must run the DEFAULT message-handshake mode -- workers do real
2851        // wake-cycle work -- matching schbench's startup-divide-before-gate. NOT
2852        // a zero-rate RPS mode (which would leave the workers parked with no
2853        // requests, never advancing progress -- the spin below would then hang,
2854        // caught by the nextest timeout).
2855        let config = SchbenchConfig {
2856            message_threads: 2,
2857            worker_threads: 1,
2858            cache_footprint_kib: 16,
2859            operations: 1,
2860            sleep_usec: 0,
2861            skip_locking: false,
2862            requests_per_sec: 1, // 1 / 2 message threads = 0 per thread -> default
2863            auto_rps: 0,
2864            split_percent: None,
2865            pipe_transfer_bytes: 0,
2866        };
2867        let stop = AtomicBool::new(false);
2868        let progress = AtomicU64::new(0);
2869        let outcome = std::thread::scope(|s| {
2870            let runner = s.spawn(|| run(&config, &stop, &progress, None));
2871            while progress.load(Ordering::Relaxed) < 50 {
2872                core::hint::spin_loop();
2873            }
2874            stop.store(true, Ordering::Release);
2875            runner.join().expect("run panicked")
2876        });
2877        assert!(
2878            outcome.whole_run.loop_count >= 50,
2879            "sub-1-per-thread RPS ran default-mode work: {}",
2880            outcome.whole_run.loop_count
2881        );
2882    }
2883
2884    #[test]
2885    fn auto_scale_grows_when_idle_and_shrinks_when_busy() {
2886        // Pure scaling math (no /proc/stat I/O) at injected busy levels, matching
2887        // schbench's auto_scale_rps body (`schbench.c:1203-1250`).
2888
2889        // Far under target: delta = 80/10 = 8, capped to 3 -> ceil(100*3) = 300.
2890        // Not a near-target transition, so just_hit is false (the >3 cap branch
2891        // never trips target_hit).
2892        let rate = AtomicUsize::new(100);
2893        let hit = AtomicBool::new(false);
2894        assert!(!scale_rps_for_busy(10.0, &rate, &hit, 80));
2895        assert_eq!(rate.load(Ordering::Acquire), 300);
2896        assert!(!hit.load(Ordering::Acquire));
2897
2898        // Far over target: delta = 20/100 = 0.2, clamped to 0.3 -> floor(300*0.3)
2899        // = 90. Also not a near-target transition.
2900        let rate = AtomicUsize::new(300);
2901        let hit = AtomicBool::new(false);
2902        assert!(!scale_rps_for_busy(100.0, &rate, &hit, 20));
2903        assert_eq!(rate.load(Ordering::Acquire), 90);
2904        assert!(!hit.load(Ordering::Acquire));
2905
2906        // Exactly on target: rate held, target_hit set on the first arrival
2907        // (`schbench.c:1243-1248`).
2908        let rate = AtomicUsize::new(123);
2909        let hit = AtomicBool::new(false);
2910        assert!(scale_rps_for_busy(50.0, &rate, &hit, 50));
2911        assert_eq!(rate.load(Ordering::Acquire), 123);
2912        assert!(hit.load(Ordering::Acquire));
2913    }
2914
2915    #[test]
2916    fn auto_scale_target_hit_uses_post_damping_delta() {
2917        // The near-target test reads the delta AFTER its damping reassignment
2918        // (`schbench.c:1209-1213` grow, `:1232-1236` shrink) -- the bug the
2919        // pre-reassignment check had. These cases trip target_hit ONLY under the
2920        // correct post-damping order.
2921
2922        // Grow band: busy 90, target 100 -> delta 1.111 (NOT < 1.05), damped to
2923        // 1 + 0.111/8 = 1.0139 (< 1.05) -> target_hit. A pre-damping check on
2924        // 1.111 would miss it.
2925        let rate = AtomicUsize::new(1000);
2926        let hit = AtomicBool::new(false);
2927        assert!(scale_rps_for_busy(90.0, &rate, &hit, 100));
2928        assert!(hit.load(Ordering::Acquire));
2929
2930        // Shrink band: busy 100, target 95 -> delta 0.95 (NOT > 0.95), damped to
2931        // 0.95 + 0.05/8 = 0.95625 (> 0.95) -> target_hit. A pre-damping check on
2932        // 0.95 would miss it.
2933        let rate = AtomicUsize::new(1000);
2934        let hit = AtomicBool::new(false);
2935        assert!(scale_rps_for_busy(100.0, &rate, &hit, 95));
2936        assert!(hit.load(Ordering::Acquire));
2937
2938        // Already hit: no second transition is reported (just_hit guards on
2939        // !already_hit), even on an exact-target tick.
2940        let rate = AtomicUsize::new(1000);
2941        let hit = AtomicBool::new(true);
2942        assert!(!scale_rps_for_busy(50.0, &rate, &hit, 50));
2943    }
2944
2945    #[test]
2946    fn reset_rps_accumulators_clears_both_in_lockstep() {
2947        // The auto-RPS target-hit reset MUST clear the whole-run AND per-epoch rps
2948        // accumulators together; if a future edit drops one, Σ-per-epoch != whole-run
2949        // (asymmetric-counter-bookkeeping). Pins the lockstep deterministically — the
2950        // engine just_hit path is timing-dependent (the controller must reach target).
2951        let mut whole = PlatStats::default();
2952        whole.add_lat(500);
2953        let mut per_epoch: std::collections::BTreeMap<u32, PlatStats> =
2954            std::collections::BTreeMap::new();
2955        per_epoch.entry(1).or_default().add_lat(500);
2956        reset_rps_accumulators(&mut whole, &mut per_epoch);
2957        assert_eq!(whole.sample_count(), 0, "whole-run rps reset");
2958        assert!(per_epoch.is_empty(), "per-epoch rps reset in lockstep");
2959    }
2960
2961    #[test]
2962    fn control_loop_rps_sample_floors_sub_quantum_window() {
2963        // The bounded teardown sleep can return on `stop` with a tiny dt; a
2964        // count/dt rate is unbounded as dt->0, so a sub-quantum window is dropped
2965        // rather than spiking rps_max / the top rps percentiles (assertable
2966        // metrics). Pins the floor deterministically -- the race that produces the
2967        // tiny dt is timing-dependent, but the math + floor it feeds is pure.
2968        // dt==0 is dropped, not a divide-by-zero:
2969        assert_eq!(control_loop_rps_sample(3, 0), None);
2970        // A 1us / one-completion window would read as ~1e6 cycles/sec -- dropped:
2971        assert_eq!(control_loop_rps_sample(1, 1_000), None);
2972        // Just under one quantum is still dropped:
2973        assert_eq!(control_loop_rps_sample(5, STOP_POLL_QUANTUM_NS - 1), None);
2974        // Exactly one quantum is admitted; the rate is bounded to
2975        // delta_loops * 1e9 / quantum, no inflation: 1 cycle / 50ms = 20/s.
2976        assert_eq!(control_loop_rps_sample(1, STOP_POLL_QUANTUM_NS), Some(20));
2977        // Even a burst of 1000 completions in one quantum caps at 1000*20 =
2978        // 20_000/s -- not the millions an unfloored 1us window would yield.
2979        assert_eq!(
2980            control_loop_rps_sample(1_000, STOP_POLL_QUANTUM_NS),
2981            Some(20_000)
2982        );
2983        // The common case -- a full ~1s window -- is unchanged: 5000/s.
2984        assert_eq!(control_loop_rps_sample(5_000, 1_000_000_000), Some(5_000));
2985    }
2986
2987    #[test]
2988    fn engine_auto_rps_mode_runs_and_terminates() {
2989        // auto_rps != 0 turns on the once-per-second control thread that
2990        // auto-scales the live rate toward the busy target. Seeded with an
2991        // explicit -R so the injector services requests immediately (progress
2992        // hits 50 well before the first 1s control tick), this proves the
2993        // auto-RPS path boots, injects, and run() terminates cleanly (the control
2994        // thread observes stop; a hang is the nextest timeout). The scaling MATH
2995        // is pinned by the scale_rps_for_busy unit tests; host-busy + target-hit
2996        // timing is environment-dependent, so there is no sample-count assertion.
2997        let config = SchbenchConfig {
2998            message_threads: 1,
2999            worker_threads: 2,
3000            cache_footprint_kib: 16,
3001            operations: 1,
3002            sleep_usec: 0,
3003            skip_locking: false,
3004            requests_per_sec: 10_000,
3005            auto_rps: 50,
3006            split_percent: None,
3007            pipe_transfer_bytes: 0,
3008        };
3009        let stop = AtomicBool::new(false);
3010        let progress = AtomicU64::new(0);
3011        let outcome = std::thread::scope(|s| {
3012            let runner = s.spawn(|| run(&config, &stop, &progress, None));
3013            while progress.load(Ordering::Relaxed) < 50 {
3014                core::hint::spin_loop();
3015            }
3016            stop.store(true, Ordering::Release);
3017            runner.join().expect("run panicked")
3018        });
3019        assert!(
3020            outcome.whole_run.loop_count >= 50,
3021            "auto-RPS injector serviced requests: {}",
3022            outcome.whole_run.loop_count
3023        );
3024    }
3025
3026    #[test]
3027    fn engine_splits_stats_across_phase_epochs() {
3028        // Drive the shared phase_epoch 1 -> 2 mid-run and confirm the engine
3029        // partitions its histograms + loop_count into per-epoch snapshots
3030        // (the scx-phase vs detached-EEVDF-phase mechanism). Event-driven via
3031        // the progress counter, no sleeps.
3032        let config = SchbenchConfig {
3033            message_threads: 1,
3034            worker_threads: 2,
3035            cache_footprint_kib: 16,
3036            operations: 1,
3037            sleep_usec: 0,
3038            skip_locking: false,
3039            requests_per_sec: 0,
3040            auto_rps: 0,
3041            split_percent: None,
3042            pipe_transfer_bytes: 0,
3043        };
3044        let stop = AtomicBool::new(false);
3045        let progress = AtomicU64::new(0);
3046        let epoch = AtomicU32::new(1); // start in a real step epoch (phase 1)
3047        let outcome = std::thread::scope(|s| {
3048            let runner = s.spawn(|| run(&config, &stop, &progress, Some(&epoch)));
3049            while progress.load(Ordering::Relaxed) < 50 {
3050                core::hint::spin_loop();
3051            }
3052            let after_phase1 = progress.load(Ordering::Relaxed);
3053            epoch.store(2, Ordering::Release); // transition to phase 2
3054            while progress.load(Ordering::Relaxed) < after_phase1 + 50 {
3055                core::hint::spin_loop();
3056            }
3057            stop.store(true, Ordering::Release);
3058            runner.join().expect("run panicked")
3059        });
3060
3061        let by_epoch: std::collections::BTreeMap<u32, &SchbenchPhaseStats> =
3062            outcome.phases.iter().map(|(e, s)| (*e, s)).collect();
3063        let p1 = by_epoch.get(&1).expect("phase 1 present");
3064        let p2 = by_epoch.get(&2).expect("phase 2 present");
3065        // Both phases ran work cycles (we waited for >=50 cycles in each
3066        // window). loop_count is the robust split proof: latency samples gate
3067        // on delta_us>0, which a sub-µs cycle can miss, but a cycle always
3068        // increments loop_count.
3069        assert!(p1.loop_count > 0, "phase 1 ran cycles: {}", p1.loop_count);
3070        assert!(p2.loop_count > 0, "phase 2 ran cycles: {}", p2.loop_count);
3071        // Per-phase run-delay split is populated per thread class. This guards a
3072        // DISTINCT path the loop_count/histogram asserts don't touch: the
3073        // schedstat raw-pair baseline/re-baseline (phase_ss_start) + the
3074        // per-class Σ fold (worker vs msg). pcount is the deterministic
3075        // denominator — any thread scheduled in the phase accrues pcount>=1
3076        // (sched_info_arrive), and both phases ran cycles (workers dispatched)
3077        // while the msg thread is scheduled on every wake. run_delay_ns itself
3078        // is NOT asserted: it can be 0 on an uncontended host, so a value check
3079        // would flake; pcount>0 pins that the class was measured and folded.
3080        assert!(
3081            p1.worker_pcount > 0,
3082            "phase 1 worker run-delay split populated"
3083        );
3084        assert!(
3085            p2.worker_pcount > 0,
3086            "phase 2 worker run-delay split populated"
3087        );
3088        assert!(p1.msg_pcount > 0, "phase 1 msg run-delay split populated");
3089        assert!(p2.msg_pcount > 0, "phase 2 msg run-delay split populated");
3090        // loop_count partitions across phases, summing to the global progress —
3091        // no cycle is lost or double-counted at the boundary.
3092        let loop_sum: u64 = outcome.phases.iter().map(|(_, s)| s.loop_count).sum();
3093        assert_eq!(
3094            loop_sum, outcome.whole_run.loop_count,
3095            "per-phase loop_count partitions the whole-run count"
3096        );
3097        // The whole-run histogram is EXACTLY the union of the per-phase ones
3098        // (the drained snapshots, recombined), so the per-phase split loses no
3099        // samples vs the whole-run accounting.
3100        let phase_request_sum: u64 = outcome
3101            .phases
3102            .iter()
3103            .map(|(_, s)| s.request.percentiles().nr_samples)
3104            .sum();
3105        assert_eq!(
3106            outcome.whole_run.request.nr_samples, phase_request_sum,
3107            "whole-run request count == Σ per-phase counts"
3108        );
3109        let phase_wakeup_sum: u64 = outcome
3110            .phases
3111            .iter()
3112            .map(|(_, s)| s.wakeup.percentiles().nr_samples)
3113            .sum();
3114        assert_eq!(
3115            outcome.whole_run.wakeup.nr_samples, phase_wakeup_sum,
3116            "whole-run wakeup count == Σ per-phase counts"
3117        );
3118        // Per-phase RPS: the additive invariant (deterministic) + a timing-tolerant
3119        // bound on the brief first phase. These phases run far under the 1s control
3120        // cadence, so phase 1 normally gets ZERO rps ticks (the ~1s tick lands in
3121        // phase 2, epoch already advanced); on a slow/contended host phase 1 may catch
3122        // a single straddle tick, so we tolerate <= 1 rather than a wall-clock-fragile
3123        // == 0. The deterministic empty-rps -> ABSENT contract is pinned by
3124        // derive_rps_distribution_values_and_absent_guard; this bound still catches
3125        // gross mis-attribution of the bulk of rps to the brief first phase.
3126        assert!(
3127            p1.rps.sample_count() <= 1,
3128            "brief phase 1 gets at most one straddle rps tick, got {}",
3129            p1.rps.sample_count()
3130        );
3131        let phase_rps_sum: u64 = outcome
3132            .phases
3133            .iter()
3134            .map(|(_, s)| s.rps.sample_count())
3135            .sum();
3136        assert_eq!(
3137            outcome.whole_run.rps.nr_samples, phase_rps_sum,
3138            "whole-run rps count == Σ per-phase rps counts"
3139        );
3140    }
3141
3142    #[test]
3143    fn engine_per_phase_rps_populates_and_sums_to_whole_run() {
3144        // POPULATED per-phase RPS: the control thread samples once/sec, so a phase
3145        // window >= ~2s gets multiple per-phase rps samples. Time-gated (the 1s
3146        // control cadence is the granularity floor) -- the progress-gated
3147        // engine_splits phases are too short to tick. Drive epoch 1 for ~2.2s then
3148        // epoch 2 for ~2.2s and confirm BOTH epochs carry rps samples AND the
3149        // per-epoch samples sum to the whole-run count (the additive invariant:
3150        // every sample lands in one epoch bucket AND the whole-run histogram).
3151        // The ~4.4s wall-clock is the irreducible cost of exercising the real 1s control
3152        // cadence end-to-end (2 ticks/epoch for robustness against a delayed tick on a
3153        // loaded host); the deterministic per-epoch invariants — the lockstep reset and
3154        // Σ-per-epoch==whole-run — are pinned sleep-free by
3155        // reset_rps_accumulators_clears_both_in_lockstep + engine_splits_stats_across_phase_epochs.
3156        // This test uniquely confirms the LIVE control thread POPULATES both epochs.
3157        let config = SchbenchConfig {
3158            message_threads: 1,
3159            worker_threads: 2,
3160            cache_footprint_kib: 16,
3161            operations: 1,
3162            sleep_usec: 0,
3163            skip_locking: false,
3164            requests_per_sec: 0,
3165            auto_rps: 0,
3166            split_percent: None,
3167            pipe_transfer_bytes: 0,
3168        };
3169        let stop = AtomicBool::new(false);
3170        let progress = AtomicU64::new(0);
3171        let epoch = AtomicU32::new(1);
3172        let outcome = std::thread::scope(|s| {
3173            let runner = s.spawn(|| run(&config, &stop, &progress, Some(&epoch)));
3174            // ~2.2s per epoch => >= 2 control ticks each (first tick at ~1s).
3175            std::thread::sleep(std::time::Duration::from_millis(2200));
3176            epoch.store(2, Ordering::Release);
3177            std::thread::sleep(std::time::Duration::from_millis(2200));
3178            stop.store(true, Ordering::Release);
3179            runner.join().expect("run panicked")
3180        });
3181        let by_epoch: std::collections::BTreeMap<u32, &SchbenchPhaseStats> =
3182            outcome.phases.iter().map(|(e, s)| (*e, s)).collect();
3183        let p1 = by_epoch.get(&1).expect("phase 1 present");
3184        let p2 = by_epoch.get(&2).expect("phase 2 present");
3185        assert!(
3186            p1.rps.sample_count() > 0,
3187            "phase 1 per-phase rps populated: {}",
3188            p1.rps.sample_count()
3189        );
3190        assert!(
3191            p2.rps.sample_count() > 0,
3192            "phase 2 per-phase rps populated: {}",
3193            p2.rps.sample_count()
3194        );
3195        // Additive invariant (no auto-RPS reset in default mode, so EXACT): Σ
3196        // per-epoch rps samples == whole-run rps samples.
3197        let phase_rps_sum: u64 = outcome
3198            .phases
3199            .iter()
3200            .map(|(_, s)| s.rps.sample_count())
3201            .sum();
3202        assert_eq!(
3203            phase_rps_sum, outcome.whole_run.rps.nr_samples,
3204            "Σ per-epoch rps samples == whole-run rps samples"
3205        );
3206    }
3207
3208    #[test]
3209    fn schbench_config_serde_roundtrips() {
3210        // The new serialized type roundtrips unchanged.
3211        let cfg = SchbenchConfig::default()
3212            .message_threads(3)
3213            .worker_threads(7)
3214            .cache_footprint_kib(512)
3215            .operations(9)
3216            .sleep_usec(250)
3217            .skip_locking(true)
3218            // Exercise the new field's Some(p) serde surface, not just the None
3219            // default (the API-review roundtrip requirement).
3220            .split_percent(Some(33))
3221            // Exercise the pipe field's non-default serde surface too.
3222            .pipe_transfer_bytes(4096);
3223        let json = serde_json::to_string(&cfg).expect("SchbenchConfig must serialize");
3224        let back: SchbenchConfig =
3225            serde_json::from_str(&json).expect("SchbenchConfig must deserialize");
3226        assert_eq!(cfg, back, "config roundtrips unchanged");
3227    }
3228
3229    #[test]
3230    fn worktype_schbench_registration_and_serde() {
3231        use crate::workload::WorkType;
3232        let wt = WorkType::schbench(
3233            SchbenchConfig::default()
3234                .message_threads(2)
3235                .worker_threads(4),
3236        );
3237        assert_eq!(wt.name(), "Schbench");
3238        // from_name yields the default-config variant.
3239        assert_eq!(
3240            WorkType::from_name("Schbench"),
3241            Some(WorkType::Schbench {
3242                config: SchbenchConfig::default()
3243            })
3244        );
3245        // The variant serde-roundtrips, carrying its config.
3246        let json = serde_json::to_string(&wt).expect("WorkType::Schbench must serialize");
3247        let back: WorkType =
3248            serde_json::from_str(&json).expect("WorkType::Schbench must deserialize");
3249        assert_eq!(wt, back);
3250    }
3251
3252    #[test]
3253    fn schbench_config_reachable_via_prelude() {
3254        // Regression-pin the prelude placement: test authors construct the
3255        // config via `use ktstr::prelude::*`. Dropping SchbenchConfig from the
3256        // prelude would fail this compile. Also exercises the Eq derive.
3257        let cfg: crate::prelude::SchbenchConfig = crate::prelude::SchbenchConfig::default();
3258        assert_eq!(cfg, SchbenchConfig::default());
3259    }
3260
3261    #[test]
3262    fn read_schedstat_raw_parses_own_and_handles_missing() {
3263        // The current thread has been scheduled, so its /proc/<tid>/schedstat
3264        // parses without panic into a (run_delay, pcount) pair.
3265        let _own = read_schedstat_raw(gettid_self());
3266        // A non-existent tid -> (0,0) via the file-read-failure path (thread
3267        // exited), matching schbench's fopen-failure handling. The parse
3268        // boundaries (incl. the pcount==0 mean guard) are covered below.
3269        assert_eq!(
3270            read_schedstat_raw(-1),
3271            (0, 0),
3272            "absent schedstat yields (0,0), no panic"
3273        );
3274    }
3275
3276    #[test]
3277    fn schedstat_raw_parse_and_mean_pcount_guard() {
3278        // Raw parse keeps run_delay + pcount undivided (the re-poolable pair).
3279        assert_eq!(parse_schedstat_raw("123456 50 5"), (50, 5));
3280        // mean_sched_delay divides with the pcount==0 guard (no div-by-zero).
3281        assert_eq!(mean_sched_delay((50, 5)), 10); // run_delay/pcount
3282        assert_eq!(mean_sched_delay((50, 0)), 0); // pcount==0 guard
3283        assert_eq!(mean_sched_delay(parse_schedstat_raw("0 0 0")), 0); // !sched_info_on()
3284        assert_eq!(mean_sched_delay((0, 5)), 0); // 0 run_delay -> 0 mean
3285    }
3286
3287    #[test]
3288    #[should_panic(expected = "schedstat field 3")]
3289    fn parse_schedstat_raw_short_line_panics() {
3290        // A present-but-short line is a kernel/parse bug: fail loud, not a
3291        // silent 0 (the fail-loud ruling).
3292        parse_schedstat_raw("100 50");
3293    }
3294
3295    #[test]
3296    #[should_panic(expected = "schedstat field 2")]
3297    fn parse_schedstat_raw_nonnumeric_panics() {
3298        parse_schedstat_raw("alpha beta gamma");
3299    }
3300}