ktstr/workload/schbench/run.rs
1//! schbench's run engine, ported faithfully from `schbench.c`: the
2//! message-thread / worker-thread topology, the lockless wait-list, the
3//! handshake-driven wakeup-latency loop, and the matrix work under the
4//! per-CPU lock. The default (non-RPS) mode is the wakeup-latency +
5//! request-latency benchmark, with per-phase schedstat (run-delay) capture; the
6//! RPS-injector mode (`-R`) and its auto-RPS closed-loop rate control (`-A`)
7//! layer on the request queue + once-per-second control thread; pipe mode (`-p`)
8//! swaps the matrix work for a memory-transfer simulation over the same handshake.
9//!
10//! # Topology (`schbench.c` `message_thread` :1540, `worker_thread` :1419)
11//!
12//! `message_threads` message threads each spawn `worker_threads` worker
13//! threads in one process (schbench is single-process pthreads; schbench_rs
14//! re-expresses that with [`std::thread`], so the
15//! [`handshake`](super::handshake) futex is PRIVATE and the per-CPU locks +
16//! matrices live in one address space). A worker loops: block until its
17//! message thread wakes it (measuring wakeup latency), then think-sleep +
18//! matrix work under the per-CPU lock (measuring request latency). The message
19//! thread batch-wakes all waiting workers (`run_msg_thread` :1166 /
20//! `xlist_wake_all` :969).
21//!
22//! # Fidelity
23//!
24//! - Clock: `CLOCK_MONOTONIC` (ruling), not schbench's `gettimeofday`
25//! wall-clock. Monotonic is freeze-robust (a host-side VM pause cannot make
26//! a delta go negative) and is the correct source for a latency delta. The
27//! measured quantity -- elapsed nanoseconds between two reads -- is identical
28//! to what schbench measures; only the clock id differs.
29//! - The lockless wait-list is a Treiber stack ([`TreiberStack`]) over an
30//! intrusive next-pointer, matching schbench's `xlist_add`/`xlist_splice`
31//! cmpxchg list (`schbench.c:866-896`): userspace CAS only, no lock/syscall
32//! on the hot path, so the syscall profile stays futex-dominated like
33//! schbench's. (schbench hand-duplicates this for its thread list and its
34//! request list; schbench_rs shares one generic implementation.)
35//! - do_work timing: the matrix multiply compiles to the SAME scalar inner loop
36//! as schbench's `-O2` `do_some_math` -- a `u64` multiply-accumulate
37//! (`imul` with a fused memory operand, then `add`), NOT SIMD. It does not
38//! auto-vectorize: there is no vector `u64` multiply on the build target
39//! (`x86-64-v3`; `vpmullq` is AVX-512) and the serial accumulator reduction
40//! blocks SLP -- exactly as for schbench's `-O2` build (its Makefile
41//! `CFLAGS = -Wall -O2`, no `-march`, likewise scalar; verified by disassembly
42//! of both). A debug build (opt-level 0) emits far more overhead (no register
43//! allocation, unrolling, or memory-operand fusion), so request latency runs
44//! higher than reference schbench's. ABSOLUTE side-by-side fidelity vs the
45//! reference (the `ktstr-schbench-validate` comparison) therefore needs a
46//! release build. The in-ktstr A/B comparison (perf-delta / per-phase claims)
47//! is build-invariant -- both sides use the same build -- so it is unaffected.
48
49use core::cell::UnsafeCell;
50use core::ptr;
51use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, AtomicU64, AtomicUsize, Ordering};
52use std::fs::File;
53use std::io::{Read, Seek, SeekFrom};
54
55use super::percpu_lock::PerCpuLocks;
56use super::plat::{Percentiles, PlatStats};
57
58/// Read `CLOCK_MONOTONIC` as nanoseconds (ruling: monotonic, not wall-clock).
59/// Self-contained so the schbench modules build into the standalone validation
60/// binary without reaching into `worker`'s `pub(super)` clock wrapper.
61fn monotonic_nanos() -> u64 {
62 // SAFETY: `clock_gettime` writes a `timespec` through the out-pointer and
63 // reads nothing else; CLOCK_MONOTONIC is always available on Linux.
64 let mut ts: libc::timespec = unsafe { core::mem::zeroed() };
65 let rc = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut ts) };
66 assert_eq!(rc, 0, "clock_gettime(CLOCK_MONOTONIC) failed");
67 (ts.tv_sec as u64) * 1_000_000_000 + ts.tv_nsec as u64
68}
69
70/// A node that can be linked into a [`TreiberStack`] via an intrusive
71/// next-pointer it owns.
72trait Linked: Sized {
73 fn next_link(&self) -> &AtomicPtr<Self>;
74}
75
76/// Lockless LIFO stack via an intrusive next-pointer, the shared form of
77/// schbench's `xlist_add`/`xlist_splice` (`schbench.c:866-896`). Nodes are
78/// referenced by raw pointer and must outlive every concurrent operation; the
79/// caller owns their storage and lifetimes.
80struct TreiberStack<T: Linked> {
81 head: AtomicPtr<T>,
82}
83
84impl<T: Linked> TreiberStack<T> {
85 fn new() -> Self {
86 Self {
87 head: AtomicPtr::new(ptr::null_mut()),
88 }
89 }
90
91 /// Push `node` onto the stack. Faithful port of `xlist_add`
92 /// (`schbench.c:866`): set the node's link to the current head, then
93 /// CAS-publish it, retrying on contention. The caller must not push a node
94 /// that is already on the stack (its link would be overwritten while
95 /// reachable) -- schbench upholds this (a worker re-queues only after being
96 /// spliced off and woken).
97 fn add(&self, node: *mut T) {
98 // SAFETY: `node` is a valid pointer to a `T` that outlives all list
99 // operations (caller contract).
100 let link = unsafe { (*node).next_link() };
101 loop {
102 let old = self.head.load(Ordering::Acquire);
103 link.store(old, Ordering::Relaxed);
104 if self
105 .head
106 .compare_exchange(old, node, Ordering::AcqRel, Ordering::Acquire)
107 .is_ok()
108 {
109 return;
110 }
111 }
112 }
113
114 /// Atomically take the whole stack, leaving it empty, returning the head
115 /// (null if empty). Faithful port of `xlist_splice` (`schbench.c:884`): an
116 /// atomic swap to null (the single-op equivalent of schbench's CAS loop).
117 /// The returned chain is walked via [`Linked::next_link`] in LIFO order
118 /// (schbench's thread-list splice does not reverse; wake order is
119 /// irrelevant).
120 fn splice(&self) -> *mut T {
121 self.head.swap(ptr::null_mut(), Ordering::AcqRel)
122 }
123
124 /// Atomically take the whole stack and return the chain in FIFO (insertion)
125 /// order -- the reverse of [`splice`](Self::splice)'s LIFO. Faithful port of
126 /// `request_splice` (`schbench.c:920-940`): swap to null, then reverse the
127 /// spliced chain so the consumer services the oldest queued request first
128 /// (request order matters, unlike the thread wait-list). Returns null if
129 /// empty. Single-consumer: only the owning worker drains its own queue.
130 fn splice_reversed(&self) -> *mut T {
131 let mut cur = self.head.swap(ptr::null_mut(), Ordering::AcqRel);
132 let mut rev: *mut T = ptr::null_mut();
133 while !cur.is_null() {
134 // SAFETY: `cur` is a node that was published on the stack and whose
135 // storage the caller owns; the single consumer reverses the chain it
136 // just took exclusive ownership of, so no other thread touches these
137 // links concurrently.
138 let link = unsafe { (*cur).next_link() };
139 let next = link.load(Ordering::Acquire);
140 link.store(rev, Ordering::Relaxed);
141 rev = cur;
142 cur = next;
143 }
144 rev
145 }
146}
147
148/// A queued work request in RPS-injector mode (`schbench.c` `struct request`,
149/// `:757`). The RPS thread heap-allocates one (`Box`) per request and links it
150/// onto the target worker's queue; the worker drains its queue and frees each --
151/// matching schbench's per-request malloc/free (part of the syscall profile the
152/// standalone validation compares). schbench's `start_time` field is set
153/// (`:951`) but never read, so it is omitted: the wakeup latency is measured
154/// from the worker's `wake_time` (stamped by the RPS thread at enqueue, and by
155/// the worker itself before it blocks -- see [`rps_wait`]) and the request
156/// latency from a local work-start -- neither reads `start_time`.
157/// Omitting it also drops schbench's per-request `gettimeofday` for that dead
158/// field (`:951`) -- one fewer clock syscall per request, consistent with the
159/// engine's `CLOCK_MONOTONIC`-vs-`gettimeofday` clock divergence.
160struct Request {
161 /// Intrusive link for the per-worker request [`TreiberStack`].
162 next: AtomicPtr<Request>,
163}
164
165impl Request {
166 fn new() -> Self {
167 Self {
168 next: AtomicPtr::new(ptr::null_mut()),
169 }
170 }
171}
172
173impl Linked for Request {
174 fn next_link(&self) -> &AtomicPtr<Self> {
175 &self.next
176 }
177}
178
179/// Declarative config for the [`Schbench`](crate::workload::WorkType::Schbench)
180/// workload. Construct via [`SchbenchConfig::default`] (schbench's own
181/// defaults) plus the chainable setters, e.g.
182/// `SchbenchConfig::default().message_threads(2).worker_threads(4)`. Derives
183/// Clone/Debug/PartialEq/Eq/Hash/serde; the builder shape follows
184/// [`WorkloadConfig`](crate::workload::WorkloadConfig), but `Eq`+`Hash` (which
185/// `WorkloadConfig` and `WorkSpec` omit because of their transitive `f64`) are
186/// available here since every field is integer/bool -- the ktstr f64-free
187/// leaf-config convention.
188///
189/// # schbench(8) CLI parity
190///
191/// This port re-expresses schbench's default (matrix-work) mode natively, so
192/// its tunables are config fields and topology rather than CLI flags. The
193/// mapping to schbench's option table (`schbench.c:138-187`):
194///
195/// | schbench flag | ktstr |
196/// |---|---|
197/// | `-m` message-threads | `message_threads` |
198/// | `-t` threads | `worker_threads` (workers per message thread; `0` = `ceil(cpuset_cpus / message_threads)`, see below) |
199/// | `-F` cache_footprint | `cache_footprint_kib` |
200/// | `-n` operations | `operations` |
201/// | `-s` sleep_usec | `sleep_usec` |
202/// | `-L` no-locking | `skip_locking` |
203/// | `-R` rps | `requests_per_sec` |
204/// | `-A` auto-rps | `auto_rps` |
205/// | `--split` (long-only) | `split_percent` (`None` = no split, all-private) |
206/// | `-p` pipe (also `--pipe`) | `pipe_transfer_bytes` (`0` = off; memory-transfer mode, no matrix work) |
207/// | `-r` runtime | in-VM: the scenario engine's run window (the engine runs until `stop`); host-side: the `run_secs` argument to [`run_standalone`](crate::workload::run_standalone) |
208///
209/// ## Set by ktstr topology, not a flag
210///
211/// - `-t` default: with `worker_threads = 0`, ktstr matches schbench's `-t`
212/// 0-default -- it divides the CPU count across the message threads,
213/// `ceil(cpus / message_threads)` per thread (`schbench.c:1849-1852`), so the
214/// total worker count stays near the CPU count. ktstr scopes "cpus" to the
215/// allocated guest cpuset (the worker's `sched_getaffinity` mask, set by the
216/// scenario's topology / `CgroupDef`) rather than schbench's `get_nprocs`, so
217/// the total is ≈ the cpuset's CPU count. An explicit non-zero `worker_threads`
218/// is workers-per-message-thread in both.
219/// - `-M` (message-cpus) / `-W` (worker-cpus) thread pinning: ktstr places
220/// threads through its affinity / cpuset layer, so there is deliberately no
221/// per-thread-pin knob.
222///
223/// ## Observability flags -> the metric API
224///
225/// schbench's `-w` (warmuptime), `-i` (intervaltime), `-z` (zerotime), `-j`
226/// (json), and `-J` (jobname) shape its streaming stderr/JSON report. ktstr's
227/// numbers flow through the metric API instead -- per-phase attribution and the
228/// sidecar -- so these have no flag equivalent. `ktstr-schbench-validate`
229/// reproduces schbench's stderr-table shape for a side-by-side comparison.
230///
231/// ## Split mode (`--split`)
232///
233/// `Some(p)` partitions `cache_footprint_kib` into a per-thread private matrix
234/// (`p`%) and ONE process-global shared matrix (`100-p`%) that every worker
235/// multiplies into concurrently, reproducing schbench's cross-core
236/// shared-working-set cache contention (`schbench.c:1390-1404`, `:1858-1863`).
237/// ktstr models the shared matrix with `AtomicU64` `Relaxed` accesses. Like
238/// schbench's emitted code, the shared kernel keeps the running sum in a register
239/// and STORES it to each shared C cell on every inner (`k`) iteration -- C is
240/// write-only in the loop (A and B are loaded each `k`, C is never reloaded), and
241/// that per-k store is what generates the contention. Both gcc and clang keep the
242/// per-k store: `do_some_math` reads `m1`/`m2`/`m3` as offsets into one base
243/// pointer, so neither can prove the `m3` store doesn't alias the next `k`'s
244/// `m1`/`m2` loads. On x86-64 a `Relaxed` load/store lowers to a plain `MOV` (no
245/// `LOCK`), so the contention is identical to schbench's plain shared-memory race
246/// -- but sound (atomics, no data race), with zero unsafe. `None` (default) is
247/// the legacy all-private single matrix.
248///
249/// ## Pipe mode (`-p`)
250///
251/// `pipe_transfer_bytes > 0` REPLACES the matrix workload with schbench's
252/// memory-transfer simulation (`schbench.c:177`, `pipe_test`). It rides the
253/// message-handshake path: the message thread memsets each woken worker's
254/// per-thread page to `1` (`schbench.c:980-981`) and the worker memsets its own
255/// page to `2` before blocking (`schbench.c:1003-1004`), `pipe_transfer_bytes`
256/// bytes each per handshake cycle (clamped to 1 MiB, `PIPE_TRANSFER_BUFFER`).
257/// `do_work` and the think-sleep are skipped (`schbench.c:1448`), so the only
258/// per-cycle work is the wakeup handshake + the two memsets; the report is the
259/// PER-WORKER memory-transfer throughput (`avg worker transfer` = the aggregate
260/// rate divided by the worker count, `schbench.c:1697,1942-1943,1979`) alongside
261/// the wakeup-latency table, not request latency.
262///
263/// ktstr does NOT compose `-p` with `-R`: in pipe mode it always runs the
264/// message-handshake waker (so BOTH pipe memsets fire — a full transfer) and never
265/// starts the RPS injector. schbench instead COMPOSES them, half-broken: it has no
266/// precedence (`-R` alone picks the waker, `schbench.c:1594`), so `-p -R` runs the
267/// RPS injector while the worker-side memset still fires unconditionally
268/// (`schbench.c:1003-1004`) but the waker-side memset — which lives only in
269/// `xlist_wake_all` (`schbench.c:980-981`) — does not, yielding a degenerate
270/// half-pipe. ktstr's full pipe is the more faithful `-p` behavior; the realistic
271/// use is `-p` without `-R`. schbench also zeroes warmuptime in pipe mode
272/// (`schbench.c:296`); ktstr has no warmuptime concept, so that is a no-op here.
273///
274/// ## Modes not ported
275///
276/// - `-C` (calibrate): a tuning aid that times schbench's own work loop and
277/// forces `-L` (`schbench.c:166`, `:389`). Intentionally out of scope -- ktstr
278/// measures through the metric path.
279#[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
280#[serde(rename_all = "snake_case")]
281pub struct SchbenchConfig {
282 /// Number of message threads (`schbench.c` `-m`, default 1).
283 pub message_threads: usize,
284 /// Worker threads per message thread (`schbench.c` `-t`). 0 resolves to
285 /// `ceil(cpuset_cpus / message_threads)` -- the CPU count of the allocated
286 /// guest cpuset (the worker's `sched_getaffinity` mask, per ruling) divided
287 /// across the message threads, matching schbench's 0-default
288 /// (`schbench.c:1849-1852`) scoped to the cpuset rather than `get_nprocs`.
289 /// See `resolve_worker_count` and the CLI-parity section above.
290 pub worker_threads: usize,
291 /// Per-worker matrix cache footprint in KiB (`schbench.c` `-F`, default
292 /// 256); sets the matrix dimension.
293 pub cache_footprint_kib: usize,
294 /// Matrix multiplications per work cycle (`schbench.c` `-n`, default 5).
295 pub operations: usize,
296 /// Think-time sleep before the matrix work, microseconds (`schbench.c`
297 /// `-s`, default 100); simulates networking. 0 disables.
298 pub sleep_usec: u64,
299 /// Skip the per-CPU lock around the matrix work (`schbench.c` `-L`,
300 /// default false: locking on).
301 pub skip_locking: bool,
302 /// Fixed request rate, requests/second (`schbench.c` `-R`, default 0 = off).
303 /// 0 selects the default message-handshake mode (each worker is woken by its
304 /// message thread); non-zero switches to the RPS-injector mode, where a
305 /// dedicated thread enqueues `requests_per_sec` requests/second round-robin
306 /// across the workers (`schbench.c` `run_rps_thread`, `:1258`).
307 pub requests_per_sec: usize,
308 /// Auto-RPS target CPU-busy percentage (`schbench.c` `-A`, default 0 = off).
309 /// Non-zero turns on closed-loop rate control: a once-per-second control
310 /// thread grows/shrinks the live request rate toward this host-busy% target
311 /// (`schbench.c` `auto_scale_rps`, `:1180`). Setting it seeds the rate to 10
312 /// when `requests_per_sec` is 0 (`schbench.c:286`), so auto-RPS starts low
313 /// and climbs.
314 pub auto_rps: usize,
315 /// Percent of the cache footprint that is PRIVATE per worker thread
316 /// (`schbench.c` `--split`, long-only, 0-100). `None` = no split:
317 /// schbench's legacy all-private single matrix (`schbench.c:1405-1408`,
318 /// `:1879-1880`). `Some(p)` partitions `cache_footprint_kib` into a
319 /// per-thread private matrix (`p`%) and ONE process-global shared matrix
320 /// (`100-p`%) that every worker multiplies into concurrently, reproducing
321 /// schbench's cross-core shared-working-set cache contention
322 /// (`schbench.c:1390-1404`, `:1858-1863`). `Some(0)` = all shared,
323 /// `Some(100)` = all private (same matrix sizes as `None`, but routed
324 /// through the split branch, matching schbench's `split_specified` path).
325 /// Out-of-range `Some(p > 100)` panics when the engine consumes it
326 /// (`schbench.c:362-365` exits on the same); the builder also debug-asserts
327 /// the bound.
328 pub split_percent: Option<usize>,
329 /// Pipe-mode transfer size in bytes (`schbench.c` `-p`/`--pipe`, default 0 =
330 /// off, clamped to 1 MiB `PIPE_TRANSFER_BUFFER`). Non-zero REPLACES the
331 /// matrix workload with schbench's memory-transfer simulation: the message
332 /// thread memsets each woken worker's per-thread page to `1` and the worker
333 /// memsets its own page to `2` (`schbench.c:980-981`/`:1003-1004`),
334 /// `pipe_transfer_bytes` bytes each per cycle, while `do_work` + the
335 /// think-sleep are skipped (`schbench.c:1448`). Reports PER-WORKER
336 /// memory-transfer throughput (`avg worker transfer`) rather than request
337 /// latency.
338 pub pipe_transfer_bytes: usize,
339}
340
341/// schbench's `PIPE_TRANSFER_BUFFER` (`schbench.c:41`): the per-thread pipe page
342/// is 1 MiB; a larger `pipe_transfer_bytes` is clamped to it (schbench clamps and
343/// warns, `schbench.c:291-294`).
344pub(crate) const PIPE_TRANSFER_BUFFER: usize = 1024 * 1024;
345
346/// Cap on a single RPS-injector pacing sleep (ns). `run_rps_thread` runs
347/// synchronously on the message thread that `run` joins on shutdown, so an
348/// uncapped paced sleep would delay teardown by up to a full inter-arrival
349/// interval — at a degenerate low `requests_per_sec` (interval = 1s / rps) that
350/// can exceed the scenario cleanup/watchdog budget. Capping each sleep at this
351/// quantum re-checks `stop` at least this often, bounding shutdown latency. At
352/// realistic rates the interval is well under this, so a single uncapped wait
353/// results — the cap only engages at pathological low rates. (Schbench-local; the
354/// sibling taobench engine has its own equivalent, both with self-contained
355/// `monotonic_nanos`.)
356const STOP_POLL_QUANTUM_NS: u64 = 50_000_000;
357
358/// schbench's pipe-mode (`-p`) throughput summary — the `avg worker transfer`
359/// line (`schbench.c:1979-1982`): the per-worker transfer rate as ops/sec plus
360/// the pretty-scaled bytes/sec. Built by [`pipe_transfer_report`].
361#[derive(Debug, Clone, Copy, PartialEq)]
362pub struct PipeTransferReport {
363 /// Completed transfer cycles per second, PER WORKER (`loop_count / Σ worker
364 /// runtimes`, `schbench.c:1942-1943`).
365 pub ops_per_sec: f64,
366 /// Bytes transferred per second PER WORKER, scaled into [`unit`](Self::unit)
367 /// (÷1024 per step, `pretty_size`).
368 pub scaled: f64,
369 /// The unit `scaled` is expressed in: `B`/`KB`/`MB`/`GB`/`TB`/`PB`/`EB`
370 /// (`schbench.c:1606`).
371 pub unit: &'static str,
372}
373
374/// Derive the pipe-mode `avg worker transfer` line from a run's aggregate
375/// `achieved_rps` (completed cycles/sec over the true elapsed window), the
376/// requested `pipe_transfer_bytes`, and the resolved `nr_workers`. The figure is
377/// PER WORKER: schbench divides by `loop_runtime` = Σ each worker's runtime
378/// (`schbench.c:1697` sums `worker->runtime`; `:1942-1943`/`:1979` divide by it),
379/// and `Σ worker runtimes ≈ nr_workers * elapsed`, so the per-worker rate is the
380/// aggregate `achieved_rps / nr_workers` — the label is literally "avg WORKER
381/// transfer". (Dividing the aggregate by wall-clock alone would over-report by
382/// `nr_workers`×.) The transfer size is CLAMPED to `PIPE_TRANSFER_BUFFER` first —
383/// the engine moves only the clamped size per cycle (`run` applies the same
384/// `.min()`), matching schbench's parse-time clamp (`schbench.c:291-294`) — so the
385/// throughput reflects the bytes ACTUALLY moved. Scaling is schbench's
386/// `pretty_size` (`schbench.c:1606`). `nr_workers` is floored at 1 (no division by
387/// zero).
388pub fn pipe_transfer_report(
389 achieved_rps: f64,
390 pipe_transfer_bytes: usize,
391 nr_workers: usize,
392) -> PipeTransferReport {
393 let n = nr_workers.max(1) as f64;
394 let bytes = pipe_transfer_bytes.min(PIPE_TRANSFER_BUFFER);
395 // PER-WORKER: schbench's loops_per_sec/mb_per_sec divide the aggregate by
396 // Σ worker runtimes (≈ nr_workers * elapsed), i.e. `achieved_rps / nr_workers`.
397 let ops_per_sec = achieved_rps / n;
398 let bytes_per_sec = achieved_rps * bytes as f64 / n;
399 let (scaled, unit) = pretty_size(bytes_per_sec);
400 PipeTransferReport {
401 ops_per_sec,
402 scaled,
403 unit,
404 }
405}
406
407/// schbench's `pretty_size` (`schbench.c:1606-1620`): scale a byte count by 1024
408/// into B/KB/MB/GB/TB/PB/EB. Stops at the last unit (matches schbench's
409/// `units[divs + 1] == NULL` break — never overflows the table).
410fn pretty_size(mut number: f64) -> (f64, &'static str) {
411 const UNITS: [&str; 7] = ["B", "KB", "MB", "GB", "TB", "PB", "EB"];
412 let mut divs = 0;
413 while number >= 1024.0 && divs + 1 < UNITS.len() {
414 divs += 1;
415 number /= 1024.0;
416 }
417 (number, UNITS[divs])
418}
419
420impl Default for SchbenchConfig {
421 fn default() -> Self {
422 // schbench defaults (schbench.c option table + globals).
423 Self {
424 message_threads: 1,
425 worker_threads: 0,
426 cache_footprint_kib: 256,
427 operations: 5,
428 sleep_usec: 100,
429 skip_locking: false,
430 requests_per_sec: 0,
431 auto_rps: 0,
432 split_percent: None,
433 pipe_transfer_bytes: 0,
434 }
435 }
436}
437
438impl SchbenchConfig {
439 /// Set the number of message threads (schbench `-m`).
440 #[must_use = "builder methods consume self; bind the result"]
441 pub fn message_threads(mut self, n: usize) -> Self {
442 self.message_threads = n;
443 self
444 }
445 /// Set worker threads per message thread (schbench `-t`); 0 = one per
446 /// allocated CPU.
447 #[must_use = "builder methods consume self; bind the result"]
448 pub fn worker_threads(mut self, n: usize) -> Self {
449 self.worker_threads = n;
450 self
451 }
452 /// Set the per-worker matrix cache footprint in KiB (schbench `-F`).
453 #[must_use = "builder methods consume self; bind the result"]
454 pub fn cache_footprint_kib(mut self, kib: usize) -> Self {
455 self.cache_footprint_kib = kib;
456 self
457 }
458 /// Set the matrix multiplications per work cycle (schbench `-n`).
459 #[must_use = "builder methods consume self; bind the result"]
460 pub fn operations(mut self, n: usize) -> Self {
461 self.operations = n;
462 self
463 }
464 /// Set the think-time sleep in microseconds (schbench `-s`); 0 disables.
465 #[must_use = "builder methods consume self; bind the result"]
466 pub fn sleep_usec(mut self, usec: u64) -> Self {
467 self.sleep_usec = usec;
468 self
469 }
470 /// Skip the per-CPU lock around the matrix work (schbench `-L`).
471 #[must_use = "builder methods consume self; bind the result"]
472 pub fn skip_locking(mut self, skip: bool) -> Self {
473 self.skip_locking = skip;
474 self
475 }
476 /// Set the fixed request rate in requests/second (schbench `-R`); 0 selects
477 /// the default message-handshake mode, non-zero the RPS-injector mode.
478 #[must_use = "builder methods consume self; bind the result"]
479 pub fn requests_per_sec(mut self, rps: usize) -> Self {
480 self.requests_per_sec = rps;
481 self
482 }
483 /// Set the auto-RPS target host-busy percentage (schbench `-A`); 0 disables
484 /// auto-scaling. Non-zero seeds the rate to 10 when `requests_per_sec` is 0.
485 #[must_use = "builder methods consume self; bind the result"]
486 pub fn auto_rps(mut self, target_pct: usize) -> Self {
487 self.auto_rps = target_pct;
488 self
489 }
490 /// Set the private/shared cache-footprint split percentage (schbench
491 /// `--split`); `None` (default) = no split, all-private single matrix.
492 /// `Some(p)` requires `p <= 100`: the builder debug-asserts it for early
493 /// feedback, and the engine hard-panics on an out-of-range value at the
494 /// consumption boundary in `run` -- the analog of schbench exiting on a bad
495 /// `--split` (`schbench.c:362-365`).
496 #[must_use = "builder methods consume self; bind the result"]
497 pub fn split_percent(mut self, percent: Option<usize>) -> Self {
498 debug_assert!(
499 percent.is_none_or(|p| p <= 100),
500 "split_percent must be 0..=100"
501 );
502 self.split_percent = percent;
503 self
504 }
505 /// Set the pipe-mode transfer size in bytes (schbench `-p`/`--pipe`); 0
506 /// (default) = off (the matrix workload). Non-zero switches to schbench's
507 /// memory-transfer simulation; values above 1 MiB (`PIPE_TRANSFER_BUFFER`)
508 /// are clamped when the engine consumes it (schbench clamps the same,
509 /// `schbench.c:291-294`).
510 #[must_use = "builder methods consume self; bind the result"]
511 pub fn pipe_transfer_bytes(mut self, bytes: usize) -> Self {
512 self.pipe_transfer_bytes = bytes;
513 self
514 }
515
516 /// Matrix dimension from the cache footprint, identical to schbench
517 /// (`schbench.c:1880`: `sqrt(cache_footprint_kb * 1024 / 3 /
518 /// sizeof(unsigned long))`) and to ktstr's `FanOutCompute` precompute. Zero
519 /// `operations` or `cache_footprint_kib` yields a 0 dimension (no matrix
520 /// work).
521 pub(crate) fn matrix_size(&self) -> usize {
522 if self.operations > 0 && self.cache_footprint_kib > 0 {
523 ((self.cache_footprint_kib * 1024 / 3 / core::mem::size_of::<u64>()) as f64).sqrt()
524 as usize
525 } else {
526 0
527 }
528 }
529
530 /// Shared-matrix dimension for `--split` (`schbench.c:1859,1862`:
531 /// `sqrt(cache_footprint_kb*(100-split)/100 * 1024/3/sizeof(ulong))`).
532 /// `None` split (or 0 `operations`/`cache_footprint_kib`) => 0 (no shared
533 /// matrix); `Some(100)` => 0 (all private, no shared work).
534 pub(crate) fn shared_matrix_size(&self) -> usize {
535 match self.split_percent {
536 Some(p) if self.operations > 0 && self.cache_footprint_kib > 0 => {
537 let shared_kb = self.cache_footprint_kib * (100 - p) / 100;
538 ((shared_kb * 1024 / 3 / core::mem::size_of::<u64>()) as f64).sqrt() as usize
539 }
540 _ => 0,
541 }
542 }
543
544 /// Private (per-worker) matrix dimension. `Some(p)` (`schbench.c:1860,1863`):
545 /// `sqrt(cache_footprint_kb*split/100 * 1024/3/sizeof(ulong))`. `None`
546 /// (legacy): the full-footprint [`matrix_size`](Self::matrix_size)
547 /// (`schbench.c:1880`, the all-private single matrix).
548 pub(crate) fn private_matrix_size(&self) -> usize {
549 match self.split_percent {
550 Some(p) if self.operations > 0 && self.cache_footprint_kib > 0 => {
551 let private_kb = self.cache_footprint_kib * p / 100;
552 ((private_kb * 1024 / 3 / core::mem::size_of::<u64>()) as f64).sqrt() as usize
553 }
554 _ => self.matrix_size(),
555 }
556 }
557
558 /// The effective TOTAL request rate after schbench's startup seed: the user's
559 /// `requests_per_sec`, or 10 when auto-RPS is on and no fixed rate was given
560 /// (`schbench.c:286`: auto-RPS starts at 10 and climbs toward its target).
561 /// Both the mode gate ([`rps_per_message_thread`](Self::rps_per_message_thread))
562 /// and the live auto-scaled injection rate seed from this.
563 pub(crate) fn normalized_total_rps(&self) -> usize {
564 if self.auto_rps != 0 && self.requests_per_sec == 0 {
565 10
566 } else {
567 self.requests_per_sec
568 }
569 }
570
571 /// Per-message-thread request rate driving the RPS-vs-default MODE GATE: the
572 /// normalized total ([`normalized_total_rps`](Self::normalized_total_rps))
573 /// divided across the message threads, each of which runs its own RPS thread.
574 /// schbench divides once at startup (`requests_per_sec /= message_threads`,
575 /// `schbench.c:1899`) BEFORE the per-thread mode gate (`if (requests_per_sec)`,
576 /// `schbench.c:1594`), so the gate sees the divided value: a total below the
577 /// message-thread count rounds to 0 and the thread runs the DEFAULT
578 /// message-handshake mode (not a zero-rate RPS). The gate is decided ONCE from
579 /// this seeded value; the actual injection rate is the live auto-scaled total
580 /// (which may climb above the seed). Integer division drops the remainder,
581 /// matching schbench -- so auto-RPS with more than 10 message threads (seed
582 /// 10 / m = 0) degenerates to default mode, exactly as schbench does.
583 pub(crate) fn rps_per_message_thread(&self) -> usize {
584 self.normalized_total_rps() / self.message_threads.max(1)
585 }
586
587 /// Effective per-message-thread worker count for [`run`]. A non-zero
588 /// `worker_threads` is workers-per-message-thread as-is; the `0` default
589 /// mirrors schbench's `-t` 0-default (`schbench.c:1849-1852`:
590 /// `worker_threads = (num_cpus + message_threads - 1) / message_threads`)
591 /// but over the allocated guest cpuset (`allowed_count` from
592 /// [`resolve_cpu_topology`], per the ktstr cpuset ruling) instead of
593 /// `get_nprocs`: it divides the cpuset CPU count across the message threads
594 /// (ceil), so the TOTAL worker count (`message_threads * this`) stays ≈ the
595 /// cpuset CPU count -- matching schbench's total ≈ `num_cpus`.
596 /// `message_threads` floors at 1 to avoid a zero divisor; the divide is a
597 /// no-op at the default `message_threads = 1`.
598 pub(crate) fn resolve_worker_count(&self, allowed_count: usize) -> usize {
599 if self.worker_threads != 0 {
600 return self.worker_threads;
601 }
602 allowed_count.div_ceil(self.message_threads.max(1))
603 }
604}
605
606/// Per-thread state, the Rust counterpart of schbench's `struct thread_data`
607/// (`schbench.c:766`). Shared across threads by raw pointer for the lockless
608/// wait-list, so cross-thread access is restricted to the atomic fields
609/// (`next`, `futex`, `wake_time`); the histogram fields are owned solely by the
610/// worker thread the `ThreadData` belongs to.
611pub(crate) struct ThreadData {
612 /// Treiber-stack link for the message thread's wait-list
613 /// (schbench `thread_data->next`, `:776`). Null when not queued.
614 next: AtomicPtr<ThreadData>,
615 /// Wake handshake futex (`schbench.c` `thread_data->futex`, `:791`).
616 futex: super::handshake::Handshake,
617 /// Monotonic-ns timestamp the waker stamps just before posting, so the
618 /// woken thread can measure scheduler wakeup latency (schbench `wake_time`,
619 /// `:788`, stamped in `xlist_wake_all` `:984`).
620 wake_time: AtomicU64,
621 /// Wakeup-latency histogram (`schbench.c` `wakeup_stats`, `:794`).
622 /// Owner-thread-only; see the `Sync` impl SAFETY note.
623 wakeup_stats: UnsafeCell<PlatStats>,
624 /// Request (work-cycle) latency histogram (`schbench.c` `request_stats`,
625 /// `:795`). Owner-thread-only.
626 request_stats: UnsafeCell<PlatStats>,
627 /// Mean per-schedule run-queue wait (ns), read from `/proc/<tid>/schedstat`
628 /// at thread exit (schbench's `read_sched_delay`, `:1118`). Owner-only.
629 /// Feeds the WHOLE-RUN `SchbenchResult` (mean-of-means, matching real
630 /// schbench for the side-by-side validation); the per-phase delay comes from
631 /// [`Self::phase_snapshots`] instead.
632 sched_delay_ns: UnsafeCell<u64>,
633 /// Per-phase snapshots this thread accumulated via drain-on-change against
634 /// the shared `phase_epoch` (one [`PhaseSnapshot`] per phase the thread did
635 /// work in, plus a final in-flight phase at exit). Owner-thread-only,
636 /// drained by the main thread after join — same happens-before as the
637 /// histogram cells. Empty when run non-phasic (`phase_epoch == None`) until
638 /// the single end-of-run drain.
639 phase_snapshots: UnsafeCell<Vec<PhaseSnapshot>>,
640 /// RPS-injector mode: this worker's pending request queue. The RPS thread
641 /// pushes heap-allocated [`Request`]s (round-robin across workers); the
642 /// owning worker drains via `splice_reversed` (FIFO) and frees each. Unused
643 /// in the default message-handshake mode (`requests_per_sec == 0`).
644 requests: TreiberStack<Request>,
645 /// RPS-injector backpressure counter (`schbench.c` `thread_data->pending`,
646 /// `:779`): incremented by the RPS thread per enqueue, reset to 0 by the
647 /// owning worker at each splice. The RPS thread stops enqueuing to a worker
648 /// whose `pending` exceeds the batch cap (`:1284`).
649 pending: AtomicU64,
650 /// Pipe-mode (`-p`) transfer size in bytes (0 = not pipe mode); the length of
651 /// [`Self::pipe_page`] and the per-cycle memset. Plain `Copy`, set once at
652 /// construction, read-only thereafter.
653 pipe_bytes: usize,
654 /// Pipe-mode per-thread transfer page (`schbench.c` `thread_data->pipe_page`,
655 /// `:801`). The worker memsets it to `2` before blocking and the waker memsets
656 /// it to `1` when waking the worker; the bytes are never read back (the point
657 /// is the memory-transfer cost). `UnsafeCell` because the WAKER writes the
658 /// worker's page (cross-thread) -- see the `Sync` SAFETY note for the
659 /// wait-list/futex ordering that keeps it race-free. Empty unless pipe mode.
660 pipe_page: UnsafeCell<Box<[u8]>>,
661}
662
663/// One thread's latency + run-queue-delay accumulation over a single phase
664/// (the window between two `phase_epoch` transitions). Built on the owning
665/// thread at each drain-on-change boundary; the histograms are `take`n from the
666/// live cells (snapshot-and-reset) and the run-delay is a RAW
667/// `/proc/<tid>/schedstat` `(run_delay, pcount)` DELTA over the phase, so the
668/// host re-derives the per-phase mean as `Σrun_delay / Σpcount` (the
669/// sample-weighted pooled mean — a deliberate divergence from schbench's
670/// whole-run mean-of-means, correct for a heterogeneous per-phase thread set).
671/// Message threads leave `wakeup`/`request` empty and `loop_count` 0 (they
672/// carry only run-delay); workers fill all fields.
673struct PhaseSnapshot {
674 /// The `phase_epoch` value active while this snapshot's samples were
675 /// recorded (NOT the value at drain time). `0` = BASELINE, `u32::MAX` =
676 /// inter-step gap; both are emitted as-is and discarded host-side.
677 epoch: u32,
678 wakeup: PlatStats,
679 request: PlatStats,
680 /// `/proc/<tid>/schedstat` field 2 (run_delay ns) delta over this phase.
681 run_delay_ns: u64,
682 /// `/proc/<tid>/schedstat` field 3 (pcount = timeslices) delta over this
683 /// phase. The host guards `pcount == 0` → metric absent (never a div-by-zero
684 /// 0), matching [`mean_sched_delay`].
685 pcount: u64,
686 /// Completed work cycles this worker ran in this phase (0 for the message
687 /// thread).
688 loop_count: u64,
689 /// `CLOCK_THREAD_CPUTIME_ID` delta over this phase — the CPU time this
690 /// WORKER thread actually received (0 for the message thread: the
691 /// CPU-throughput denominator is Σ worker CPU, the CPU-clock analog of
692 /// schbench's Σ worker runtimes, `schbench.c:1697`). Feeds the
693 /// CPU-second-denominated `schbench_loops_per_cpu_sec` throughput.
694 cpu_ns: u64,
695}
696
697/// Per-phase, cross-thread aggregate for one `phase_epoch`: the wakeup + request
698/// histograms merged across every worker that ran in the phase, plus the
699/// run-delay raw pairs split by thread class (message vs worker). The per-phase
700/// wire carrier — rides inside [`crate::workload::PhaseSlice`] guest→host. All
701/// fields are integer, so the host re-pools across workers/cgroups by
702/// [`PlatStats::combine`] (histogram add) + integer sums; percentiles are
703/// re-derived from the merged histogram, NEVER averaged.
704///
705/// ESTIMATOR NOTE: the per-phase run-delay mean the host derives from
706/// `*_run_delay_ns / *_pcount` is SAMPLE-WEIGHTED (`Σrun_delay / Σpcount`), a
707/// DIFFERENT estimator from the whole-run [`SchbenchResult`]'s `sched_delay_*`,
708/// which keeps schbench's mean-of-per-thread-means (`collect_sched_delay`) for
709/// schbench parity. They measure different things by design — never cross-compare a
710/// per-phase value against a whole-run threshold (or vice-versa). `pcount == 0`
711/// for a class means that class was never scheduled in the phase → the host
712/// emits the metric as ABSENT, not `0`.
713#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
714pub(crate) struct SchbenchPhaseStats {
715 /// Wakeup-latency histogram merged across the phase's workers.
716 pub(crate) wakeup: PlatStats,
717 /// Request-latency histogram merged across the phase's workers.
718 pub(crate) request: PlatStats,
719 /// Per-phase achieved-RPS distribution: the control thread's per-second
720 /// samples attributed to this phase's epoch (the whole-run rps is unchanged;
721 /// this is purely additive). A 1s sample straddling an epoch boundary lands
722 /// wholly in the sample-time epoch (bounded fuzz, <=1 sample/transition, like
723 /// the msg-thread late-drain). Empty for a phase shorter than the ~1s control
724 /// cadence -- the host gates on `sample_count() > 0` so a sub-second phase
725 /// reads ABSENT, never rps=0. A rate; re-derived from the merged histogram.
726 ///
727 /// DENOMINATION: these per-second samples stay WALL-denominated (an
728 /// exception to the CPU-second throughput policy) — each sample is
729 /// schbench's own per-tick `rps_stats` reading (Δloops over a ~1s wall
730 /// window, upstream parity), and a per-tick CPU denominator would need
731 /// cross-thread cumulative-CPU reads from the control thread every
732 /// second, changing the sampling the upstream comparison validates. The
733 /// CPU-denominated throughput lives in `worker_cpu_ns` / the
734 /// `schbench_loops_per_cpu_sec` metric instead.
735 pub(crate) rps: PlatStats,
736 /// Σ message-thread `run_delay` (ns) over the phase.
737 pub(crate) msg_run_delay_ns: u64,
738 /// Σ message-thread `pcount` over the phase (the mean's denominator).
739 pub(crate) msg_pcount: u64,
740 /// Σ worker `run_delay` (ns) over the phase.
741 pub(crate) worker_run_delay_ns: u64,
742 /// Σ worker `pcount` over the phase (the mean's denominator).
743 pub(crate) worker_pcount: u64,
744 /// Σ completed work cycles across the phase's workers.
745 pub(crate) loop_count: u64,
746 /// Σ worker-thread `CLOCK_THREAD_CPUTIME_ID` deltas over the phase (ns) —
747 /// the CPU time the phase's workers actually received, the denominator of
748 /// the CPU-second throughput `schbench_loops_per_cpu_sec` (message-thread
749 /// CPU is excluded, mirroring schbench's Σ worker runtimes). CPU-clock
750 /// denominated so the throughput measures the workload, not host/guest
751 /// scheduling delay; the wall picture reconstructs as `cpu_rate / D` with
752 /// the sidecar's `host_dilation`. `#[serde(default)]`: pre-field sidecars
753 /// deserialize with 0 (the metric then reads absent).
754 #[serde(default)]
755 pub(crate) worker_cpu_ns: u64,
756}
757
758impl SchbenchPhaseStats {
759 /// Merge `other` into `self`: combine the wakeup + request histograms
760 /// (bucket-count addition, [`PlatStats::combine`]) and integer-add the
761 /// run-delay raw pairs + loop_count. Associative AND commutative (combine +
762 /// `saturating_add` both are), so pooling is order-independent — the SAME
763 /// operation whether pooling per-engine across message threads (in [`run`])
764 /// or per-cgroup across workers host-side
765 /// (`crate::assert::PhaseCgroupStats::merge`). Percentiles are NEVER
766 /// merged; the merged histogram is re-derived to percentiles by the reader.
767 pub(crate) fn merge(&mut self, other: &SchbenchPhaseStats) {
768 self.wakeup.combine(&other.wakeup);
769 self.request.combine(&other.request);
770 self.rps.combine(&other.rps);
771 self.msg_run_delay_ns = self.msg_run_delay_ns.saturating_add(other.msg_run_delay_ns);
772 self.msg_pcount = self.msg_pcount.saturating_add(other.msg_pcount);
773 self.worker_run_delay_ns = self
774 .worker_run_delay_ns
775 .saturating_add(other.worker_run_delay_ns);
776 self.worker_pcount = self.worker_pcount.saturating_add(other.worker_pcount);
777 self.loop_count = self.loop_count.saturating_add(other.loop_count);
778 self.worker_cpu_ns = self.worker_cpu_ns.saturating_add(other.worker_cpu_ns);
779 }
780}
781
782impl Linked for ThreadData {
783 fn next_link(&self) -> &AtomicPtr<Self> {
784 &self.next
785 }
786}
787
788// SAFETY: `ThreadData` is shared across threads only via the lockless wait-list
789// and (in RPS mode) the per-worker request queue, whose operations touch
790// exclusively the atomic fields (`next`, `futex`, `wake_time`, `requests`,
791// `pending`) -- all internally synchronized. `requests` is single-producer /
792// single-consumer: the RPS thread is the only pusher to a given worker's queue
793// (round-robin assignment) and the owning worker is the only drainer, with the
794// push's Release CAS / drain's Acquire swap establishing happens-before;
795// `pending` is plain atomic counting. The `UnsafeCell` fields (`wakeup_stats`,
796// `request_stats`, `sched_delay_ns`, `phase_snapshots`) are written and read
797// ONLY by the single worker thread that owns this `ThreadData`, and only the
798// main thread reads/drains them after all workers have joined (a happens-before
799// via the join). The per-phase drain (`worker_loop` / `run_msg_thread`) `take`s
800// and pushes into the owning thread's own cells, never another's. The `pipe_page`
801// cell (pipe mode) is the one cross-thread `UnsafeCell`: the owning worker memsets
802// it (to 2) and the message thread memsets it (to 1). They never overlap, via two
803// happens-before edges:
804// - SAME cycle (worker fill-2 vs waker fill-1): the worker's fill-2 is SEQUENCED
805// BEFORE its `wait_list.add` (Release CAS), and the waker's `splice` (Acquire
806// swap) observes that add, giving fill-2 -> add -> splice -> waker fill-1. The
807// load-bearing fact is "fill-2 precedes add", which holds UNCONDITIONALLY --
808// including the no-block shutdown path (stop set, worker skips the futex wait):
809// the worker still added itself after fill-2, so a waker that then splices and
810// fills it is ordered after the fill-2, and the worker's exit path never
811// touches the page again. Blocking is NOT what makes it safe.
812// - NEXT cycle (worker's fill-2 vs the waker's prior fill-1): the futex
813// post/consume CAS (SeqCst, `Handshake`) orders waker fill-1 -> post -> the
814// worker's consume -> its next fill-2; the worker cannot reach its next fill-2
815// until it consumes the token the waker published after fill-1.
816// So the two threads never touch `pipe_page` concurrently. No two threads ever
817// touch a cell concurrently, so sharing `&ThreadData` across threads is sound.
818unsafe impl Sync for ThreadData {}
819
820impl ThreadData {
821 fn new(pipe_bytes: usize) -> Self {
822 Self {
823 next: AtomicPtr::new(ptr::null_mut()),
824 futex: super::handshake::Handshake::new(),
825 wake_time: AtomicU64::new(0),
826 wakeup_stats: UnsafeCell::new(PlatStats::default()),
827 request_stats: UnsafeCell::new(PlatStats::default()),
828 sched_delay_ns: UnsafeCell::new(0),
829 phase_snapshots: UnsafeCell::new(Vec::new()),
830 requests: TreiberStack::new(),
831 pending: AtomicU64::new(0),
832 pipe_bytes,
833 pipe_page: UnsafeCell::new(vec![0u8; pipe_bytes].into_boxed_slice()),
834 }
835 }
836
837 /// Pipe-mode transfer: fill this thread's `pipe_page` (`pipe_bytes` bytes) with
838 /// `val`. The page is never read back, so the fill would be DCE-eligible. The
839 /// trailing `black_box(page.as_ptr())` ESCAPES THE ADDRESS (not a loaded value):
840 /// the optimizer must then assume the opaque call may read through the pointer,
841 /// so every byte of the fill must be in place at that point -- the fill is
842 /// retained. This is stronger than `black_box(value)`, which only keeps a value
843 /// live and (per the optimization-resistance note in `worker::worker_main`) does
844 /// NOT by itself stop a backing STORE from being elided -- which is why the
845 /// cache-pressure workloads there use `write_volatile`. Here the faithful
846 /// schbench op is a memset (`schbench.c:980-981`/`:1003-1004`) and `slice::fill`
847 /// lowers to one; a `write_volatile` byte loop would not. The memory traffic IS
848 /// the workload.
849 ///
850 /// SAFETY: the caller guarantees no concurrent access to `pipe_page`. The only
851 /// callers are the owning worker (before it adds itself to the wait-list) and
852 /// the message thread (after it splices the worker off the wait-list); the
853 /// load-bearing edge is fill-2-before-add (holding even on the no-block
854 /// shutdown path), so the add/splice + futex handshake order these with no
855 /// overlap (see the `Sync` SAFETY note).
856 unsafe fn pipe_fill(&self, val: u8) {
857 // SAFETY: exclusive access per the caller contract; the box lives for the
858 // `ThreadData`'s lifetime.
859 let page = unsafe { &mut *self.pipe_page.get() };
860 page.fill(val);
861 std::hint::black_box(page.as_ptr());
862 }
863}
864
865/// schbench's per-request think-time sleep ("simulated networking",
866/// `schbench.c:1461`). This is the workload's defined behavior, not a
867/// synchronization wait: `usleep` maps to `clock_nanosleep`, matching
868/// schbench's syscall profile.
869fn think_sleep(usec: u64) {
870 std::thread::sleep(std::time::Duration::from_micros(usec));
871}
872
873/// schbench's `--split` op partition (`schbench.c:1391-1392`): of `operations`
874/// total, `p`% run on the private matrix and the rest on the shared matrix.
875/// Returns `(ops_shared, ops_private)`. Extracted so the integer-division split
876/// is pinned by a unit test rather than re-implemented inline in `do_work`.
877fn ops_split(operations: usize, p: usize) -> (usize, usize) {
878 let ops_private = (operations * p) / 100;
879 let ops_shared = operations - ops_private;
880 (ops_shared, ops_private)
881}
882
883/// One work cycle's matrix multiplications under the per-CPU lock. Faithful to
884/// schbench's `do_work` (`schbench.c:1379-1413`): take the current CPU's mutex
885/// (unless `skip_locking`), run `operations` matrix multiplies, unlock. With
886/// `split_percent` set, the ops are partitioned (`schbench.c:1390-1404`):
887/// `ops_shared` run on the ONE process-global shared matrix (every worker
888/// contends), then `ops_private` on this worker's private buffer; `None` is the
889/// legacy all-private single matrix (`:1406-1408`). The guard drops at the end,
890/// holding the lock across all operations exactly as schbench does (`:1387`/
891/// `:1411`).
892#[allow(clippy::too_many_arguments)]
893fn do_work(
894 private_buf: &mut [u64],
895 private_matrix_size: usize,
896 shared_buf: &[core::sync::atomic::AtomicU64],
897 shared_matrix_size: usize,
898 split_percent: Option<usize>,
899 operations: usize,
900 locks: Option<&PerCpuLocks>,
901 work_units: &mut u64,
902) {
903 let _guard = locks.map(|l| l.lock_this_cpu());
904 match split_percent {
905 Some(p) => {
906 let (ops_shared, ops_private) = ops_split(operations, p);
907 // schbench.c:1395-1398: shared ops first, on the global matrix.
908 if shared_matrix_size > 0 && ops_shared > 0 {
909 for _ in 0..ops_shared {
910 crate::workload::worker::matrix_multiply_shared(
911 shared_buf,
912 shared_matrix_size,
913 work_units,
914 );
915 }
916 }
917 // schbench.c:1401-1404: private ops second, on this worker's buffer.
918 if private_matrix_size > 0 && ops_private > 0 {
919 for _ in 0..ops_private {
920 crate::workload::worker::matrix_multiply(
921 private_buf,
922 private_matrix_size,
923 work_units,
924 );
925 }
926 }
927 }
928 None => {
929 // schbench.c:1406-1408: legacy all-private single matrix.
930 for _ in 0..operations {
931 if private_matrix_size > 0 {
932 crate::workload::worker::matrix_multiply(
933 private_buf,
934 private_matrix_size,
935 work_units,
936 );
937 }
938 }
939 }
940 }
941}
942
943/// Worker side of one wakeup cycle. Faithful to schbench's `msg_and_wait`
944/// (`schbench.c:997`, default branch): stamp our `wake_time`, push ourselves
945/// onto the message thread's wait-list, wake the message thread, then block
946/// until it wakes us back; record the wakeup (scheduler) latency. The `!stop`
947/// guard mirrors schbench's `if (!stopping)` (`schbench.c:1030`) so we do not
948/// block during shutdown.
949fn msg_and_wait(
950 td: &ThreadData,
951 msg_td: &ThreadData,
952 wait_list: &TreiberStack<ThreadData>,
953 stop: &AtomicBool,
954) {
955 // Our futex is BLOCKED here (consumed by the prior wait, or fresh).
956 // Pipe mode: write our transfer page to 2 (schbench memsets td->pipe_page
957 // before blocking, schbench.c:1003-1004). Done before we enqueue on the
958 // wait-list, so the waker's fill-to-1 (after it splices us off) never races --
959 // see the Sync SAFETY note.
960 if td.pipe_bytes > 0 {
961 // SAFETY: we own this page and have not yet enqueued, so the waker cannot
962 // touch it concurrently.
963 unsafe { td.pipe_fill(2) };
964 }
965 td.wake_time.store(monotonic_nanos(), Ordering::Release);
966 wait_list.add(td as *const ThreadData as *mut ThreadData);
967 msg_td.futex.post();
968 if !stop.load(Ordering::Acquire) {
969 td.futex.wait_forever();
970 }
971 let now = monotonic_nanos();
972 let wake = td.wake_time.load(Ordering::Acquire);
973 // schbench buckets in microseconds (gettimeofday resolution); the monotonic
974 // clock gives ns, so divide. `if delta > 0` matches schbench (`:1036`).
975 let delta_us = now.saturating_sub(wake) / 1000;
976 if delta_us > 0 {
977 // SAFETY: only this worker thread accesses its own wakeup_stats cell.
978 unsafe { (*td.wakeup_stats.get()).add_lat(delta_us.min(u32::MAX as u64) as u32) };
979 }
980}
981
982/// Worker side of one RPS-injector cycle. Faithful to schbench's `msg_and_wait`
983/// RPS branch (`schbench.c:1007-1037`): stamp our own `wake_time`, reset
984/// `pending`, then splice this worker's request queue (FIFO). If non-empty,
985/// return the chain immediately (fast path, no wait, no wakeup sample --
986/// `:1014-1017`). If empty, block on this worker's own futex until the RPS thread
987/// posts it, record the wakeup (scheduler) latency, then splice whatever it
988/// queued (null on a spurious wake or the shutdown wake-all). Unlike
989/// [`msg_and_wait`], the worker does NOT enqueue on the message thread's
990/// wait-list or post a message thread -- the RPS thread is the sole waker.
991///
992/// `wake_time` is stamped HERE before the splice (schbench `:1008`) AND by the
993/// RPS thread at enqueue (`:1294`). On a TRUE block the RPS thread's stamp wins
994/// (it overwrites ours before posting), so `now - wake_time` is the enqueue->run
995/// scheduler latency. But on a FAST wake -- a leftover RUNNING futex token from a
996/// request the worker already drained via splice (the word is consumed by splice,
997/// not by `wait`, so it stays RUNNING) -- the RPS thread does NOT re-stamp, so our
998/// own stamp makes the delta measure THIS block decision (near-zero), not the
999/// stale enqueue time of the already-served request. Omitting it inflates the
1000/// fast-wake samples to the full inter-request gap (the wakeup-p50 bug: 543ms
1001/// vs schbench's 7µs).
1002fn rps_wait(td: &ThreadData, stop: &AtomicBool) -> *mut Request {
1003 // Stamp our own wake_time before splicing (schbench `:1008`); see the doc for
1004 // why this is what keeps fast-wake samples honest.
1005 td.wake_time.store(monotonic_nanos(), Ordering::Release);
1006 // Reset the backpressure counter before draining (schbench `td->pending = 0`,
1007 // `:1012`): the RPS thread stops enqueuing while pending exceeds the batch
1008 // cap, so clearing it re-opens this worker for the next batch.
1009 td.pending.store(0, Ordering::Release);
1010 let chain = td.requests.splice_reversed();
1011 if !chain.is_null() {
1012 return chain; // work already queued: serve it without blocking
1013 }
1014 if stop.load(Ordering::Acquire) {
1015 return ptr::null_mut();
1016 }
1017 // Empty queue: block until the RPS thread posts our futex, then `now -
1018 // wake_time` is the scheduler wakeup latency (schbench `fwait` NULL then
1019 // add_lat, `:1032`/`:1034-1037`).
1020 td.futex.wait_forever();
1021 // Shutdown wake: if stop is set, the wake came from run_rps_thread's stop-path
1022 // wake-all (`:1308-1312`), which posts every worker WITHOUT restamping
1023 // wake_time. A worker blocked between bursts when stop fires would otherwise
1024 // record `now - (its own stale block-time stamp)` -- the whole inter-burst
1025 // gap -- as a phantom wakeup, one garbage tail sample per worker. That is not
1026 // a request-driven wakeup, so skip the sample (schbench's `if (!stopping)
1027 // fwait` guard, `:1030`, encodes the same intent: shutdown is not a latency
1028 // event). Return any leftover requests for the caller to free.
1029 if stop.load(Ordering::Acquire) {
1030 return td.requests.splice_reversed();
1031 }
1032 let now = monotonic_nanos();
1033 let wake = td.wake_time.load(Ordering::Acquire);
1034 let delta_us = now.saturating_sub(wake) / 1000;
1035 if delta_us > 0 {
1036 // SAFETY: only this worker thread accesses its own wakeup_stats cell.
1037 unsafe { (*td.wakeup_stats.get()).add_lat(delta_us.min(u32::MAX as u64) as u32) };
1038 }
1039 // Splice whatever the RPS thread enqueued before posting us (null if the post
1040 // was a spurious wake with nothing queued).
1041 td.requests.splice_reversed()
1042}
1043
1044/// Free a chain of [`Request`]s spliced off a worker's queue but not processed
1045/// (the shutdown path, and the post-join cleanup of any requests the RPS thread
1046/// enqueued after a worker's last splice). Each node was `Box::into_raw`'d by the
1047/// RPS thread; the draining/cleaning thread owns them exclusively after the
1048/// splice, so each is freed exactly once.
1049fn free_request_chain(mut req: *mut Request) {
1050 while !req.is_null() {
1051 // SAFETY: `req` is a node taken via `splice_reversed` (exclusive
1052 // ownership); freed exactly once.
1053 let next = unsafe { (*req).next.load(Ordering::Acquire) };
1054 drop(unsafe { Box::from_raw(req) });
1055 req = next;
1056 }
1057}
1058
1059/// Wake every worker on the wait-list, stamping each one's `wake_time` so it
1060/// can measure scheduler latency. Faithful to schbench's `xlist_wake_all`
1061/// (`schbench.c:969`): splice the whole list, read the clock ONCE, stamp every
1062/// worker with that single time, then post each. The single clock read is what
1063/// detects the scheduler preempting the waker mid-batch (`schbench.c:961-964`).
1064/// Pipe mode (`-p`) diverges, matching schbench (`schbench.c:980-984`): it memsets
1065/// each worker's transfer page to `1` and re-reads the clock PER worker (so the
1066/// memset cost is not charged to that worker's wakeup latency).
1067fn wake_all(wait_list: &TreiberStack<ThreadData>) {
1068 let mut cur = wait_list.splice();
1069 let now = monotonic_nanos();
1070 while !cur.is_null() {
1071 // SAFETY: `cur` is a worker `ThreadData` alive for the whole run; only
1072 // its atomic fields (next / wake_time / futex) are touched here.
1073 let td = unsafe { &*cur };
1074 let next = td.next.load(Ordering::Acquire);
1075 td.next.store(ptr::null_mut(), Ordering::Relaxed);
1076 if td.pipe_bytes > 0 {
1077 // Pipe mode: write this woken worker's transfer page to 1 (schbench
1078 // memsets list->pipe_page in the wake loop, schbench.c:980-981). We
1079 // spliced it off the wait-list and it is still blocked (our `post`
1080 // below wakes it), so no concurrent access -- see the Sync SAFETY note.
1081 // SAFETY: `td` is off the wait-list and blocked here.
1082 unsafe { td.pipe_fill(1) };
1083 // schbench re-reads the clock per worker in pipe mode (after the
1084 // memset, schbench.c:982) so the fill is not charged to the worker's
1085 // wakeup latency; non-pipe uses the single batched `now`.
1086 td.wake_time.store(monotonic_nanos(), Ordering::Release);
1087 } else {
1088 td.wake_time.store(now, Ordering::Release);
1089 }
1090 td.futex.post();
1091 cur = next;
1092 }
1093}
1094
1095/// The message thread's loop. Faithful to schbench's `run_msg_thread`
1096/// (`schbench.c:1166`): batch-wake all waiting workers, then block until a
1097/// worker posts us back. On stop, drain once more (to wake any worker that
1098/// queued during the final wake) and exit.
1099///
1100/// `phase_epoch` drives the per-phase drain exactly as in [`worker_loop`], but
1101/// the message thread records only per-phase RUN-DELAY (its `wakeup`/`request`
1102/// histograms stay empty, taken empty by [`drain_phase`]). It polls the epoch
1103/// after each wake cycle; a phase in which no worker ever posts it leaves that
1104/// phase's run-delay drained late at the next wake (bounded boundary fuzz,
1105/// matching the worker drain).
1106fn run_msg_thread(
1107 msg_td: &ThreadData,
1108 wait_list: &TreiberStack<ThreadData>,
1109 stop: &AtomicBool,
1110 phase_epoch: Option<&AtomicU32>,
1111) {
1112 let tid = gettid_self();
1113 let mut cur_epoch = phase_epoch.map_or(0, |e| e.load(Ordering::Relaxed));
1114 let mut phase_ss_start = read_schedstat_raw(tid);
1115 loop {
1116 // msg_td.futex is BLOCKED here (consumed by the prior wait, or fresh).
1117 wake_all(wait_list);
1118 if stop.load(Ordering::Acquire) {
1119 wake_all(wait_list);
1120 break;
1121 }
1122 msg_td.futex.wait_forever();
1123 if let Some(pe) = phase_epoch {
1124 let new_epoch = pe.load(Ordering::Relaxed);
1125 if new_epoch != cur_epoch {
1126 let ss_end = read_schedstat_raw(tid);
1127 // SAFETY: this is the thread that owns msg_td (the coordinator
1128 // thread of run_one_message_thread).
1129 unsafe { drain_phase(msg_td, cur_epoch, phase_ss_start, ss_end, 0, 0) };
1130 cur_epoch = new_epoch;
1131 phase_ss_start = ss_end;
1132 }
1133 }
1134 }
1135 // Final drain: close the still-open phase (the sole snapshot when non-phasic).
1136 let ss_end = read_schedstat_raw(tid);
1137 // SAFETY: this is the thread that owns msg_td.
1138 unsafe { drain_phase(msg_td, cur_epoch, phase_ss_start, ss_end, 0, 0) };
1139 // Record the message thread's whole-run mean run-queue wait (`schbench.c:1664`),
1140 // reusing the cumulative `ss_end` pair just read — one /proc read, so the mean
1141 // and the final-phase delta derive from one consistent snapshot.
1142 // SAFETY: owner-only access to msg_td's sched_delay_ns cell.
1143 unsafe { *msg_td.sched_delay_ns.get() = mean_sched_delay(ss_end) };
1144}
1145
1146/// The RPS-injector dispatcher loop, replacing [`run_msg_thread`] when
1147/// `requests_per_sec != 0`. Each second it enqueues `requests_per_sec` requests
1148/// round-robin across this group's `workers`, skipping a worker whose `pending`
1149/// exceeds the batch cap (backpressure, `schbench.c:1284-1290`); on stop, it
1150/// wakes every worker and exits. Runs on the message thread's calling thread, so
1151/// — like [`run_msg_thread`] — it records the dispatcher thread's per-phase +
1152/// whole-run run-queue wait into `msg_td` (the "message thread delay" in RPS
1153/// mode, `schbench.c:1664-1670`).
1154///
1155/// DIVERGENCE from schbench's `run_rps_thread` (`schbench.c:1258-1314`),
1156/// intentional: schbench FRONT-LOADS the second (bursts all N enqueues, then
1157/// `usleep`s to fill the rest of the second) and relies on its workers
1158/// interleaving with the burst to keep each queue under the cap. That interleave
1159/// is scheduling-dependent; ktstr's burst instead front-ran the workers (the
1160/// first splice grabbed a large batch the worker drained to completion before
1161/// re-splicing), so `pending` stayed pegged and the injector dropped the
1162/// remainder every second -- a ~BATCH/worker supply cap that under-delivered the
1163/// requested rate (e.g. ~257/s at a 400/s target). ktstr instead PACES the
1164/// enqueues evenly across the second (one every `1s/N`), which forces the
1165/// injector to interleave with the workers deterministically: `pending` drains
1166/// between enqueues and the full rate is delivered up to worker capacity. Above
1167/// capacity the pacing collapses to a flat-out loop and the `BATCH` backpressure
1168/// bounds the queue, matching schbench's bounded-backlog overload behavior. The
1169/// pacing sleep and the bounded backpressure usleep are workload-defined `-R`
1170/// cadence (like the per-request think-sleep), not synchronization waits. Uses
1171/// `CLOCK_MONOTONIC` (ruling), not schbench's `gettimeofday`.
1172///
1173/// Verified by `ktstr-schbench-validate` (release, standalone), not a nextest
1174/// unit test: the under-delivery signal only appears when worker capacity exceeds
1175/// the front-load cap (~2*BATCH/s), which requires a release build (debug matrix
1176/// work caps capacity well below that) on dedicated CPU (nextest's parallel
1177/// execution starves the workers, collapsing both the paced and a hypothetical
1178/// reverted injector to the same capacity-limited rate). The fixed `-R` axis in
1179/// the schbench validation comparison is the regression evidence.
1180fn run_rps_thread(
1181 workers: &[ThreadData],
1182 msg_td: &ThreadData,
1183 stop: &AtomicBool,
1184 phase_epoch: Option<&AtomicU32>,
1185 live_rate: &AtomicUsize,
1186) {
1187 let tid = gettid_self();
1188 let mut cur_epoch = phase_epoch.map_or(0, |e| e.load(Ordering::Relaxed));
1189 let mut phase_ss_start = read_schedstat_raw(tid);
1190 // schbench's per-worker queue-depth cap before the injector backs off
1191 // (`schbench.c:1267`).
1192 const BATCH: u64 = 128;
1193 const ONE_SEC_NS: u64 = 1_000_000_000;
1194 let worker_count = workers.len().max(1);
1195 let mut cur_tid: usize = 0;
1196 while !stop.load(Ordering::Acquire) {
1197 // Re-read the live per-message-thread rate each second. `live_rate` is
1198 // already the per-thread value (schbench's post-`/= message_threads`
1199 // global `requests_per_sec`, `:1899`), injected directly with no further
1200 // division (`schbench.c:1290`). Auto-RPS retargets it from the control
1201 // thread; a fixed `-R` leaves it constant.
1202 let requests_per_sec = live_rate.load(Ordering::Relaxed);
1203 let start = monotonic_nanos();
1204 // PACE the enqueues evenly across the second rather than front-loading
1205 // them. schbench bursts all N then fills the rest of the second
1206 // (`schbench.c:1300-1306`), relying on its workers interleaving with the
1207 // burst to keep each worker's queue under the `BATCH` cap. ktstr's burst
1208 // front-ran the workers: the first splice grabbed a large batch the worker
1209 // processed to completion before re-splicing, so `pending` stayed pegged
1210 // and the injector dropped the remainder every second (a steady-state
1211 // ~BATCH/worker supply cap -> only ~257/s at a 400/s target). Pacing one
1212 // enqueue every `1s/N` forces the injector to interleave with the workers,
1213 // so `pending` drains between enqueues and the full rate is delivered up
1214 // to worker capacity. Above capacity `due` falls behind real time, the
1215 // pacing sleep collapses to zero, and the `BATCH` backpressure bounds the
1216 // queue -- matching schbench's overload behavior (a bounded backlog).
1217 let interval_ns = if requests_per_sec > 0 {
1218 ONE_SEC_NS / requests_per_sec as u64
1219 } else {
1220 ONE_SEC_NS
1221 };
1222 let mut due = start;
1223 for _ in 0..requests_per_sec {
1224 if stop.load(Ordering::Acquire) {
1225 break;
1226 }
1227 // Wait until this enqueue's scheduled time. A no-op once `due` falls
1228 // behind real time (rate above what the workers can drain), so the
1229 // loop then runs flat-out and the backpressure below bounds the queue.
1230 // `due` is re-based to `start` each second and accrues at most
1231 // N*(ONE_SEC_NS/N) ~= 1s within a pass, so saturating_add never wraps
1232 // (the saturating form documents that the per-pass re-base is the
1233 // load-bearing invariant).
1234 due = due.saturating_add(interval_ns);
1235 // Pace to this enqueue's scheduled time, but cap each sleep at
1236 // STOP_POLL_QUANTUM_NS and re-check `stop`, so a shutdown during a long
1237 // inter-arrival gap (a degenerate low rate, where the interval can
1238 // exceed the cleanup/watchdog budget) is observed within the quantum
1239 // instead of after a full interval — mirroring the taobench open-loop
1240 // pacing. A stop observed here breaks the enqueue loop BEFORE issuing
1241 // this slot's request.
1242 let mut stop_during_pace = false;
1243 loop {
1244 if stop.load(Ordering::Acquire) {
1245 stop_during_pace = true;
1246 break;
1247 }
1248 let now = monotonic_nanos();
1249 if now >= due {
1250 break;
1251 }
1252 let nap = (due - now).min(STOP_POLL_QUANTUM_NS);
1253 std::thread::sleep(std::time::Duration::from_nanos(nap));
1254 }
1255 if stop_during_pace {
1256 break;
1257 }
1258 let worker = &workers[cur_tid % worker_count];
1259 cur_tid += 1;
1260 // Backpressure: don't queue more to a worker already `BATCH` deep
1261 // (`schbench.c:1284-1290`). The bounded usleep is schbench's injector
1262 // throttle (workload behavior), not a sync poll. The round-robin
1263 // cursor already advanced, so the skipped slot moves to the next
1264 // worker, matching schbench (the request for this slot is dropped).
1265 if worker.pending.load(Ordering::Acquire) > BATCH {
1266 std::thread::sleep(std::time::Duration::from_micros(100));
1267 continue;
1268 }
1269 worker.pending.fetch_add(1, Ordering::AcqRel);
1270 // schbench mallocs one request per enqueue; the worker frees it after
1271 // processing (`schbench.c:1292`/`:1476`). Box matches that profile.
1272 let req = Box::into_raw(Box::new(Request::new()));
1273 worker.requests.add(req);
1274 // Stamp the target's wake_time so it measures wakeup latency as
1275 // now - enqueue (`schbench.c:1294`), then post it.
1276 worker.wake_time.store(monotonic_nanos(), Ordering::Release);
1277 worker.futex.post();
1278 }
1279 // The pacing loop already spans ~1 second; top up only if it finished
1280 // early (rate 0, or `stop`), preserving schbench's per-second cadence. Cap
1281 // each sleep at STOP_POLL_QUANTUM_NS and re-check `stop` so a shutdown
1282 // during the top-up is observed within the quantum, not after a full
1283 // remaining second (the same teardown bound as the pacing sleep above).
1284 loop {
1285 if stop.load(Ordering::Acquire) {
1286 break;
1287 }
1288 let elapsed = monotonic_nanos().saturating_sub(start);
1289 if elapsed >= ONE_SEC_NS {
1290 break;
1291 }
1292 let nap = (ONE_SEC_NS - elapsed).min(STOP_POLL_QUANTUM_NS);
1293 std::thread::sleep(std::time::Duration::from_nanos(nap));
1294 }
1295 // Dispatcher per-phase run-delay drain (mirrors run_msg_thread): on epoch
1296 // change, finalize this thread's run-delay for the ended phase. Empty
1297 // histograms (the dispatcher records only run-delay).
1298 if let Some(pe) = phase_epoch {
1299 let new_epoch = pe.load(Ordering::Relaxed);
1300 if new_epoch != cur_epoch {
1301 let ss_end = read_schedstat_raw(tid);
1302 // SAFETY: this thread owns msg_td (the dispatcher's ThreadData).
1303 unsafe { drain_phase(msg_td, cur_epoch, phase_ss_start, ss_end, 0, 0) };
1304 cur_epoch = new_epoch;
1305 phase_ss_start = ss_end;
1306 }
1307 }
1308 if stop.load(Ordering::Acquire) {
1309 // Wake every worker so a blocked one observes stop and exits
1310 // (`schbench.c:1308-1312`).
1311 for w in workers {
1312 w.futex.post();
1313 }
1314 break;
1315 }
1316 }
1317 // Final drain + whole-run mean run-delay for the dispatcher thread, matching
1318 // run_msg_thread's exit (`schbench.c:1664`).
1319 let ss_end = read_schedstat_raw(tid);
1320 // SAFETY: this thread owns msg_td.
1321 unsafe { drain_phase(msg_td, cur_epoch, phase_ss_start, ss_end, 0, 0) };
1322 // SAFETY: owner-only access to msg_td's sched_delay_ns cell.
1323 unsafe { *msg_td.sched_delay_ns.get() = mean_sched_delay(ss_end) };
1324}
1325
1326/// The calling thread's kernel tid (`gettid`), for reading its own schedstat.
1327fn gettid_self() -> libc::pid_t {
1328 // SAFETY: gettid takes no arguments and only returns the caller's tid.
1329 unsafe { libc::syscall(libc::SYS_gettid) as libc::pid_t }
1330}
1331
1332/// Raw `(run_delay_ns, pcount)` pair from `/proc/<tid>/schedstat` (fields 2, 3),
1333/// UNDIVIDED — schbench's `read_sched_delay` source (`schbench.c:1118`) without
1334/// the division. Both consumers want the raw pair, not a pre-divided mean: the
1335/// per-phase path re-pools `Σrun_delay / Σpcount` host-side, and the whole-run
1336/// per-thread mean is [`mean_sched_delay`] of the thread's final cumulative
1337/// pair. An absent file (the thread exited) yields `(0, 0)`, like schbench's
1338/// fopen-failure path.
1339///
1340/// `CONFIG_SCHED_INFO` (selected by `CONFIG_SCHEDSTATS`, and also by
1341/// `CONFIG_TASK_DELAY_ACCT` -- ktstr.kconfig enables both) populates these
1342/// fields even under sched_ext and regardless of the `kernel.sched_schedstats`
1343/// sysctl (that sysctl gates the separate per-rq/domain counters, not
1344/// `sched_info`).
1345fn read_schedstat_raw(tid: libc::pid_t) -> (u64, u64) {
1346 match std::fs::read_to_string(format!("/proc/{tid}/schedstat")) {
1347 Ok(s) => parse_schedstat_raw(&s),
1348 // The thread may have exited; schbench's fopen-failure path also -> 0.
1349 Err(_) => (0, 0),
1350 }
1351}
1352
1353/// Parse a `/proc/<tid>/schedstat` line into the raw `(run_delay_ns, pcount)`
1354/// pair (field 2 = run_delay ns, field 3 = pcount = timeslices). The kernel's
1355/// `proc_pid_schedstat` always emits three integer fields, so a malformed
1356/// *present* line is a kernel/parse bug: panic rather than silently report 0
1357/// (matching the handshake's fail-loud stance + the no-silent-wrong-answer
1358/// rule). An absent file is the caller's concern, handled as `(0, 0)`.
1359fn parse_schedstat_raw(s: &str) -> (u64, u64) {
1360 let mut fields = s.split_whitespace();
1361 let _run = fields.next(); // field 1: sum_exec_runtime (on-CPU ns), unused
1362 let run_delay: u64 = fields
1363 .next()
1364 .and_then(|f| f.parse().ok())
1365 .expect("schedstat field 2 (run_delay) must be a present integer");
1366 let pcount: u64 = fields
1367 .next()
1368 .and_then(|f| f.parse().ok())
1369 .expect("schedstat field 3 (pcount) must be a present integer");
1370 (run_delay, pcount)
1371}
1372
1373/// Mean per-schedule run-queue wait (ns) from a raw `(run_delay_ns, pcount)`
1374/// pair: `run_delay / pcount`, schbench's `read_sched_delay` arithmetic
1375/// (`schbench.c:1146`). Guards `pcount == 0` (a never-scheduled thread, or the
1376/// `!sched_info_on()` "0 0 0" line) to 0, where schbench divides by zero. Used
1377/// for the WHOLE-RUN per-thread mean (the mean-of-means component); the
1378/// per-phase path instead keeps the raw pair and re-pools `Σrd/Σpc` host-side.
1379fn mean_sched_delay((run_delay, pcount): (u64, u64)) -> u64 {
1380 if pcount == 0 { 0 } else { run_delay / pcount }
1381}
1382
1383/// Finalize the just-ended phase on the OWNING thread: `take` the live
1384/// histograms (snapshot-and-reset, so the next phase starts clean) and push a
1385/// [`PhaseSnapshot`] tagged `epoch`, carrying the schedstat run-delay/pcount
1386/// delta (`ss_end − ss_start`) and this thread's `loop_count` for the phase.
1387/// Mirrors the worker/mod.rs:3858-3894 backdrop drain-on-change. For the
1388/// message thread the histograms are unwritten (taken empty) and `loop_count`
1389/// is 0; an empty histogram folds harmlessly host-side ([`PlatStats::combine`]).
1390///
1391/// # Safety
1392/// Must be called by the thread that owns `td` — it is the sole writer of
1393/// `td`'s `wakeup_stats` / `request_stats` / `phase_snapshots` cells (the
1394/// [`ThreadData`] `Sync` contract).
1395unsafe fn drain_phase(
1396 td: &ThreadData,
1397 epoch: u32,
1398 ss_start: (u64, u64),
1399 ss_end: (u64, u64),
1400 loop_count: u64,
1401 cpu_ns: u64,
1402) {
1403 // SAFETY: owner-only cell access, per this function's contract.
1404 unsafe {
1405 let wakeup = (*td.wakeup_stats.get()).take();
1406 let request = (*td.request_stats.get()).take();
1407 (*td.phase_snapshots.get()).push(PhaseSnapshot {
1408 epoch,
1409 wakeup,
1410 request,
1411 run_delay_ns: ss_end.0.saturating_sub(ss_start.0),
1412 pcount: ss_end.1.saturating_sub(ss_start.1),
1413 loop_count,
1414 cpu_ns,
1415 });
1416 }
1417}
1418
1419/// The per-message-thread shared context every worker borrows — bundled so
1420/// [`worker_loop`]'s signature stays within clippy's argument budget. All
1421/// fields are `Copy` shared references; every worker spawned by one
1422/// [`run_one_message_thread`] reads the same instance.
1423struct WorkerCtx<'a> {
1424 msg_td: &'a ThreadData,
1425 wait_list: &'a TreiberStack<ThreadData>,
1426 locks: Option<&'a PerCpuLocks>,
1427 config: &'a SchbenchConfig,
1428 stop: &'a AtomicBool,
1429 progress: &'a AtomicU64,
1430 phase_epoch: Option<&'a AtomicU32>,
1431 /// The ONE process-global shared matrix (`--split` mode), shared across
1432 /// every worker of every message thread; empty slice when not splitting.
1433 shared: &'a [core::sync::atomic::AtomicU64],
1434 shared_matrix_size: usize,
1435}
1436
1437/// One worker thread's loop. Faithful to schbench's `worker_thread`
1438/// (`schbench.c:1419`, default branch): block until woken (recording wakeup
1439/// latency), then think-sleep + matrix work under the per-CPU lock (recording
1440/// request latency), until stop. `ctx.progress` counts completed work cycles
1441/// across all workers (the live cycle counter / achieved request rate).
1442///
1443/// `ctx.phase_epoch` (when `Some`) is the shared per-phase generation the
1444/// scenario engine bumps at each step boundary. The worker polls it once per
1445/// cycle and, on change, drains the just-ended phase ([`drain_phase`]) —
1446/// `take`ing its own histograms + a schedstat raw delta — so per-phase
1447/// percentiles are isolated across e.g. an scx→detached-EEVDF transition. A
1448/// final drain at exit closes the last phase (and is the SOLE snapshot when
1449/// non-phasic, `cur_epoch` 0).
1450fn worker_loop(td: &ThreadData, ctx: &WorkerCtx) {
1451 // Destructure the all-`Copy` context into the same bindings the body used
1452 // before the bundle, so the loop logic below reads unchanged.
1453 let WorkerCtx {
1454 msg_td,
1455 wait_list,
1456 locks,
1457 config,
1458 stop,
1459 progress,
1460 phase_epoch,
1461 shared,
1462 shared_matrix_size,
1463 } = *ctx;
1464 let tid = gettid_self();
1465 // Per-worker PRIVATE matrix (schbench.c:1568-1574: private_matrix_size when
1466 // splitting, else the full matrix_size). The shared matrix (split mode) is
1467 // the process-global `shared` allocated once in run().
1468 let private_matrix_size = config.private_matrix_size();
1469 let mut private_buf = if private_matrix_size > 0 {
1470 vec![0u64; 3 * private_matrix_size * private_matrix_size]
1471 } else {
1472 Vec::new()
1473 };
1474 let mut work_units = 0u64;
1475 // Per-phase drain-on-change state. `cur_epoch` is the epoch the current
1476 // phase's samples belong to; `phase_ss_start` baselines the schedstat raw
1477 // pair so each phase reports its own delta.
1478 let mut cur_epoch = phase_epoch.map_or(0, |e| e.load(Ordering::Relaxed));
1479 let mut phase_ss_start = read_schedstat_raw(tid);
1480 let mut phase_loop_count = 0u64;
1481 // CLOCK_THREAD_CPUTIME_ID baseline for the per-phase worker CPU delta
1482 // (the `schbench_loops_per_cpu_sec` denominator). Re-baselined at each
1483 // drain like `phase_ss_start`.
1484 let mut phase_cpu_start = crate::workload::worker::sched::thread_cpu_time_ns();
1485 // RPS mode iff the per-thread rate is non-zero (the `-R` total covers at
1486 // least one request/sec per message thread). Below that it rounds to 0 and
1487 // the worker runs the default message-handshake mode -- matching schbench's
1488 // startup-divide-before-gate ([`SchbenchConfig::rps_per_message_thread`]).
1489 // Pipe mode (`-p`): the memory-transfer workload. It runs the message-handshake
1490 // path (where the pipe-page memsets live), NOT the RPS injector -- a ktstr
1491 // divergence; schbench composes -p+-R half-broken (see the `pipe_transfer_bytes`
1492 // field doc).
1493 let pipe_mode = config.pipe_transfer_bytes > 0;
1494 let rps_mode = !pipe_mode && config.rps_per_message_thread() != 0;
1495 while !stop.load(Ordering::Acquire) {
1496 // Acquire work. Default mode: the message-thread handshake (records
1497 // wakeup latency; no request chain -> a single work cycle). RPS mode:
1498 // splice this worker's request queue, blocking on its own futex until
1499 // the RPS thread posts; returns the spliced chain (FIFO) to drain.
1500 let mut req: *mut Request = if rps_mode {
1501 let chain = rps_wait(td, stop);
1502 if stop.load(Ordering::Acquire) {
1503 // Free any spliced-but-unprocessed requests, then exit.
1504 free_request_chain(chain);
1505 break;
1506 }
1507 if chain.is_null() {
1508 continue; // spurious wake with an empty queue
1509 }
1510 chain
1511 } else {
1512 msg_and_wait(td, msg_td, wait_list, stop);
1513 if stop.load(Ordering::Acquire) {
1514 break;
1515 }
1516 ptr::null_mut()
1517 };
1518 // Process the work: one cycle in default mode (`req` null), or one cycle
1519 // per queued request in RPS mode (freeing each). `work_start` is stamped
1520 // before the think-sleep so the request latency covers think-time +
1521 // matrix work (`schbench.c:1464-1481`).
1522 loop {
1523 let work_start = monotonic_nanos();
1524 // Pipe mode: the per-cycle transfer is the wake/block memsets
1525 // (msg_and_wait fills our page to 2; wake_all fills it to 1). The work
1526 // loop only counts cycles for the MB/s throughput; schbench likewise
1527 // skips do_work + the think-sleep in pipe mode (schbench.c:1448).
1528 if !pipe_mode {
1529 if config.sleep_usec > 0 {
1530 think_sleep(config.sleep_usec);
1531 }
1532 do_work(
1533 &mut private_buf,
1534 private_matrix_size,
1535 shared,
1536 shared_matrix_size,
1537 config.split_percent,
1538 config.operations,
1539 locks,
1540 &mut work_units,
1541 );
1542 }
1543 let now = monotonic_nanos();
1544 let delta_us = now.saturating_sub(work_start) / 1000;
1545 if delta_us > 0 {
1546 // SAFETY: only this worker thread accesses its own request_stats cell.
1547 unsafe { (*td.request_stats.get()).add_lat(delta_us.min(u32::MAX as u64) as u32) };
1548 }
1549 progress.fetch_add(1, Ordering::Relaxed);
1550 phase_loop_count += 1;
1551 if req.is_null() {
1552 break; // default mode: a single work cycle
1553 }
1554 // RPS mode: advance to the next queued request and free this one.
1555 // SAFETY: `req` was `Box::into_raw`'d by the RPS thread and handed to
1556 // this worker exclusively via the queue splice; we own it and free it
1557 // exactly once here.
1558 let next = unsafe { (*req).next.load(Ordering::Acquire) };
1559 drop(unsafe { Box::from_raw(req) });
1560 req = next;
1561 if req.is_null() {
1562 break;
1563 }
1564 }
1565 // Drain-on-change: when the parent advances `phase_epoch`, finalize the
1566 // phase just ended (tagged with the OLD epoch the samples were recorded
1567 // under) and re-baseline. A worker blocked across a whole phase simply
1568 // contributes an empty histogram for it (no cycle ran), which folds
1569 // harmlessly.
1570 if let Some(pe) = phase_epoch {
1571 let new_epoch = pe.load(Ordering::Relaxed);
1572 if new_epoch != cur_epoch {
1573 let ss_end = read_schedstat_raw(tid);
1574 let cpu_end = crate::workload::worker::sched::thread_cpu_time_ns();
1575 // SAFETY: this is the thread that owns `td`.
1576 unsafe {
1577 drain_phase(
1578 td,
1579 cur_epoch,
1580 phase_ss_start,
1581 ss_end,
1582 phase_loop_count,
1583 cpu_end.saturating_sub(phase_cpu_start),
1584 )
1585 };
1586 cur_epoch = new_epoch;
1587 phase_ss_start = ss_end;
1588 phase_loop_count = 0;
1589 phase_cpu_start = cpu_end;
1590 }
1591 }
1592 }
1593 // Shutdown wake: the loop exits on `stop` without a final `msg_and_wait`, so
1594 // the message thread may still be parked in `wait_forever` (its only waker is
1595 // a worker post). schbench's MAIN thread handles this -- it fposts each
1596 // message thread after `stopping = 1` (`schbench.c:1832`, `:1933`) -- but our
1597 // `run()` joins passively and `msg_td` is private to `run_one_message_thread`,
1598 // so the exiting worker wakes its own message thread instead. Without this, a
1599 // worker that exits mid-`do_work` leaves the message thread parked forever and
1600 // `run()` deadlocks. Idempotent across workers: the first post the message
1601 // thread observes makes it re-check `stop` and break; later posts are no-ops.
1602 // In RPS mode this is itself a harmless no-op: the dispatcher runs
1603 // `run_rps_thread` (which never parks on `msg_td`) and observes `stop` itself.
1604 msg_td.futex.post();
1605 // Final drain: close the still-open phase. When non-phasic this is the only
1606 // snapshot (`cur_epoch` 0), so run() builds the whole-run result uniformly
1607 // from snapshots.
1608 let ss_end = read_schedstat_raw(tid);
1609 let cpu_end = crate::workload::worker::sched::thread_cpu_time_ns();
1610 // SAFETY: this is the thread that owns `td`.
1611 unsafe {
1612 drain_phase(
1613 td,
1614 cur_epoch,
1615 phase_ss_start,
1616 ss_end,
1617 phase_loop_count,
1618 cpu_end.saturating_sub(phase_cpu_start),
1619 )
1620 };
1621 // Record this worker's whole-run mean run-queue wait at exit — the
1622 // mean-of-means component for the whole-run SchbenchResult (schbench reads
1623 // each thread's schedstat for the final aggregate, `schbench.c:1664-1670`).
1624 // Reuses the cumulative `ss_end` pair just read (one /proc read; mean and
1625 // final-phase delta share one consistent snapshot).
1626 // SAFETY: owner-only access to this thread's sched_delay_ns cell.
1627 unsafe { *td.sched_delay_ns.get() = mean_sched_delay(ss_end) };
1628}
1629
1630/// Resolve the worker-thread default and the per-CPU lock-array size from the
1631/// calling thread's CPU affinity (the allocated cpuset, per ruling). Returns
1632/// `(allowed_cpu_count, lock_array_size)` where the array size is the highest
1633/// allowed CPU id + 1, so `sched_getcpu` indexes it without clamping even on a
1634/// sparse cpuset.
1635fn resolve_cpu_topology() -> (usize, usize) {
1636 // SAFETY: a zeroed cpu_set_t filled by sched_getaffinity for the calling
1637 // thread (pid 0); CPU_ISSET only reads it.
1638 unsafe {
1639 let mut set: libc::cpu_set_t = core::mem::zeroed();
1640 let rc = libc::sched_getaffinity(0, core::mem::size_of::<libc::cpu_set_t>(), &mut set);
1641 if rc != 0 {
1642 return (1, 1);
1643 }
1644 let mut count = 0usize;
1645 let mut max_id = 0usize;
1646 for cpu in 0..libc::CPU_SETSIZE as usize {
1647 if libc::CPU_ISSET(cpu, &set) {
1648 count += 1;
1649 max_id = cpu;
1650 }
1651 }
1652 (count.max(1), (max_id + 1).max(1))
1653 }
1654}
1655
1656/// Combined WHOLE-RUN results of a schbench run: the merged wakeup + request
1657/// latency percentiles and the achieved request rate (completed work
1658/// cycles/second). The histograms are the union of every phase; `sched_delay_*`
1659/// keep schbench's mean-of-means (`collect_sched_delay`) so the side-by-side
1660/// validation matches real schbench's reported number.
1661pub(crate) struct SchbenchResult {
1662 pub(crate) wakeup: Percentiles,
1663 pub(crate) request: Percentiles,
1664 /// Per-second achieved-RPS distribution (schbench's `rps_stats`), sampled
1665 /// once per second by the control thread. For auto-RPS, samples are gated
1666 /// until the target is hit (`schbench.c:1781`); for fixed/default mode every
1667 /// second is sampled.
1668 pub(crate) rps: Percentiles,
1669 pub(crate) loop_count: u64,
1670 /// Resolved total worker-thread count (`message_threads * worker_threads`).
1671 /// Divisor for schbench's PER-WORKER pipe-mode rate (`avg worker transfer`):
1672 /// schbench reports `loop_count / Σ worker runtimes` (`schbench.c:1697`,
1673 /// `:1942-1943`/`:1979`), and `Σ worker runtimes ≈ nr_workers * elapsed`, so
1674 /// the per-worker rate is `achieved_rps / nr_workers`.
1675 pub(crate) nr_workers: usize,
1676 /// Aggregate completed cycles / WALL second over the run window.
1677 /// Deliberately wall-denominated: this feeds ONLY the standalone /
1678 /// validate surface (`run_standalone` → `ktstr-schbench-validate`),
1679 /// whose output must stay byte-identical to upstream schbench's
1680 /// wall-clock report. The ktstr sidecar's throughput metric is the
1681 /// CPU-second-denominated `schbench_loops_per_cpu_sec` (from the
1682 /// per-phase `worker_cpu_ns` carrier), not this field.
1683 pub(crate) achieved_rps: f64,
1684 /// Auto-RPS final TOTAL target rate at run exit: the live per-message-thread
1685 /// rate * message_threads (schbench's `requests_per_sec * message_threads`,
1686 /// `schbench.c:1995`). For fixed `-R` and default mode this equals the seeded
1687 /// total; only auto-RPS makes it diverge as the control loop retargets.
1688 pub(crate) final_rps_goal: usize,
1689 /// Mean message-thread run-queue wait (ns), averaged across message threads
1690 /// (schbench's `message_thread_delay`, `schbench.c:1673`).
1691 pub(crate) sched_delay_msg_ns: u64,
1692 /// Mean worker-thread run-queue wait (ns), averaged across all workers
1693 /// (schbench's `worker_thread_delay`, `schbench.c:1674`).
1694 pub(crate) sched_delay_worker_ns: u64,
1695}
1696
1697/// The full result of [`run`]: the whole-run [`SchbenchResult`] plus the
1698/// per-phase aggregates keyed by `phase_epoch` (each the cross-thread merge for
1699/// one scenario step's HOLD window). When non-phasic (`phase_epoch == None`)
1700/// `phases` holds the single `(0, ..)` baseline entry the host discards; tests
1701/// read `whole_run`. The worker dispatch turns each `(epoch, SchbenchPhaseStats)`
1702/// into a `PhaseSlice`.
1703pub(crate) struct SchbenchOutcome {
1704 pub(crate) whole_run: SchbenchResult,
1705 pub(crate) phases: Vec<(u32, SchbenchPhaseStats)>,
1706}
1707
1708/// One message thread's pooled results, returned from [`run_one_message_thread`]
1709/// to [`run`]: the whole-run histograms (Σ over its workers' phase snapshots),
1710/// the run-delay components for the whole-run mean-of-means, and the per-epoch
1711/// aggregate keyed by `phase_epoch`.
1712struct MessageThreadResult {
1713 whole_wakeup: PlatStats,
1714 whole_request: PlatStats,
1715 msg_sched_delay_ns: u64,
1716 workers_sched_delay_sum: u64,
1717 phases: std::collections::BTreeMap<u32, SchbenchPhaseStats>,
1718}
1719
1720/// Run one message thread plus its workers, returning the whole-run histograms
1721/// AND the per-epoch aggregate. The message thread runs on the calling thread;
1722/// workers are scoped so they are joined before their owner-only phase snapshots
1723/// are drained.
1724#[allow(clippy::too_many_arguments)]
1725fn run_one_message_thread(
1726 worker_threads: usize,
1727 locks: Option<&PerCpuLocks>,
1728 config: &SchbenchConfig,
1729 stop: &AtomicBool,
1730 progress: &AtomicU64,
1731 phase_epoch: Option<&AtomicU32>,
1732 live_rate: &AtomicUsize,
1733 shared: &[core::sync::atomic::AtomicU64],
1734 shared_matrix_size: usize,
1735) -> MessageThreadResult {
1736 // schbench clamps -p to PIPE_TRANSFER_BUFFER (schbench.c:291-294). Workers get
1737 // the per-thread transfer page; the message thread (waker) needs none.
1738 let pipe_bytes = config.pipe_transfer_bytes.min(PIPE_TRANSFER_BUFFER);
1739 let workers: Vec<ThreadData> = (0..worker_threads)
1740 .map(|_| ThreadData::new(pipe_bytes))
1741 .collect();
1742 let msg_td = ThreadData::new(0);
1743 let wait_list = TreiberStack::new();
1744 let ctx = WorkerCtx {
1745 msg_td: &msg_td,
1746 wait_list: &wait_list,
1747 locks,
1748 config,
1749 stop,
1750 progress,
1751 phase_epoch,
1752 shared,
1753 shared_matrix_size,
1754 };
1755
1756 std::thread::scope(|inner| {
1757 for w in &workers {
1758 inner.spawn(|| worker_loop(w, &ctx));
1759 }
1760 // Dispatcher: RPS-injector mode runs the rate-driven request thread;
1761 // default mode runs the message-thread handshake. Both run on this
1762 // calling thread and record the dispatcher's run-delay into `msg_td`
1763 // (schbench `message_thread`, `:1594`).
1764 // Pipe mode (`-p`) uses the message-handshake waker (run_msg_thread does
1765 // the per-worker pipe-page fill in wake_all) and takes precedence over -R.
1766 // This DIVERGES from schbench, where -R alone picks the path
1767 // (`schbench.c:1594`): schbench's `-p -R` runs the RPS injector (no
1768 // xlist_wake_all -> no waker-side memset -> a half-pipe). ktstr runs the
1769 // full pipe; the realistic use is `-p` alone. Otherwise -R selects the
1770 // injector.
1771 if config.pipe_transfer_bytes == 0 && config.rps_per_message_thread() != 0 {
1772 run_rps_thread(&workers, &msg_td, stop, phase_epoch, live_rate);
1773 } else {
1774 run_msg_thread(&msg_td, &wait_list, stop, phase_epoch);
1775 }
1776 // Stop is set: wake every worker so a blocked one observes stop and
1777 // exits (schbench fposts each worker before joining, `:1599-1602`).
1778 for w in &workers {
1779 w.futex.post();
1780 }
1781 // The inner scope joins the workers here.
1782 });
1783
1784 // RPS mode: free any requests the RPS thread enqueued after a worker's last
1785 // splice (the worker exited on stop without draining them). After the join,
1786 // this is the sole access to each worker's queue, so no request leaks across
1787 // run() calls.
1788 if config.rps_per_message_thread() != 0 {
1789 for w in &workers {
1790 free_request_chain(w.requests.splice_reversed());
1791 }
1792 }
1793
1794 // After join (happens-before): drain each thread's owner-only phase
1795 // snapshots into the whole-run histograms + the per-epoch aggregate.
1796 let mut whole_wakeup = PlatStats::default();
1797 let mut whole_request = PlatStats::default();
1798 let mut workers_sched_delay_sum = 0u64;
1799 let mut phases: std::collections::BTreeMap<u32, SchbenchPhaseStats> =
1800 std::collections::BTreeMap::new();
1801 for w in &workers {
1802 // SAFETY: every worker has joined (inner scope ended), so this is the
1803 // sole access to their cells.
1804 unsafe {
1805 workers_sched_delay_sum =
1806 workers_sched_delay_sum.saturating_add(*w.sched_delay_ns.get());
1807 for snap in (*w.phase_snapshots.get()).drain(..) {
1808 whole_wakeup.combine(&snap.wakeup);
1809 whole_request.combine(&snap.request);
1810 let e = phases.entry(snap.epoch).or_default();
1811 e.wakeup.combine(&snap.wakeup);
1812 e.request.combine(&snap.request);
1813 e.worker_run_delay_ns = e.worker_run_delay_ns.saturating_add(snap.run_delay_ns);
1814 e.worker_pcount = e.worker_pcount.saturating_add(snap.pcount);
1815 e.loop_count = e.loop_count.saturating_add(snap.loop_count);
1816 e.worker_cpu_ns = e.worker_cpu_ns.saturating_add(snap.cpu_ns);
1817 }
1818 }
1819 }
1820 // SAFETY: run_msg_thread ran on this thread inside the scope above (before
1821 // the join), so msg_td's cells are settled and this is sole access.
1822 let msg_sched_delay = unsafe { *msg_td.sched_delay_ns.get() };
1823 // SAFETY: same — sole access to msg_td's snapshot cell after the scope.
1824 unsafe {
1825 for snap in (*msg_td.phase_snapshots.get()).drain(..) {
1826 let e = phases.entry(snap.epoch).or_default();
1827 e.msg_run_delay_ns = e.msg_run_delay_ns.saturating_add(snap.run_delay_ns);
1828 e.msg_pcount = e.msg_pcount.saturating_add(snap.pcount);
1829 }
1830 }
1831 MessageThreadResult {
1832 whole_wakeup,
1833 whole_request,
1834 msg_sched_delay_ns: msg_sched_delay,
1835 workers_sched_delay_sum,
1836 phases,
1837 }
1838}
1839
1840/// Persistent /proc/stat reader state for auto-RPS, carrying the previous
1841/// cumulative totals so each call computes a delta over the interval (schbench
1842/// keeps one fd + the prior total/idle, `schbench.c` `read_busy`, `:1046`).
1843#[derive(Default)]
1844struct ReadBusyState {
1845 fd: Option<File>,
1846 prev_total: u64,
1847 prev_idle: u64,
1848}
1849
1850/// Host-busy percentage over the interval since the last call, from the
1851/// aggregate `cpu` line of /proc/stat (schbench `read_busy`,
1852/// `schbench.c:1046-1112`). `None` means "no usable reading this tick": the first
1853/// call (baseline seed, schbench's `first_run`) OR a zero jiffy-delta (two reads
1854/// in the same tick -- schbench has no guard and divides by zero into a NaN that
1855/// spuriously trips its target-hit `else` branch; we skip the tick instead). A
1856/// real reading is `Some(0.0..=100.0)`, including a genuine fully-idle `Some(0.0)`
1857/// -- distinguished from the seed via the `prev_total` state, NOT via the value,
1858/// so an idle interval still drives a grow (the bug a `busy == 0.0` test had). In
1859/// the guest VM /proc/stat covers exactly the allocated cpuset (the guest has
1860/// only `cores=N`), so this system-wide read is correctly scoped to the
1861/// workload's CPUs. f32 to match schbench's `float` arithmetic.
1862fn read_busy(s: &mut ReadBusyState) -> Option<f32> {
1863 if s.fd.is_none() {
1864 s.fd = File::open("/proc/stat").ok();
1865 }
1866 let f = s.fd.as_mut()?;
1867 f.seek(SeekFrom::Start(0)).ok()?;
1868 let mut buf = [0u8; 512];
1869 let n = f.read(&mut buf).ok()?;
1870 let text = core::str::from_utf8(&buf[..n]).ok()?;
1871 // First line only; first token must be "cpu" (the aggregate line). Sum every
1872 // numeric field -> total; field index 3 (after "cpu") is idle -- kernel
1873 // show_stat order is user/nice/system/IDLE/iowait/irq/softirq/steal/guest/
1874 // guest_nice. `split_whitespace` collapses the "cpu " double space, matching
1875 // schbench's `strtok_r` (`schbench.c:1082-1096`).
1876 let line = text.lines().next().unwrap_or("");
1877 let mut fields = line.split_whitespace();
1878 if fields.next() != Some("cpu") {
1879 return None;
1880 }
1881 let mut total = 0u64;
1882 let mut idle = 0u64;
1883 for (i, tok) in fields.enumerate() {
1884 let v: u64 = tok.parse().unwrap_or(0);
1885 if i == 3 {
1886 idle = v;
1887 }
1888 total += v;
1889 }
1890 // First call: seed the baseline, no busy% yet (`schbench.c:1098-1101`).
1891 if s.prev_total == 0 {
1892 s.prev_total = total;
1893 s.prev_idle = idle;
1894 return None;
1895 }
1896 let dt = total.saturating_sub(s.prev_total);
1897 let di = idle.saturating_sub(s.prev_idle);
1898 s.prev_total = total;
1899 s.prev_idle = idle;
1900 if dt == 0 {
1901 // Zero jiffy-delta: skip rather than NaN (see the doc; schbench diverges).
1902 return None;
1903 }
1904 Some(100.0 - (di as f32 / dt as f32) * 100.0)
1905}
1906
1907/// Grow or shrink the live per-message-thread request rate toward `target_pct`
1908/// host-busy, given an already-read `busy` percentage (schbench `auto_scale_rps`'s
1909/// scaling body, `schbench.c:1203-1250`). Pure: no I/O, so the scaling decision is
1910/// unit-tested with injected `busy` values. Stores the new per-thread rate into
1911/// `live_rate` -- schbench scales its global `requests_per_sec` (already per-thread
1912/// after `:1899`) and writes it back at `:1250`. Returns true on the transition
1913/// INTO the near-target band (when `target_hit` flips false->true), so the caller
1914/// resets the rps histogram (schbench memsets `rps_stats` on that transition). The
1915/// near-target test reads the delta AFTER its damping reassignment, exactly as
1916/// schbench does (`:1210`,`:1233`). f32 math + `ceil` (grow) / `floor` (shrink)
1917/// match schbench's float deltas + rounding.
1918fn scale_rps_for_busy(
1919 busy: f32,
1920 live_rate: &AtomicUsize,
1921 target_hit: &AtomicBool,
1922 target_pct: usize,
1923) -> bool {
1924 let target = target_pct as f32;
1925 let rps = live_rate.load(Ordering::Relaxed) as f32;
1926 let already_hit = target_hit.load(Ordering::Relaxed);
1927 let mut just_hit = false;
1928 let new_rate: usize = if busy < target {
1929 // Under target: grow (`schbench.c:1203-1224`).
1930 let mut delta = target / busy;
1931 if delta > 3.0 {
1932 delta = 3.0;
1933 } else if delta < 1.2 {
1934 delta = 1.0 + (delta - 1.0) / 8.0;
1935 // Threshold check AFTER the damping reassignment (`schbench.c:1209-1213`).
1936 if delta < 1.05 && !already_hit {
1937 just_hit = true;
1938 }
1939 } else if delta < 1.5 {
1940 delta = 1.0 + (delta - 1.0) / 4.0;
1941 }
1942 let t = (rps * delta).ceil();
1943 // Not enough capacity to reach the target -> hold (`schbench.c:1219-1224`).
1944 if t >= (1u64 << 31) as f32 {
1945 rps as usize
1946 } else {
1947 t as usize
1948 }
1949 } else if busy > target {
1950 // Over target: shrink (`schbench.c:1226-1242`).
1951 let mut delta = target / busy;
1952 if delta < 0.3 {
1953 delta = 0.3;
1954 } else if delta > 0.9 {
1955 delta += (1.0 - delta) / 8.0;
1956 // Threshold check AFTER the damping reassignment (`schbench.c:1232-1236`).
1957 if delta > 0.95 && !already_hit {
1958 just_hit = true;
1959 }
1960 } else if delta > 0.8 {
1961 delta += (1.0 - delta) / 4.0;
1962 }
1963 (rps * delta).floor().max(0.0) as usize
1964 } else {
1965 // Exactly on target (`schbench.c:1243-1248`).
1966 if !already_hit {
1967 just_hit = true;
1968 }
1969 rps as usize
1970 };
1971 live_rate.store(new_rate, Ordering::Relaxed);
1972 if just_hit {
1973 target_hit.store(true, Ordering::Relaxed);
1974 }
1975 just_hit
1976}
1977
1978/// Read host-busy from /proc/stat and scale the live rate toward the target
1979/// (schbench `auto_scale_rps`, `schbench.c:1180-1251`). The first read only seeds
1980/// the baseline (schbench's `first_run` early return, `:1201`); a zero-delta tick
1981/// is skipped (see [`read_busy`]). Returns [`scale_rps_for_busy`]'s just-hit flag.
1982fn auto_scale_rps(
1983 busy_state: &mut ReadBusyState,
1984 live_rate: &AtomicUsize,
1985 target_hit: &AtomicBool,
1986 target_pct: usize,
1987) -> bool {
1988 let Some(busy) = read_busy(busy_state) else {
1989 return false;
1990 };
1991 scale_rps_for_busy(busy, live_rate, target_hit, target_pct)
1992}
1993
1994/// Reset BOTH per-phase rps accumulators in LOCKSTEP at an auto-RPS target-hit.
1995/// schbench memsets `rps_stats` on the target-hit transition to discard the
1996/// pre-target ramp; the per-epoch map MUST reset with it, or the per-phase view
1997/// keeps ramp samples the whole-run view dropped and `Σ per-epoch == whole-run`
1998/// breaks (asymmetric-counter-bookkeeping).
1999fn reset_rps_accumulators(
2000 whole: &mut PlatStats,
2001 per_epoch: &mut std::collections::BTreeMap<u32, PlatStats>,
2002) {
2003 *whole = PlatStats::default();
2004 per_epoch.clear();
2005}
2006
2007/// The once-per-second control thread, schbench's main control loop
2008/// (`schbench.c:1756-1827`): sample the achieved per-second RPS into `rps_stats`
2009/// and -- when auto-RPS is on -- adjust the live rate toward the target. Returns
2010/// BOTH the whole-run `rps_stats` AND a per-epoch RPS map (the same samples
2011/// bucketed by the phase epoch observed at sample time), moved out on join. The
2012/// whole-run `rps_stats` is byte-identical to the pre-per-phase behavior -- the
2013/// per-epoch map is PURELY ADDITIVE (each sample lands in the whole-run histogram
2014/// AND exactly one epoch bucket), so `Σ per-epoch == whole-run`. `phase_epoch` is
2015/// read Relaxed (telemetry, gates no coupled memory); a 1s sample straddling an
2016/// epoch change is attributed wholly to the sample-time epoch (bounded fuzz,
2017/// mirroring the worker drain-on-change). The 1-second cadence is the workload's
2018/// defined sampling/scaling interval, not a synchronization wait; it observes
2019/// `stop` after each interval.
2020fn control_loop(
2021 progress: &AtomicU64,
2022 stop: &AtomicBool,
2023 config: &SchbenchConfig,
2024 live_rate: &AtomicUsize,
2025 target_hit: &AtomicBool,
2026 phase_epoch: Option<&AtomicU32>,
2027) -> (PlatStats, std::collections::BTreeMap<u32, PlatStats>) {
2028 let mut rps_stats = PlatStats::default();
2029 let mut per_epoch_rps: std::collections::BTreeMap<u32, PlatStats> =
2030 std::collections::BTreeMap::new();
2031 let mut last_loop = 0u64;
2032 let mut last_t = monotonic_nanos();
2033 let mut busy_state = ReadBusyState::default();
2034 while !stop.load(Ordering::Acquire) {
2035 // Sleep ~1s between samples, but in STOP_POLL_QUANTUM_NS chunks that
2036 // re-check `stop`, so a shutdown is observed within a quantum instead of
2037 // blocking run()'s teardown for a full second (the control thread is joined
2038 // by run(), so an uncapped sleep here is the dominant teardown latency —
2039 // worse than, and unconditional unlike, the RPS injector's pacing sleep).
2040 let sleep_target = monotonic_nanos().saturating_add(1_000_000_000);
2041 loop {
2042 if stop.load(Ordering::Acquire) {
2043 break;
2044 }
2045 let now = monotonic_nanos();
2046 if now >= sleep_target {
2047 break;
2048 }
2049 std::thread::sleep(std::time::Duration::from_nanos(
2050 (sleep_target - now).min(STOP_POLL_QUANTUM_NS),
2051 ));
2052 }
2053 // Sample the just-elapsed window BEFORE re-checking stop at the loop top,
2054 // so the in-progress second is recorded rather than dropped. `dt` is the
2055 // ACTUAL elapsed time (now - last_t).
2056 //
2057 // schbench samples-first then sleeps the full second (`sleep(1)` only
2058 // `if (!done)`, schbench.c:1827), so every schbench sample -- including the
2059 // boundary one taken when `done` is detected -- spans a real ~1s window
2060 // (rps over a real `delta = tvdelta(..)`, schbench.c:1774,1777). ktstr
2061 // sleeps-first/samples-last with a BOUNDED sleep, which diverges two ways:
2062 // - On early `stop` this boundary sample spans a true PARTIAL window: for
2063 // a substantial window (>= one quantum) the real rate over the
2064 // sub-window the workers were active (more granular than schbench, which
2065 // would have waited the full second); a degenerate sub-quantum window is
2066 // dropped by `control_loop_rps_sample` (a count/dt rate is unbounded as
2067 // dt->0). The bounded sleep is a ktstr scenario-teardown requirement
2068 // schbench does not face.
2069 // - The sleep-first shape omits schbench's tiny FIRST-window sample
2070 // (schbench reads `now` at loop start before any work, schbench.c:1757,
2071 // so its first `delta` is ~us); ktstr's first sample follows a full ~1s
2072 // sleep.
2073 let now = monotonic_nanos();
2074 let lc = progress.load(Ordering::Relaxed);
2075 let dt = now.saturating_sub(last_t);
2076 // Achieved per-second rate = Δcompleted-cycles / Δt (schbench
2077 // `combine_message_thread_rps` over the shared loop count, schbench.c:1777),
2078 // floored at one quantum by `control_loop_rps_sample` (a ktstr teardown
2079 // guard, not in schbench — see its doc).
2080 if let Some(sample) = control_loop_rps_sample(lc.saturating_sub(last_loop), dt) {
2081 // Gate: for auto-RPS, sample only once the target is hit
2082 // (`schbench.c:1781`); fixed/default mode samples every second.
2083 if config.auto_rps == 0 || target_hit.load(Ordering::Relaxed) {
2084 rps_stats.add_lat(sample);
2085 // Per-phase parity: also bucket the sample by the epoch observed
2086 // NOW (the sample-time epoch; a straddling 1s sample lands wholly
2087 // here, bounded fuzz). 0/u32::MAX sentinel epochs route into the
2088 // same all_phases entries the host discards (the run() merge).
2089 let epoch = phase_epoch.map_or(0, |e| e.load(Ordering::Relaxed));
2090 per_epoch_rps.entry(epoch).or_default().add_lat(sample);
2091 }
2092 }
2093 last_loop = lc;
2094 last_t = now;
2095 if config.auto_rps != 0 {
2096 let just_hit = auto_scale_rps(&mut busy_state, live_rate, target_hit, config.auto_rps);
2097 if just_hit {
2098 // Discard ALL pre-target ramp samples (schbench memsets rps_stats
2099 // on the target-hit transition -- grow/shrink/on-target bands all
2100 // funnel through just_hit, `schbench.c:1212`/`:1235`/`:1247`). The
2101 // per-epoch map resets in LOCKSTEP so the per-phase view never
2102 // includes ramp samples the whole-run view discards (the
2103 // Σ-per-epoch == whole-run invariant survives the reset).
2104 reset_rps_accumulators(&mut rps_stats, &mut per_epoch_rps);
2105 }
2106 }
2107 }
2108 (rps_stats, per_epoch_rps)
2109}
2110
2111/// Achieved per-second rate sample for one control-loop window:
2112/// Δcompleted-cycles over `dt_ns`, in cycles/sec, clamped to `u32`. `None` when
2113/// the window is shorter than one `STOP_POLL_QUANTUM_NS` quantum.
2114///
2115/// The floor is a correctness guard, not a nicety. The bounded teardown sleep in
2116/// `control_loop` can return on `stop` with a sub-millisecond `dt` (the window
2117/// between a sample's `last_t = now` and the next iteration's first stop check),
2118/// and a count/dt RATE is unbounded as `dt -> 0` (a single completion in a 1us
2119/// window reads as ~1e6 cycles/sec), so an unfloored sub-quantum sample would
2120/// spike `rps_max` and the top rps percentiles -- assertable metrics. A real
2121/// partial window is always >= one quantum (the sleep loop only observes `stop`
2122/// at a quantum boundary), so the floor drops only the degenerate race window; an
2123/// admitted window bounds the sample to `<= delta_loops * 1e9 /
2124/// STOP_POLL_QUANTUM_NS` (~ the real rate, no inflation). This diverges from
2125/// schbench's `isfinite(rps)` guard (schbench.c:1782), which catches only an
2126/// exact-zero delta -- a finite-but-huge rate from a tiny window slips past it,
2127/// but schbench's full-second cadence (schbench.c:1827) never produces one.
2128fn control_loop_rps_sample(delta_loops: u64, dt_ns: u64) -> Option<u32> {
2129 if dt_ns < STOP_POLL_QUANTUM_NS {
2130 return None;
2131 }
2132 let rps = delta_loops as f64 * 1e9 / dt_ns as f64;
2133 Some((rps as u64).min(u32::MAX as u64) as u32)
2134}
2135
2136/// Run the schbench workload until `stop` is set, returning the whole-run
2137/// percentiles + achieved rate AND the per-phase aggregates. `progress` is the
2138/// live count of completed work cycles across all workers. `phase_epoch` (when
2139/// `Some`) is the scenario engine's shared per-phase generation; the engine
2140/// splits its histograms at each transition (see [`worker_loop`]).
2141pub(crate) fn run(
2142 config: &SchbenchConfig,
2143 stop: &AtomicBool,
2144 progress: &AtomicU64,
2145 phase_epoch: Option<&AtomicU32>,
2146) -> SchbenchOutcome {
2147 let (allowed_count, lock_array_size) = resolve_cpu_topology();
2148 let worker_threads = config.resolve_worker_count(allowed_count);
2149 let locks = if config.skip_locking {
2150 None
2151 } else {
2152 Some(PerCpuLocks::new(lock_array_size))
2153 };
2154 // schbench rejects a `--split` outside 0..=100 at parse and exits
2155 // (schbench.c:362-365). ktstr's analog is a loud panic here at the
2156 // consumption boundary -- before any matrix sizing or the shared Arc
2157 // allocation below -- so an out-of-range config (a `pub split_percent` set
2158 // via struct literal bypasses the builder's debug_assert) can never
2159 // underflow `100 - p` (a release `usize` wrap that would size a garbage
2160 // shared matrix and OOM the allocation) or silently mis-split.
2161 if let Some(p) = config.split_percent {
2162 assert!(
2163 p <= 100,
2164 "schbench split_percent must be in 0..=100, got {p} (schbench.c:362-365 rejects out of range)"
2165 );
2166 }
2167 // schbench.c:1871-1872: ONE process-global shared matrix for `--split`,
2168 // allocated once and shared (interior-mutable AtomicU64) by every worker
2169 // across all message threads. Empty when not splitting (`None`) or when the
2170 // shared portion rounds to 0 (split=100): the `shared_matrix_size > 0`
2171 // guard in do_work then skips the shared branch, so the empty slice is
2172 // never indexed.
2173 let shared_matrix_size = config.shared_matrix_size();
2174 let shared: std::sync::Arc<[core::sync::atomic::AtomicU64]> = if shared_matrix_size > 0 {
2175 (0..3 * shared_matrix_size * shared_matrix_size)
2176 .map(|_| core::sync::atomic::AtomicU64::new(0))
2177 .collect()
2178 } else {
2179 std::sync::Arc::from([] as [core::sync::atomic::AtomicU64; 0])
2180 };
2181
2182 let start = monotonic_nanos();
2183 let mut all_wakeup = PlatStats::default();
2184 let mut all_request = PlatStats::default();
2185 let mut total_msg_sched_delay = 0u64;
2186 let mut total_worker_sched_delay = 0u64;
2187 let mut all_phases: std::collections::BTreeMap<u32, SchbenchPhaseStats> =
2188 std::collections::BTreeMap::new();
2189
2190 // The live PER-MESSAGE-THREAD request rate the RPS injectors read each second
2191 // -- schbench's global `requests_per_sec` AFTER its startup `/= message_threads`
2192 // (`schbench.c:1899`), which is exactly the value `auto_scale_rps` scales
2193 // (`schbench.c:1250`) and each `run_rps_thread` injects directly with no
2194 // further division (`schbench.c:1290`). Seeded from `rps_per_message_thread()`
2195 // (the -R total / m, or the auto-RPS seed 10 / m). The control thread
2196 // auto-scales it when auto-RPS is on; otherwise it stays constant -- a fixed
2197 // -R then injects that seed rate unchanged. `target_hit` gates the rps_stats sampling
2198 // for auto-RPS (`schbench.c:1781`); `rps_stats` is moved out of the control
2199 // thread on join. `live_rate` is read back after the scope as the auto-RPS
2200 // "final rps goal" (`schbench.c:1995`, per-thread * m).
2201 //
2202 // Ordering: both atomics are lone scalars that gate NO coupled memory --
2203 // `live_rate` is a plain count the RPS thread loops on (the requests it
2204 // enqueues publish via the Treiber stack's own AcqRel), and `target_hit` is
2205 // touched only by the control thread. So Relaxed is correct (a 1-second-stale
2206 // read is benign within the auto-RPS cadence; schbench reads its global with
2207 // no sync at all). The final read after the scope is synchronized by the join.
2208 let live_rate = AtomicUsize::new(config.rps_per_message_thread());
2209 let target_hit = AtomicBool::new(false);
2210 let mut rps_stats = PlatStats::default();
2211
2212 std::thread::scope(|outer| {
2213 let handles: Vec<_> = (0..config.message_threads)
2214 .map(|_| {
2215 let locks = locks.as_ref();
2216 let live_rate = &live_rate;
2217 let shared = &shared;
2218 outer.spawn(move || {
2219 run_one_message_thread(
2220 worker_threads,
2221 locks,
2222 config,
2223 stop,
2224 progress,
2225 phase_epoch,
2226 live_rate,
2227 &shared[..],
2228 shared_matrix_size,
2229 )
2230 })
2231 })
2232 .collect();
2233 // Control thread: schbench's main control loop -- once/sec, sample the
2234 // achieved RPS into rps_stats and (for auto-RPS) auto-scale `live_rate`.
2235 let control = outer
2236 .spawn(|| control_loop(progress, stop, config, &live_rate, &target_hit, phase_epoch));
2237 for h in handles {
2238 let mtr = h.join().expect("schbench message thread panicked");
2239 all_wakeup.combine(&mtr.whole_wakeup);
2240 all_request.combine(&mtr.whole_request);
2241 total_msg_sched_delay = total_msg_sched_delay.saturating_add(mtr.msg_sched_delay_ns);
2242 total_worker_sched_delay =
2243 total_worker_sched_delay.saturating_add(mtr.workers_sched_delay_sum);
2244 for (epoch, sps) in mtr.phases {
2245 all_phases.entry(epoch).or_default().merge(&sps);
2246 }
2247 }
2248 let (whole_rps, control_per_epoch_rps) =
2249 control.join().expect("schbench control thread panicked");
2250 rps_stats = whole_rps;
2251 // Merge the control thread's per-epoch RPS into the per-phase aggregate
2252 // EXACTLY ONCE, post-join, OUTSIDE the per-message-thread loop above (else
2253 // it would double-count by message_threads). The control thread is the
2254 // SOLE rps source; the workers contribute empty rps via mtr.phases.
2255 for (epoch, hist) in control_per_epoch_rps {
2256 all_phases.entry(epoch).or_default().rps.combine(&hist);
2257 }
2258 });
2259
2260 // The auto-RPS "final rps goal" (`schbench.c:1995`): the live per-thread rate
2261 // at exit * message_threads. Read after the scope (all threads joined, so the
2262 // control loop's last store is visible).
2263 let final_rps_goal = live_rate.load(Ordering::Relaxed) * config.message_threads;
2264 let loop_count = progress.load(Ordering::Relaxed);
2265 let elapsed_ns = monotonic_nanos().saturating_sub(start);
2266 let achieved_rps = if elapsed_ns > 0 {
2267 loop_count as f64 / (elapsed_ns as f64 / 1e9)
2268 } else {
2269 0.0
2270 };
2271 // Average the per-thread run-queue waits, matching schbench's
2272 // collect_sched_delay (`schbench.c:1673-1674`): message delay over
2273 // message_threads, worker delay over all workers. (Whole-run mean-of-means;
2274 // the per-phase path uses sample-weighted Σrd/Σpc — see SchbenchPhaseStats.)
2275 let sched_delay_msg_ns = total_msg_sched_delay / (config.message_threads.max(1) as u64);
2276 let total_workers = (config.message_threads * worker_threads).max(1) as u64;
2277 let sched_delay_worker_ns = total_worker_sched_delay / total_workers;
2278
2279 SchbenchOutcome {
2280 whole_run: SchbenchResult {
2281 wakeup: all_wakeup.percentiles(),
2282 request: all_request.percentiles(),
2283 rps: rps_stats.percentiles(),
2284 loop_count,
2285 nr_workers: total_workers as usize,
2286 achieved_rps,
2287 final_rps_goal,
2288 sched_delay_msg_ns,
2289 sched_delay_worker_ns,
2290 },
2291 phases: all_phases.into_iter().collect(),
2292 }
2293}
2294
2295#[cfg(test)]
2296mod tests {
2297 use super::*;
2298
2299 /// Lightweight node for stack stress tests (ThreadData is far heavier —
2300 /// two boxed `PlatStats` histograms plus per-thread state — too heavy to
2301 /// allocate by the thousand).
2302 struct TestNode {
2303 next: AtomicPtr<TestNode>,
2304 }
2305 impl Linked for TestNode {
2306 fn next_link(&self) -> &AtomicPtr<Self> {
2307 &self.next
2308 }
2309 }
2310 impl TestNode {
2311 fn new() -> Self {
2312 Self {
2313 next: AtomicPtr::new(ptr::null_mut()),
2314 }
2315 }
2316 }
2317
2318 #[test]
2319 fn resolve_worker_count_divides_cpuset_across_message_threads() {
2320 // Explicit non-zero worker_threads is honored as-is (per message thread).
2321 let c = SchbenchConfig::default()
2322 .worker_threads(3)
2323 .message_threads(4);
2324 assert_eq!(c.resolve_worker_count(8), 3);
2325
2326 // 0-default mirrors schbench's ceil(cpuset_cpus / message_threads)
2327 // (schbench.c:1849-1852), scoped to the cpuset count:
2328 // m=1: 8/1 = 8 per thread, total 8 (== the cpuset count; no divide).
2329 let c = SchbenchConfig::default().message_threads(1);
2330 assert_eq!(c.resolve_worker_count(8), 8);
2331 // m=2: 8/2 = 4 per thread, total 8.
2332 let c = SchbenchConfig::default().message_threads(2);
2333 assert_eq!(c.resolve_worker_count(8), 4);
2334 // m=3: ceil(8/3) = 3 per thread, total 9 (≈ 8; schbench rounds up too).
2335 let c = SchbenchConfig::default().message_threads(3);
2336 assert_eq!(c.resolve_worker_count(8), 3);
2337 // message_threads floored at 1 — no div-by-zero.
2338 let c = SchbenchConfig::default().message_threads(0);
2339 assert_eq!(c.resolve_worker_count(8), 8);
2340 }
2341
2342 #[test]
2343 fn matrix_size_matches_schbench_formula() {
2344 // sqrt(256*1024/3/8) = sqrt(10922) = 104.
2345 assert_eq!(
2346 SchbenchConfig {
2347 cache_footprint_kib: 256,
2348 operations: 5,
2349 ..Default::default()
2350 }
2351 .matrix_size(),
2352 104
2353 );
2354 // Zero operations -> no matrix work.
2355 assert_eq!(
2356 SchbenchConfig {
2357 operations: 0,
2358 ..Default::default()
2359 }
2360 .matrix_size(),
2361 0
2362 );
2363 }
2364
2365 #[test]
2366 fn split_matrix_sizes_match_schbench_formula() {
2367 // schbench.c:1858-1863: with --split=p the footprint splits into a
2368 // private matrix (p%) and a shared matrix (100-p%); each dimension is
2369 // sqrt(kib*frac/100 * 1024/3/sizeof(ulong)). None => the legacy single
2370 // all-private matrix (full footprint) and no shared matrix.
2371 let cfg = |split| SchbenchConfig {
2372 cache_footprint_kib: 256,
2373 operations: 5,
2374 split_percent: split,
2375 ..Default::default()
2376 };
2377 // None: private == the full-footprint matrix_size (104), no shared.
2378 assert_eq!(cfg(None).private_matrix_size(), 104);
2379 assert_eq!(cfg(None).shared_matrix_size(), 0);
2380 // 0%: nothing private; the whole footprint is the shared matrix.
2381 assert_eq!(cfg(Some(0)).private_matrix_size(), 0);
2382 assert_eq!(cfg(Some(0)).shared_matrix_size(), 104);
2383 // 25%: 64 KiB private (sqrt(64*1024/3/8)=52), 192 KiB shared (=90).
2384 assert_eq!(cfg(Some(25)).private_matrix_size(), 52);
2385 assert_eq!(cfg(Some(25)).shared_matrix_size(), 90);
2386 // 100%: all private (full footprint), no shared work.
2387 assert_eq!(cfg(Some(100)).private_matrix_size(), 104);
2388 assert_eq!(cfg(Some(100)).shared_matrix_size(), 0);
2389 // Zero operations -> no matrix work either way.
2390 let none_ops = SchbenchConfig {
2391 operations: 0,
2392 split_percent: Some(50),
2393 ..Default::default()
2394 };
2395 assert_eq!(none_ops.private_matrix_size(), 0);
2396 assert_eq!(none_ops.shared_matrix_size(), 0);
2397 }
2398
2399 #[test]
2400 fn ops_split_matches_schbench_integer_division() {
2401 // schbench.c:1391-1392: ops_private = operations*split/100 (integer
2402 // truncation), ops_shared = operations - ops_private. Returns
2403 // (ops_shared, ops_private).
2404 assert_eq!(ops_split(5, 0), (5, 0)); // all shared
2405 assert_eq!(ops_split(5, 25), (4, 1)); // 5*25/100 = 1 private
2406 assert_eq!(ops_split(5, 50), (3, 2)); // 5*50/100 = 2 private
2407 assert_eq!(ops_split(5, 100), (0, 5)); // all private
2408 }
2409
2410 #[test]
2411 fn engine_split_runs_shared_matrix_concurrently() {
2412 // --split with two workers: both concurrently multiply into the ONE
2413 // process-global shared AtomicU64 matrix (per-k stores to the shared C
2414 // cells -- the deliberate cache contention schbench models). Reaching the
2415 // assertion proves the shared-matrix path
2416 // neither deadlocks nor trips a data-race trap -- the atomics make the
2417 // shared race sound (vs schbench's plain shared-memory race). operations=8
2418 // @ split=50 => 4 shared + 4 private ops/cycle, so both paths run. Stop is
2419 // event-driven (spin on the progress counter, not a sleep).
2420 let config = SchbenchConfig {
2421 message_threads: 1,
2422 worker_threads: 2,
2423 cache_footprint_kib: 16,
2424 operations: 8,
2425 sleep_usec: 0,
2426 skip_locking: false,
2427 requests_per_sec: 0,
2428 auto_rps: 0,
2429 split_percent: Some(50),
2430 pipe_transfer_bytes: 0,
2431 };
2432 // Sanity: both split matrices are non-empty at this footprint, so the
2433 // test exercises shared + private work, not a degenerate 0-dim path.
2434 assert!(config.shared_matrix_size() > 0, "shared matrix present");
2435 assert!(config.private_matrix_size() > 0, "private matrix present");
2436 let stop = AtomicBool::new(false);
2437 let progress = AtomicU64::new(0);
2438 let outcome = std::thread::scope(|s| {
2439 let runner = s.spawn(|| run(&config, &stop, &progress, None));
2440 while progress.load(Ordering::Relaxed) < 50 {
2441 core::hint::spin_loop();
2442 }
2443 stop.store(true, Ordering::Release);
2444 runner.join().expect("run panicked")
2445 });
2446 assert!(
2447 outcome.whole_run.loop_count >= 50,
2448 "engine did split work: {}",
2449 outcome.whole_run.loop_count
2450 );
2451 }
2452
2453 #[test]
2454 fn pipe_fill_writes_every_byte_and_sizes_the_page() {
2455 // pipe_fill is the per-cycle memory transfer (schbench's pipe_page memset,
2456 // schbench.c:980-981/:1003-1004): it must touch EVERY byte of a
2457 // pipe_bytes-sized page. The bytes are never read back, so a dead-store
2458 // elision would silently zero the workload -- the black_box in pipe_fill
2459 // prevents that; this pins the observable effect (every byte written).
2460 let td = ThreadData::new(64);
2461 assert_eq!(td.pipe_bytes, 64);
2462 // SAFETY: single-threaded test, exclusive access to this td's page.
2463 unsafe { td.pipe_fill(2) };
2464 // SAFETY: same thread; no concurrent access.
2465 assert_eq!(
2466 unsafe { &*td.pipe_page.get() }.len(),
2467 64,
2468 "page sized to pipe_bytes"
2469 );
2470 assert!(
2471 unsafe { &*td.pipe_page.get() }.iter().all(|&b| b == 2),
2472 "worker fill (2) touches every byte"
2473 );
2474 // SAFETY: exclusive.
2475 unsafe { td.pipe_fill(1) };
2476 assert!(
2477 unsafe { &*td.pipe_page.get() }.iter().all(|&b| b == 1),
2478 "waker fill (1) overwrites every byte"
2479 );
2480 // Non-pipe ThreadData (pipe_bytes == 0) allocates an empty page: no transfer.
2481 let none = ThreadData::new(0);
2482 assert_eq!(none.pipe_bytes, 0);
2483 // SAFETY: exclusive.
2484 assert_eq!(unsafe { &*none.pipe_page.get() }.len(), 0);
2485 }
2486
2487 #[test]
2488 fn pipe_transfer_bytes_clamps_to_one_mib() {
2489 // schbench clamps -p to PIPE_TRANSFER_BUFFER (schbench.c:41,291-294); run()
2490 // applies the same `.min()` before sizing each worker's page.
2491 assert_eq!(PIPE_TRANSFER_BUFFER, 1024 * 1024);
2492 assert_eq!(
2493 (2 * PIPE_TRANSFER_BUFFER).min(PIPE_TRANSFER_BUFFER),
2494 PIPE_TRANSFER_BUFFER,
2495 "over-cap pipe size clamps to the 1 MiB buffer"
2496 );
2497 // At/under-cap values pass through unchanged.
2498 assert_eq!(4096_usize.min(PIPE_TRANSFER_BUFFER), 4096);
2499 }
2500
2501 #[test]
2502 fn pipe_transfer_report_is_per_worker_scales_and_clamps() {
2503 // PER-WORKER: ops/sec = achieved_rps / nr_workers
2504 // (schbench.c:1942-1943 divide by Σ worker runtimes ≈ nr_workers*elapsed).
2505 // 2048 aggregate cycles/sec / 2 workers = 1024 per-worker ops/sec.
2506 let r = pipe_transfer_report(2048.0, 1, 2);
2507 assert_eq!(r.ops_per_sec, 1024.0);
2508 // pretty_size (schbench.c:1606-1620): 1024 per-worker B/s -> 1.00 KB/s.
2509 assert_eq!((r.scaled, r.unit), (1.0, "KB"));
2510 // 1 worker, 1 B/cycle @ 1 cycle/s -> 1.00 B/s.
2511 let one = pipe_transfer_report(1.0, 1, 1);
2512 assert_eq!((one.ops_per_sec, one.scaled, one.unit), (1.0, 1.0, "B"));
2513 // 1 MiB/cycle @ 1 cycle/s/worker -> 1.00 MB/s.
2514 assert_eq!(pipe_transfer_report(1.0, 1 << 20, 1).unit, "MB");
2515 // CLAMP: an over-cap pipe size reports the clamped 1 MiB, not
2516 // the requested 2 MiB -- the engine moves only the clamped size per cycle
2517 // (schbench clamps at parse, schbench.c:291-294).
2518 let over = pipe_transfer_report(1.0, 2 << 20, 1);
2519 let cap = pipe_transfer_report(1.0, 1 << 20, 1);
2520 assert_eq!(
2521 (over.scaled, over.unit),
2522 (cap.scaled, cap.unit),
2523 "over-cap pipe size clamps to 1 MiB"
2524 );
2525 // EB cap: a huge byte rate never scales past the last unit (no overflow).
2526 assert_eq!(pipe_transfer_report(f64::MAX, 1 << 20, 1).unit, "EB");
2527 // nr_workers 0 -> floored at 1, never a division by zero.
2528 assert_eq!(pipe_transfer_report(100.0, 4096, 0).ops_per_sec, 100.0);
2529 }
2530
2531 #[test]
2532 fn engine_pipe_mode_transfers_without_matrix_work() {
2533 // Pipe mode (-p): the worker loop skips do_work + the think-sleep, and the
2534 // per-cycle work is the cross-thread pipe_page handshake -- the worker
2535 // memsets its page to 2 before blocking, the waker memsets it to 1 on wake
2536 // (schbench.c:980-981/:1003-1004). Two workers exercise the concurrent
2537 // waker->worker page writes. Reaching the assertions proves the handshake
2538 // runs to completion: no deadlock and no data-race trap on the cross-thread
2539 // `pipe_page` UnsafeCell (the wait-list + futex ordering keeps it race-free,
2540 // see the Sync SAFETY note). `operations`/`cache_footprint_kib` are set
2541 // LARGE on purpose: if do_work erroneously ran, every cycle would log a
2542 // multi-us request-latency sample -- the second assertion would then fail.
2543 // Stop is event-driven (spin on the progress counter, not a sleep).
2544 let config = SchbenchConfig::default()
2545 .message_threads(1)
2546 .worker_threads(2)
2547 .sleep_usec(0)
2548 .operations(50)
2549 .cache_footprint_kib(256)
2550 .pipe_transfer_bytes(4096);
2551 assert!(config.pipe_transfer_bytes > 0, "pipe mode engaged");
2552 assert!(
2553 config.matrix_size() > 0,
2554 "matrix work would be non-trivial if do_work ran (regression guard)"
2555 );
2556 let stop = AtomicBool::new(false);
2557 let progress = AtomicU64::new(0);
2558 let outcome = std::thread::scope(|s| {
2559 let runner = s.spawn(|| run(&config, &stop, &progress, None));
2560 while progress.load(Ordering::Relaxed) < 50 {
2561 core::hint::spin_loop();
2562 }
2563 stop.store(true, Ordering::Release);
2564 runner.join().expect("run panicked")
2565 });
2566 // loop_count accrues one per completed transfer cycle (schbench.c:1479),
2567 // proving the pipe handshake + memsets ran end-to-end without deadlock.
2568 assert!(
2569 outcome.whole_run.loop_count >= 50,
2570 "pipe engine transferred {} cycles",
2571 outcome.whole_run.loop_count
2572 );
2573 // Discriminating assertion: do_work is SKIPPED, so the
2574 // back-to-back work_start->now span yields sub-us request deltas that the
2575 // `delta_us > 0` filter drops -- request stats stay ~empty. If matrix work
2576 // erroneously ran at operations=50/256KiB, nearly every cycle would record a
2577 // >0 request delta and request samples would approach loop_count, failing
2578 // this. (A rare us-straddle may record a handful; `* 4 < loop_count` tolerates
2579 // that while still catching a real do_work regression.)
2580 assert!(
2581 outcome.whole_run.request.nr_samples * 4 < outcome.whole_run.loop_count,
2582 "pipe mode skips matrix work: {} request samples vs {} cycles",
2583 outcome.whole_run.request.nr_samples,
2584 outcome.whole_run.loop_count
2585 );
2586 }
2587
2588 #[test]
2589 #[should_panic(expected = "split_percent must be in 0..=100")]
2590 fn engine_panics_on_out_of_range_split() {
2591 // Regression pin for the out-of-range finding: a `pub split_percent` set
2592 // past 100 via STRUCT LITERAL (the path that bypasses the builder's
2593 // debug_assert) must fail LOUDLY at the run() consumption boundary in
2594 // every build profile -- the analog of schbench exiting on a bad
2595 // `--split` -- never silently underflow `100 - p` into a garbage-huge
2596 // shared matrix. The assert fires before any thread spawn or allocation,
2597 // so this is cheap (`stop` is pre-set just in case).
2598 let config = SchbenchConfig {
2599 message_threads: 1,
2600 worker_threads: 1,
2601 cache_footprint_kib: 16,
2602 operations: 1,
2603 sleep_usec: 0,
2604 skip_locking: false,
2605 requests_per_sec: 0,
2606 auto_rps: 0,
2607 split_percent: Some(101),
2608 pipe_transfer_bytes: 0,
2609 };
2610 let stop = AtomicBool::new(true);
2611 let progress = AtomicU64::new(0);
2612 let _ = run(&config, &stop, &progress, None);
2613 }
2614
2615 #[test]
2616 fn stack_add_splice_is_lifo() {
2617 let a = TestNode::new();
2618 let b = TestNode::new();
2619 let stack: TreiberStack<TestNode> = TreiberStack::new();
2620 assert!(stack.splice().is_null(), "empty stack splices to null");
2621 stack.add(&a as *const _ as *mut _);
2622 stack.add(&b as *const _ as *mut _);
2623 // LIFO: b (pushed last) is the head; a follows.
2624 let head = stack.splice();
2625 assert_eq!(head.cast_const(), &b as *const TestNode);
2626 // SAFETY: head -> b (alive on stack); its link -> a.
2627 let second = unsafe { (*head).next.load(Ordering::Acquire) };
2628 assert_eq!(second.cast_const(), &a as *const TestNode);
2629 // SAFETY: second -> a; its link is null.
2630 assert!(unsafe { (*second).next.load(Ordering::Acquire) }.is_null());
2631 assert!(stack.splice().is_null(), "splice emptied the stack");
2632 }
2633
2634 #[test]
2635 fn stack_concurrent_add_loses_no_nodes() {
2636 // N threads each concurrently push K distinct nodes; after all join, a
2637 // single splice must return exactly N*K nodes. A broken CAS (lost
2638 // update / ABA) would drop nodes and the count would be short. Push is
2639 // deterministic (each node pushed once), so this never hangs.
2640 const THREADS: usize = 8;
2641 const PER_THREAD: usize = 2000;
2642 let nodes: Vec<TestNode> = (0..THREADS * PER_THREAD).map(|_| TestNode::new()).collect();
2643 let stack = TreiberStack::new();
2644
2645 std::thread::scope(|s| {
2646 for chunk in nodes.chunks(PER_THREAD) {
2647 let stack = &stack;
2648 s.spawn(move || {
2649 for n in chunk {
2650 stack.add(n as *const TestNode as *mut TestNode);
2651 }
2652 });
2653 }
2654 });
2655
2656 // All pushes complete (scope joined). Drain and count distinct nodes.
2657 let mut seen = std::collections::HashSet::new();
2658 let mut cur = stack.splice();
2659 while !cur.is_null() {
2660 assert!(seen.insert(cur), "node observed twice");
2661 // SAFETY: cur is one of the live `nodes`; next is its link.
2662 cur = unsafe { (*cur).next.load(Ordering::Acquire) };
2663 }
2664 assert_eq!(
2665 seen.len(),
2666 THREADS * PER_THREAD,
2667 "no node lost under contention"
2668 );
2669 }
2670
2671 #[test]
2672 fn engine_runs_and_produces_latency_samples() {
2673 // Small topology; run until ~50 work cycles complete, then stop. The
2674 // test completing proves the shutdown does not hang (a scope-join
2675 // shutdown bug would block here until the nextest timeout). Stop is
2676 // event-driven: spin on the shared progress counter, not a sleep.
2677 let config = SchbenchConfig {
2678 message_threads: 1,
2679 worker_threads: 2,
2680 cache_footprint_kib: 16,
2681 operations: 1,
2682 sleep_usec: 0,
2683 skip_locking: false,
2684 requests_per_sec: 0,
2685 auto_rps: 0,
2686 split_percent: None,
2687 pipe_transfer_bytes: 0,
2688 };
2689 let stop = AtomicBool::new(false);
2690 let progress = AtomicU64::new(0);
2691 let outcome = std::thread::scope(|s| {
2692 let runner = s.spawn(|| run(&config, &stop, &progress, None));
2693 while progress.load(Ordering::Relaxed) < 50 {
2694 core::hint::spin_loop();
2695 }
2696 stop.store(true, Ordering::Release);
2697 runner.join().expect("run panicked")
2698 });
2699 let result = &outcome.whole_run;
2700 assert!(
2701 result.loop_count >= 50,
2702 "engine did work: {}",
2703 result.loop_count
2704 );
2705 assert!(result.wakeup.nr_samples > 0, "wakeup samples recorded");
2706 assert!(result.request.nr_samples > 0, "request samples recorded");
2707 assert!(result.achieved_rps > 0.0, "positive achieved rps");
2708 // Non-phasic (phase_epoch None): the only snapshot is the baseline
2709 // epoch 0, which the host discards. No per-phase metrics are emitted.
2710 assert_eq!(
2711 outcome.phases.len(),
2712 1,
2713 "non-phasic => single baseline phase"
2714 );
2715 assert_eq!(outcome.phases[0].0, 0, "the lone phase is BASELINE epoch 0");
2716 }
2717
2718 #[test]
2719 fn engine_terminates_when_lone_worker_stops() {
2720 // Regression for the shutdown deadlock: with a SINGLE worker there is no
2721 // second worker to post the message thread, so a worker that exits on
2722 // `stop` while the message thread is parked in `wait_forever` must wake it
2723 // (the unconditional post at worker_loop's exit), or run() hangs forever.
2724 // Reaching the assertion at all proves run() returned. `sleep_usec` 0
2725 // keeps the lone worker almost always mid-`do_work` when stop fires (the
2726 // deadlock-prone window). engine_runs / engine_splits use 2 workers and
2727 // missed this -- a second worker posting the message thread masks the bug;
2728 // a hang here is the nextest timeout, exactly how the bug first surfaced.
2729 let config = SchbenchConfig {
2730 message_threads: 1,
2731 worker_threads: 1,
2732 cache_footprint_kib: 256,
2733 operations: 5,
2734 sleep_usec: 0,
2735 skip_locking: false,
2736 requests_per_sec: 0,
2737 auto_rps: 0,
2738 split_percent: None,
2739 pipe_transfer_bytes: 0,
2740 };
2741 let stop = AtomicBool::new(false);
2742 let progress = AtomicU64::new(0);
2743 let outcome = std::thread::scope(|s| {
2744 let runner = s.spawn(|| run(&config, &stop, &progress, None));
2745 while progress.load(Ordering::Relaxed) < 10 {
2746 core::hint::spin_loop();
2747 }
2748 stop.store(true, Ordering::Release);
2749 // Deadlocks here on regression: the lone worker exits without waking
2750 // the parked message thread, so this join never returns.
2751 runner.join().expect("run panicked")
2752 });
2753 assert!(
2754 outcome.whole_run.loop_count >= 10,
2755 "engine did work and returned: {}",
2756 outcome.whole_run.loop_count
2757 );
2758 }
2759
2760 #[test]
2761 fn engine_rps_mode_injects_drains_and_terminates() {
2762 // RPS-injector mode (requests_per_sec != 0): a dedicated thread enqueues
2763 // requests round-robin across the workers, which splice + drain their
2764 // queues instead of the message-thread handshake. Verify the RPS path
2765 // runs (loop_count + request samples) and terminates cleanly -- run()
2766 // returns (no deadlock: the RPS thread is the waker, the worker the
2767 // waitee, and shutdown wakes the workers via run_one_message_thread's
2768 // worker-wake + the RPS thread's stop-time fpost-all; a hang is the
2769 // nextest timeout) with no request leak (the post-join free).
2770 let config = SchbenchConfig {
2771 message_threads: 1,
2772 worker_threads: 2,
2773 cache_footprint_kib: 16,
2774 operations: 1,
2775 sleep_usec: 0,
2776 skip_locking: false,
2777 requests_per_sec: 10_000,
2778 auto_rps: 0,
2779 split_percent: None,
2780 pipe_transfer_bytes: 0,
2781 };
2782 let stop = AtomicBool::new(false);
2783 let progress = AtomicU64::new(0);
2784 let outcome = std::thread::scope(|s| {
2785 let runner = s.spawn(|| run(&config, &stop, &progress, None));
2786 while progress.load(Ordering::Relaxed) < 50 {
2787 core::hint::spin_loop();
2788 }
2789 stop.store(true, Ordering::Release);
2790 // Deadlocks here on regression: a worker parked in rps_wait that the
2791 // shutdown path fails to wake, or the RPS thread failing to observe
2792 // stop, would hang this join.
2793 runner.join().expect("run panicked")
2794 });
2795 let result = &outcome.whole_run;
2796 assert!(
2797 result.loop_count >= 50,
2798 "RPS workers serviced requests: {}",
2799 result.loop_count
2800 );
2801 assert!(
2802 result.request.nr_samples > 0,
2803 "RPS request-latency samples recorded"
2804 );
2805 }
2806
2807 #[test]
2808 fn rps_injector_pacing_sleep_is_bounded_for_prompt_shutdown() {
2809 // requests_per_sec=1, message_threads=1 => per-thread inter-arrival
2810 // interval ~1s. A stop set while the injector is in that paced sleep must be
2811 // observed within STOP_POLL_QUANTUM_NS (50 ms), so run()'s message-thread
2812 // join completes well under one interval. An unbounded pacing sleep would
2813 // block the join ~1s. 500 ms is 10x the quantum (robust to scheduling
2814 // jitter) and half the interval (cleanly distinguishes bounded from
2815 // unbounded). The 20 ms warmup reaches the long paced sleep before stop (at
2816 // 1 req/s a progress-spin could outlast the test).
2817 let config = SchbenchConfig {
2818 message_threads: 1,
2819 worker_threads: 1,
2820 cache_footprint_kib: 16,
2821 operations: 1,
2822 sleep_usec: 0,
2823 skip_locking: false,
2824 requests_per_sec: 1, // interval = 1s / 1 = 1s
2825 auto_rps: 0,
2826 split_percent: None,
2827 pipe_transfer_bytes: 0,
2828 };
2829 let stop = AtomicBool::new(false);
2830 let progress = AtomicU64::new(0);
2831 let start = std::time::Instant::now();
2832 std::thread::scope(|s| {
2833 let runner = s.spawn(|| run(&config, &stop, &progress, None));
2834 std::thread::sleep(std::time::Duration::from_millis(20));
2835 stop.store(true, Ordering::Release);
2836 let _ = runner.join().expect("run panicked");
2837 });
2838 let elapsed = start.elapsed();
2839 assert!(
2840 elapsed < std::time::Duration::from_millis(500),
2841 "RPS injector joined in {elapsed:?}; the paced sleep must be bounded \
2842 (interval ~1s — an unbounded sleep would block the join that long)",
2843 );
2844 }
2845
2846 #[test]
2847 fn engine_rps_below_message_threads_falls_to_default() {
2848 // Regression: when the -R total is below the message-thread count,
2849 // the per-thread rate (rps_per_message_thread) rounds to 0, and the
2850 // engine must run the DEFAULT message-handshake mode -- workers do real
2851 // wake-cycle work -- matching schbench's startup-divide-before-gate. NOT
2852 // a zero-rate RPS mode (which would leave the workers parked with no
2853 // requests, never advancing progress -- the spin below would then hang,
2854 // caught by the nextest timeout).
2855 let config = SchbenchConfig {
2856 message_threads: 2,
2857 worker_threads: 1,
2858 cache_footprint_kib: 16,
2859 operations: 1,
2860 sleep_usec: 0,
2861 skip_locking: false,
2862 requests_per_sec: 1, // 1 / 2 message threads = 0 per thread -> default
2863 auto_rps: 0,
2864 split_percent: None,
2865 pipe_transfer_bytes: 0,
2866 };
2867 let stop = AtomicBool::new(false);
2868 let progress = AtomicU64::new(0);
2869 let outcome = std::thread::scope(|s| {
2870 let runner = s.spawn(|| run(&config, &stop, &progress, None));
2871 while progress.load(Ordering::Relaxed) < 50 {
2872 core::hint::spin_loop();
2873 }
2874 stop.store(true, Ordering::Release);
2875 runner.join().expect("run panicked")
2876 });
2877 assert!(
2878 outcome.whole_run.loop_count >= 50,
2879 "sub-1-per-thread RPS ran default-mode work: {}",
2880 outcome.whole_run.loop_count
2881 );
2882 }
2883
2884 #[test]
2885 fn auto_scale_grows_when_idle_and_shrinks_when_busy() {
2886 // Pure scaling math (no /proc/stat I/O) at injected busy levels, matching
2887 // schbench's auto_scale_rps body (`schbench.c:1203-1250`).
2888
2889 // Far under target: delta = 80/10 = 8, capped to 3 -> ceil(100*3) = 300.
2890 // Not a near-target transition, so just_hit is false (the >3 cap branch
2891 // never trips target_hit).
2892 let rate = AtomicUsize::new(100);
2893 let hit = AtomicBool::new(false);
2894 assert!(!scale_rps_for_busy(10.0, &rate, &hit, 80));
2895 assert_eq!(rate.load(Ordering::Acquire), 300);
2896 assert!(!hit.load(Ordering::Acquire));
2897
2898 // Far over target: delta = 20/100 = 0.2, clamped to 0.3 -> floor(300*0.3)
2899 // = 90. Also not a near-target transition.
2900 let rate = AtomicUsize::new(300);
2901 let hit = AtomicBool::new(false);
2902 assert!(!scale_rps_for_busy(100.0, &rate, &hit, 20));
2903 assert_eq!(rate.load(Ordering::Acquire), 90);
2904 assert!(!hit.load(Ordering::Acquire));
2905
2906 // Exactly on target: rate held, target_hit set on the first arrival
2907 // (`schbench.c:1243-1248`).
2908 let rate = AtomicUsize::new(123);
2909 let hit = AtomicBool::new(false);
2910 assert!(scale_rps_for_busy(50.0, &rate, &hit, 50));
2911 assert_eq!(rate.load(Ordering::Acquire), 123);
2912 assert!(hit.load(Ordering::Acquire));
2913 }
2914
2915 #[test]
2916 fn auto_scale_target_hit_uses_post_damping_delta() {
2917 // The near-target test reads the delta AFTER its damping reassignment
2918 // (`schbench.c:1209-1213` grow, `:1232-1236` shrink) -- the bug the
2919 // pre-reassignment check had. These cases trip target_hit ONLY under the
2920 // correct post-damping order.
2921
2922 // Grow band: busy 90, target 100 -> delta 1.111 (NOT < 1.05), damped to
2923 // 1 + 0.111/8 = 1.0139 (< 1.05) -> target_hit. A pre-damping check on
2924 // 1.111 would miss it.
2925 let rate = AtomicUsize::new(1000);
2926 let hit = AtomicBool::new(false);
2927 assert!(scale_rps_for_busy(90.0, &rate, &hit, 100));
2928 assert!(hit.load(Ordering::Acquire));
2929
2930 // Shrink band: busy 100, target 95 -> delta 0.95 (NOT > 0.95), damped to
2931 // 0.95 + 0.05/8 = 0.95625 (> 0.95) -> target_hit. A pre-damping check on
2932 // 0.95 would miss it.
2933 let rate = AtomicUsize::new(1000);
2934 let hit = AtomicBool::new(false);
2935 assert!(scale_rps_for_busy(100.0, &rate, &hit, 95));
2936 assert!(hit.load(Ordering::Acquire));
2937
2938 // Already hit: no second transition is reported (just_hit guards on
2939 // !already_hit), even on an exact-target tick.
2940 let rate = AtomicUsize::new(1000);
2941 let hit = AtomicBool::new(true);
2942 assert!(!scale_rps_for_busy(50.0, &rate, &hit, 50));
2943 }
2944
2945 #[test]
2946 fn reset_rps_accumulators_clears_both_in_lockstep() {
2947 // The auto-RPS target-hit reset MUST clear the whole-run AND per-epoch rps
2948 // accumulators together; if a future edit drops one, Σ-per-epoch != whole-run
2949 // (asymmetric-counter-bookkeeping). Pins the lockstep deterministically — the
2950 // engine just_hit path is timing-dependent (the controller must reach target).
2951 let mut whole = PlatStats::default();
2952 whole.add_lat(500);
2953 let mut per_epoch: std::collections::BTreeMap<u32, PlatStats> =
2954 std::collections::BTreeMap::new();
2955 per_epoch.entry(1).or_default().add_lat(500);
2956 reset_rps_accumulators(&mut whole, &mut per_epoch);
2957 assert_eq!(whole.sample_count(), 0, "whole-run rps reset");
2958 assert!(per_epoch.is_empty(), "per-epoch rps reset in lockstep");
2959 }
2960
2961 #[test]
2962 fn control_loop_rps_sample_floors_sub_quantum_window() {
2963 // The bounded teardown sleep can return on `stop` with a tiny dt; a
2964 // count/dt rate is unbounded as dt->0, so a sub-quantum window is dropped
2965 // rather than spiking rps_max / the top rps percentiles (assertable
2966 // metrics). Pins the floor deterministically -- the race that produces the
2967 // tiny dt is timing-dependent, but the math + floor it feeds is pure.
2968 // dt==0 is dropped, not a divide-by-zero:
2969 assert_eq!(control_loop_rps_sample(3, 0), None);
2970 // A 1us / one-completion window would read as ~1e6 cycles/sec -- dropped:
2971 assert_eq!(control_loop_rps_sample(1, 1_000), None);
2972 // Just under one quantum is still dropped:
2973 assert_eq!(control_loop_rps_sample(5, STOP_POLL_QUANTUM_NS - 1), None);
2974 // Exactly one quantum is admitted; the rate is bounded to
2975 // delta_loops * 1e9 / quantum, no inflation: 1 cycle / 50ms = 20/s.
2976 assert_eq!(control_loop_rps_sample(1, STOP_POLL_QUANTUM_NS), Some(20));
2977 // Even a burst of 1000 completions in one quantum caps at 1000*20 =
2978 // 20_000/s -- not the millions an unfloored 1us window would yield.
2979 assert_eq!(
2980 control_loop_rps_sample(1_000, STOP_POLL_QUANTUM_NS),
2981 Some(20_000)
2982 );
2983 // The common case -- a full ~1s window -- is unchanged: 5000/s.
2984 assert_eq!(control_loop_rps_sample(5_000, 1_000_000_000), Some(5_000));
2985 }
2986
2987 #[test]
2988 fn engine_auto_rps_mode_runs_and_terminates() {
2989 // auto_rps != 0 turns on the once-per-second control thread that
2990 // auto-scales the live rate toward the busy target. Seeded with an
2991 // explicit -R so the injector services requests immediately (progress
2992 // hits 50 well before the first 1s control tick), this proves the
2993 // auto-RPS path boots, injects, and run() terminates cleanly (the control
2994 // thread observes stop; a hang is the nextest timeout). The scaling MATH
2995 // is pinned by the scale_rps_for_busy unit tests; host-busy + target-hit
2996 // timing is environment-dependent, so there is no sample-count assertion.
2997 let config = SchbenchConfig {
2998 message_threads: 1,
2999 worker_threads: 2,
3000 cache_footprint_kib: 16,
3001 operations: 1,
3002 sleep_usec: 0,
3003 skip_locking: false,
3004 requests_per_sec: 10_000,
3005 auto_rps: 50,
3006 split_percent: None,
3007 pipe_transfer_bytes: 0,
3008 };
3009 let stop = AtomicBool::new(false);
3010 let progress = AtomicU64::new(0);
3011 let outcome = std::thread::scope(|s| {
3012 let runner = s.spawn(|| run(&config, &stop, &progress, None));
3013 while progress.load(Ordering::Relaxed) < 50 {
3014 core::hint::spin_loop();
3015 }
3016 stop.store(true, Ordering::Release);
3017 runner.join().expect("run panicked")
3018 });
3019 assert!(
3020 outcome.whole_run.loop_count >= 50,
3021 "auto-RPS injector serviced requests: {}",
3022 outcome.whole_run.loop_count
3023 );
3024 }
3025
3026 #[test]
3027 fn engine_splits_stats_across_phase_epochs() {
3028 // Drive the shared phase_epoch 1 -> 2 mid-run and confirm the engine
3029 // partitions its histograms + loop_count into per-epoch snapshots
3030 // (the scx-phase vs detached-EEVDF-phase mechanism). Event-driven via
3031 // the progress counter, no sleeps.
3032 let config = SchbenchConfig {
3033 message_threads: 1,
3034 worker_threads: 2,
3035 cache_footprint_kib: 16,
3036 operations: 1,
3037 sleep_usec: 0,
3038 skip_locking: false,
3039 requests_per_sec: 0,
3040 auto_rps: 0,
3041 split_percent: None,
3042 pipe_transfer_bytes: 0,
3043 };
3044 let stop = AtomicBool::new(false);
3045 let progress = AtomicU64::new(0);
3046 let epoch = AtomicU32::new(1); // start in a real step epoch (phase 1)
3047 let outcome = std::thread::scope(|s| {
3048 let runner = s.spawn(|| run(&config, &stop, &progress, Some(&epoch)));
3049 while progress.load(Ordering::Relaxed) < 50 {
3050 core::hint::spin_loop();
3051 }
3052 let after_phase1 = progress.load(Ordering::Relaxed);
3053 epoch.store(2, Ordering::Release); // transition to phase 2
3054 while progress.load(Ordering::Relaxed) < after_phase1 + 50 {
3055 core::hint::spin_loop();
3056 }
3057 stop.store(true, Ordering::Release);
3058 runner.join().expect("run panicked")
3059 });
3060
3061 let by_epoch: std::collections::BTreeMap<u32, &SchbenchPhaseStats> =
3062 outcome.phases.iter().map(|(e, s)| (*e, s)).collect();
3063 let p1 = by_epoch.get(&1).expect("phase 1 present");
3064 let p2 = by_epoch.get(&2).expect("phase 2 present");
3065 // Both phases ran work cycles (we waited for >=50 cycles in each
3066 // window). loop_count is the robust split proof: latency samples gate
3067 // on delta_us>0, which a sub-µs cycle can miss, but a cycle always
3068 // increments loop_count.
3069 assert!(p1.loop_count > 0, "phase 1 ran cycles: {}", p1.loop_count);
3070 assert!(p2.loop_count > 0, "phase 2 ran cycles: {}", p2.loop_count);
3071 // Per-phase run-delay split is populated per thread class. This guards a
3072 // DISTINCT path the loop_count/histogram asserts don't touch: the
3073 // schedstat raw-pair baseline/re-baseline (phase_ss_start) + the
3074 // per-class Σ fold (worker vs msg). pcount is the deterministic
3075 // denominator — any thread scheduled in the phase accrues pcount>=1
3076 // (sched_info_arrive), and both phases ran cycles (workers dispatched)
3077 // while the msg thread is scheduled on every wake. run_delay_ns itself
3078 // is NOT asserted: it can be 0 on an uncontended host, so a value check
3079 // would flake; pcount>0 pins that the class was measured and folded.
3080 assert!(
3081 p1.worker_pcount > 0,
3082 "phase 1 worker run-delay split populated"
3083 );
3084 assert!(
3085 p2.worker_pcount > 0,
3086 "phase 2 worker run-delay split populated"
3087 );
3088 assert!(p1.msg_pcount > 0, "phase 1 msg run-delay split populated");
3089 assert!(p2.msg_pcount > 0, "phase 2 msg run-delay split populated");
3090 // loop_count partitions across phases, summing to the global progress —
3091 // no cycle is lost or double-counted at the boundary.
3092 let loop_sum: u64 = outcome.phases.iter().map(|(_, s)| s.loop_count).sum();
3093 assert_eq!(
3094 loop_sum, outcome.whole_run.loop_count,
3095 "per-phase loop_count partitions the whole-run count"
3096 );
3097 // The whole-run histogram is EXACTLY the union of the per-phase ones
3098 // (the drained snapshots, recombined), so the per-phase split loses no
3099 // samples vs the whole-run accounting.
3100 let phase_request_sum: u64 = outcome
3101 .phases
3102 .iter()
3103 .map(|(_, s)| s.request.percentiles().nr_samples)
3104 .sum();
3105 assert_eq!(
3106 outcome.whole_run.request.nr_samples, phase_request_sum,
3107 "whole-run request count == Σ per-phase counts"
3108 );
3109 let phase_wakeup_sum: u64 = outcome
3110 .phases
3111 .iter()
3112 .map(|(_, s)| s.wakeup.percentiles().nr_samples)
3113 .sum();
3114 assert_eq!(
3115 outcome.whole_run.wakeup.nr_samples, phase_wakeup_sum,
3116 "whole-run wakeup count == Σ per-phase counts"
3117 );
3118 // Per-phase RPS: the additive invariant (deterministic) + a timing-tolerant
3119 // bound on the brief first phase. These phases run far under the 1s control
3120 // cadence, so phase 1 normally gets ZERO rps ticks (the ~1s tick lands in
3121 // phase 2, epoch already advanced); on a slow/contended host phase 1 may catch
3122 // a single straddle tick, so we tolerate <= 1 rather than a wall-clock-fragile
3123 // == 0. The deterministic empty-rps -> ABSENT contract is pinned by
3124 // derive_rps_distribution_values_and_absent_guard; this bound still catches
3125 // gross mis-attribution of the bulk of rps to the brief first phase.
3126 assert!(
3127 p1.rps.sample_count() <= 1,
3128 "brief phase 1 gets at most one straddle rps tick, got {}",
3129 p1.rps.sample_count()
3130 );
3131 let phase_rps_sum: u64 = outcome
3132 .phases
3133 .iter()
3134 .map(|(_, s)| s.rps.sample_count())
3135 .sum();
3136 assert_eq!(
3137 outcome.whole_run.rps.nr_samples, phase_rps_sum,
3138 "whole-run rps count == Σ per-phase rps counts"
3139 );
3140 }
3141
3142 #[test]
3143 fn engine_per_phase_rps_populates_and_sums_to_whole_run() {
3144 // POPULATED per-phase RPS: the control thread samples once/sec, so a phase
3145 // window >= ~2s gets multiple per-phase rps samples. Time-gated (the 1s
3146 // control cadence is the granularity floor) -- the progress-gated
3147 // engine_splits phases are too short to tick. Drive epoch 1 for ~2.2s then
3148 // epoch 2 for ~2.2s and confirm BOTH epochs carry rps samples AND the
3149 // per-epoch samples sum to the whole-run count (the additive invariant:
3150 // every sample lands in one epoch bucket AND the whole-run histogram).
3151 // The ~4.4s wall-clock is the irreducible cost of exercising the real 1s control
3152 // cadence end-to-end (2 ticks/epoch for robustness against a delayed tick on a
3153 // loaded host); the deterministic per-epoch invariants — the lockstep reset and
3154 // Σ-per-epoch==whole-run — are pinned sleep-free by
3155 // reset_rps_accumulators_clears_both_in_lockstep + engine_splits_stats_across_phase_epochs.
3156 // This test uniquely confirms the LIVE control thread POPULATES both epochs.
3157 let config = SchbenchConfig {
3158 message_threads: 1,
3159 worker_threads: 2,
3160 cache_footprint_kib: 16,
3161 operations: 1,
3162 sleep_usec: 0,
3163 skip_locking: false,
3164 requests_per_sec: 0,
3165 auto_rps: 0,
3166 split_percent: None,
3167 pipe_transfer_bytes: 0,
3168 };
3169 let stop = AtomicBool::new(false);
3170 let progress = AtomicU64::new(0);
3171 let epoch = AtomicU32::new(1);
3172 let outcome = std::thread::scope(|s| {
3173 let runner = s.spawn(|| run(&config, &stop, &progress, Some(&epoch)));
3174 // ~2.2s per epoch => >= 2 control ticks each (first tick at ~1s).
3175 std::thread::sleep(std::time::Duration::from_millis(2200));
3176 epoch.store(2, Ordering::Release);
3177 std::thread::sleep(std::time::Duration::from_millis(2200));
3178 stop.store(true, Ordering::Release);
3179 runner.join().expect("run panicked")
3180 });
3181 let by_epoch: std::collections::BTreeMap<u32, &SchbenchPhaseStats> =
3182 outcome.phases.iter().map(|(e, s)| (*e, s)).collect();
3183 let p1 = by_epoch.get(&1).expect("phase 1 present");
3184 let p2 = by_epoch.get(&2).expect("phase 2 present");
3185 assert!(
3186 p1.rps.sample_count() > 0,
3187 "phase 1 per-phase rps populated: {}",
3188 p1.rps.sample_count()
3189 );
3190 assert!(
3191 p2.rps.sample_count() > 0,
3192 "phase 2 per-phase rps populated: {}",
3193 p2.rps.sample_count()
3194 );
3195 // Additive invariant (no auto-RPS reset in default mode, so EXACT): Σ
3196 // per-epoch rps samples == whole-run rps samples.
3197 let phase_rps_sum: u64 = outcome
3198 .phases
3199 .iter()
3200 .map(|(_, s)| s.rps.sample_count())
3201 .sum();
3202 assert_eq!(
3203 phase_rps_sum, outcome.whole_run.rps.nr_samples,
3204 "Σ per-epoch rps samples == whole-run rps samples"
3205 );
3206 }
3207
3208 #[test]
3209 fn schbench_config_serde_roundtrips() {
3210 // The new serialized type roundtrips unchanged.
3211 let cfg = SchbenchConfig::default()
3212 .message_threads(3)
3213 .worker_threads(7)
3214 .cache_footprint_kib(512)
3215 .operations(9)
3216 .sleep_usec(250)
3217 .skip_locking(true)
3218 // Exercise the new field's Some(p) serde surface, not just the None
3219 // default (the API-review roundtrip requirement).
3220 .split_percent(Some(33))
3221 // Exercise the pipe field's non-default serde surface too.
3222 .pipe_transfer_bytes(4096);
3223 let json = serde_json::to_string(&cfg).expect("SchbenchConfig must serialize");
3224 let back: SchbenchConfig =
3225 serde_json::from_str(&json).expect("SchbenchConfig must deserialize");
3226 assert_eq!(cfg, back, "config roundtrips unchanged");
3227 }
3228
3229 #[test]
3230 fn worktype_schbench_registration_and_serde() {
3231 use crate::workload::WorkType;
3232 let wt = WorkType::schbench(
3233 SchbenchConfig::default()
3234 .message_threads(2)
3235 .worker_threads(4),
3236 );
3237 assert_eq!(wt.name(), "Schbench");
3238 // from_name yields the default-config variant.
3239 assert_eq!(
3240 WorkType::from_name("Schbench"),
3241 Some(WorkType::Schbench {
3242 config: SchbenchConfig::default()
3243 })
3244 );
3245 // The variant serde-roundtrips, carrying its config.
3246 let json = serde_json::to_string(&wt).expect("WorkType::Schbench must serialize");
3247 let back: WorkType =
3248 serde_json::from_str(&json).expect("WorkType::Schbench must deserialize");
3249 assert_eq!(wt, back);
3250 }
3251
3252 #[test]
3253 fn schbench_config_reachable_via_prelude() {
3254 // Regression-pin the prelude placement: test authors construct the
3255 // config via `use ktstr::prelude::*`. Dropping SchbenchConfig from the
3256 // prelude would fail this compile. Also exercises the Eq derive.
3257 let cfg: crate::prelude::SchbenchConfig = crate::prelude::SchbenchConfig::default();
3258 assert_eq!(cfg, SchbenchConfig::default());
3259 }
3260
3261 #[test]
3262 fn read_schedstat_raw_parses_own_and_handles_missing() {
3263 // The current thread has been scheduled, so its /proc/<tid>/schedstat
3264 // parses without panic into a (run_delay, pcount) pair.
3265 let _own = read_schedstat_raw(gettid_self());
3266 // A non-existent tid -> (0,0) via the file-read-failure path (thread
3267 // exited), matching schbench's fopen-failure handling. The parse
3268 // boundaries (incl. the pcount==0 mean guard) are covered below.
3269 assert_eq!(
3270 read_schedstat_raw(-1),
3271 (0, 0),
3272 "absent schedstat yields (0,0), no panic"
3273 );
3274 }
3275
3276 #[test]
3277 fn schedstat_raw_parse_and_mean_pcount_guard() {
3278 // Raw parse keeps run_delay + pcount undivided (the re-poolable pair).
3279 assert_eq!(parse_schedstat_raw("123456 50 5"), (50, 5));
3280 // mean_sched_delay divides with the pcount==0 guard (no div-by-zero).
3281 assert_eq!(mean_sched_delay((50, 5)), 10); // run_delay/pcount
3282 assert_eq!(mean_sched_delay((50, 0)), 0); // pcount==0 guard
3283 assert_eq!(mean_sched_delay(parse_schedstat_raw("0 0 0")), 0); // !sched_info_on()
3284 assert_eq!(mean_sched_delay((0, 5)), 0); // 0 run_delay -> 0 mean
3285 }
3286
3287 #[test]
3288 #[should_panic(expected = "schedstat field 3")]
3289 fn parse_schedstat_raw_short_line_panics() {
3290 // A present-but-short line is a kernel/parse bug: fail loud, not a
3291 // silent 0 (the fail-loud ruling).
3292 parse_schedstat_raw("100 50");
3293 }
3294
3295 #[test]
3296 #[should_panic(expected = "schedstat field 2")]
3297 fn parse_schedstat_raw_nonnumeric_panics() {
3298 parse_schedstat_raw("alpha beta gamma");
3299 }
3300}