ktstr/workload/taobench/
run.rs

1//! The taobench engine: a bounded, sharded, evicting key-value cache served by a
2//! client population (closed-loop by default, or open-loop fixed-rate arrival),
3//! with a fast in-cache path (hit) and a slow backing-store-miss path (a
4//! worker-defined sleep on a dispatcher thread). It is
5//! entirely in-process — no sockets, no TLS, no subprocess — and re-expresses the
6//! taobench ACCESS PATTERN in ktstr primitives so its qps / hit-ratio flow
7//! through the metric API. See `validation.md` for the per-aspect real-vs-port
8//! fidelity comparison.
9//!
10//! ## Model
11//!
12//! - A `Cache` of `SHARDS` independently-locked shards, each a FIFO-evicting
13//!   map bounded to a per-shard object cap. Total capacity is `cache_capacity_mib`
14//!   worth of objects; the key range is sized `capacity / target_hit` so a
15//!   uniform-random key stream hits the resident set with probability
16//!   ≈ `target_hit` at equilibrium. Eviction is the load-bearing mechanism: with
17//!   no eviction a self-healing cache drifts to a 1.0 hit ratio; a bounded cache
18//!   whose key range exceeds its capacity holds a steady-state miss stream.
19//! - `client_threads` CLIENT threads pick a key, look it up, and on a HIT touch
20//!   the stored value bytes (the cache read-bandwidth cost) and count a fast op;
21//!   on a MISS hand the key to a slow dispatcher and block until it is filled,
22//!   then count a slow op.
23//! - Arrival: `arrival_rate == 0` (default) is CLOSED loop — each client issues
24//!   its next request as soon as the prior completes. `arrival_rate > 0` is OPEN
25//!   loop — each client has a fixed intended-arrival SCHEDULE
26//!   (`arrival_rate / client_threads` per client) independent of completion, and
27//!   serve latency is measured from that intended time. A client still holds at
28//!   most one outstanding request (it blocks on a miss before issuing the next),
29//!   so a slow completion delays the next issue; that backlog is folded into the
30//!   late requests' serve latency (coordinated-omission correction) instead of
31//!   being omitted. The serve-latency histogram is per-phase data (empty in
32//!   closed loop).
33//! - `slow_threads` DISPATCHER threads serve misses: sleep the simulated
34//!   backing-store fetch (a worker-defined cost, the same model schbench's
35//!   think-sleep uses, not a synchronization wait), insert a freshly sized+touched
36//!   value, and wake the waiting client. The fetch latency is a fixed
37//!   `slow_path_sleep_us` by default, or — when `slow_path_p99_us` is set above it
38//!   — a per-fetch heavy-tailed Pareto draw with median `slow_path_sleep_us` and
39//!   99th percentile `slow_path_p99_us` (most fetches near the median, a rare slow
40//!   tail), so the open-loop serve-latency tail reflects realistic backing-store
41//!   variance.
42//!
43//! ## Counters (request-time vs response-time)
44//!
45//! `get_cmds` / `get_misses` are counted at LOOKUP (request) time; `fast_ops` /
46//! `slow_ops` at COMPLETION (response) time. In a closed loop they are equal once
47//! every in-flight request drains, but a request that straddles a phase boundary
48//! lands its lookup in one phase and its completion in the next, so a per-phase
49//! command-time hit_rate (`1 - get_misses/get_cmds`) differs slightly from the
50//! response-time hit_ratio (`fast_ops/(fast_ops+slow_ops)`) — the same skew the
51//! real reports between its interval hit_rate and its final hit_ratio.
52
53use crate::workload::schbench::plat::PlatStats;
54use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
55use std::collections::HashMap;
56use std::collections::{BTreeMap, VecDeque};
57use std::sync::{Condvar, Mutex};
58
59/// Number of independently-locked cache shards (power of two). Sized to keep
60/// per-shard lock contention low for typical guest thread counts while bounding
61/// the lock-array footprint; the cache capacity is split evenly across shards.
62const SHARDS: usize = 256;
63
64/// Representative value-size distribution: a small-object-heavy, long-tailed
65/// histogram (bytes → relative weight) approximating the empirical object-size
66/// profile of the real workload (mean ≈ 332 B, tail to 64 KiB). This is ktstr's
67/// own approximation, not a copy of any external size table.
68const VALUE_SIZES: [usize; 8] = [64, 128, 256, 512, 1024, 4096, 16384, 65536];
69const VALUE_WEIGHTS: [u32; 8] = [450, 300, 130, 70, 30, 12, 3, 1];
70
71/// Cap on a single open-loop pacing sleep (ns). A client waiting for its next
72/// scheduled arrival re-checks `stop` at least this often, so a shutdown is
73/// observed within this bound rather than after a full inter-arrival interval —
74/// which at a degenerate low `arrival_rate` (`interval_ns = 1e9 * n_clients /
75/// arrival_rate`) can exceed the scenario cleanup budget and risk a watchdog kill.
76/// At realistic open-loop rates the per-arrival interval is well under this, so the
77/// pacing sleep is a single uncapped wait — the cap only engages at pathological
78/// low rates.
79const STOP_POLL_QUANTUM_NS: u64 = 50_000_000;
80
81/// Read `CLOCK_MONOTONIC` as nanoseconds (monotonic, not wall-clock), matching
82/// the schbench engine's clock source.
83fn monotonic_nanos() -> u64 {
84    // SAFETY: `clock_gettime` writes a `timespec` through the out-pointer and
85    // reads nothing else; CLOCK_MONOTONIC is always available on Linux.
86    let mut ts: libc::timespec = unsafe { core::mem::zeroed() };
87    let rc = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut ts) };
88    assert_eq!(rc, 0, "clock_gettime(CLOCK_MONOTONIC) failed");
89    (ts.tv_sec as u64) * 1_000_000_000 + ts.tv_nsec as u64
90}
91
92/// The simulated backing-store fetch on the slow path. Worker-defined cost (like
93/// schbench's think-sleep), not a synchronization wait: `std::thread::sleep` maps
94/// to `clock_nanosleep` on Linux. `0` is a no-op (the dispatcher still does the
95/// fill + wakeup, so the slow path remains a distinct thread hop). The caller
96/// passes either the fixed `slow_path_sleep_us` or, when the heavy tail is enabled,
97/// a per-fetch [`sample_service_us`] Pareto draw — this function just sleeps `usec`.
98fn backing_store_fetch(usec: u64) {
99    if usec > 0 {
100        std::thread::sleep(std::time::Duration::from_micros(usec));
101    }
102}
103
104/// Upper bound on a single heavy-tailed slow-path fetch (µs). Pareto is unbounded;
105/// a pathological tail draw must not park a dispatcher off-CPU for an unbounded
106/// time — the dispatcher must complete the fetch + fill + wake for the blocked
107/// client (it cannot be interrupted on shutdown without breaking that contract), so
108/// this caps each fetch at 2 s (far above any realistic backing-store p99.9). The
109/// worst-case client-join/teardown is NOT a single fetch: at shutdown up to
110/// `n_clients` misses are outstanding and each dispatcher drains its queue share
111/// SERIALLY, so the last client can wait up to `ceil(n_clients / n_slow)` fetches —
112/// with the default staffing (`n_slow ≈ n_clients/3`) ~3·MAX_SERVICE_US. Size the
113/// scenario watchdog / cleanup budget accordingly when configuring a deep tail with
114/// many clients per dispatcher. This caps only the per-FETCH off-CPU time, not a
115/// whole serve-latency sample (fetch + open-loop queueing/coordinated-omission
116/// backlog, which can exceed it); the serve sample is kept inside the `PlatStats`
117/// u32 range by `record_serve_lat`'s own `(ns/1000).min(u32::MAX)` clamp,
118/// independent of this cap. Only the fixed legacy path is uncapped (the user sets
119/// that latency directly).
120const MAX_SERVICE_US: u64 = 2_000_000;
121
122/// Draw one heavy-tailed slow-path service time (µs) from the Pareto resolved by
123/// [`TaobenchConfig::resolve_service_pareto`]: `scale · U^(−1/α)` with `U ∈ (0, 1]`
124/// ([`Rng::f64_open01`]). `U = 1` yields the scale (the floor, below the median);
125/// smaller `U` yields the tail. Clamped to [`MAX_SERVICE_US`]. `inv_neg_alpha`
126/// is the precomputed `−1/α` exponent.
127fn sample_service_us(rng: &mut Rng, scale: f64, inv_neg_alpha: f64) -> u64 {
128    let us = scale * rng.f64_open01().powf(inv_neg_alpha);
129    (us as u64).min(MAX_SERVICE_US)
130}
131
132/// Touch every byte of a served/filled value so the read cannot be elided — the
133/// cache memory-bandwidth cost that makes this a cache workload rather than a
134/// control-flow micro-benchmark (schbench's `black_box`-guarded memset is the
135/// precedent). Returns the (black-boxed) checksum.
136fn touch(bytes: &[u8]) -> u64 {
137    let mut acc = 0u64;
138    for &b in bytes {
139        acc = acc.wrapping_add(b as u64);
140    }
141    std::hint::black_box(acc)
142}
143
144/// A small, fast, per-thread xorshift64 PRNG (workload key/size sampling only —
145/// not cryptographic). Seeded distinctly per thread so threads do not march in
146/// lockstep.
147struct Rng(u64);
148
149impl Rng {
150    fn new(seed: u64) -> Self {
151        // Avoid the xorshift fixed point at 0.
152        Rng(seed ^ 0x9E37_79B9_7F4A_7C15)
153    }
154    fn next_u64(&mut self) -> u64 {
155        let mut x = self.0;
156        x ^= x << 13;
157        x ^= x >> 7;
158        x ^= x << 17;
159        self.0 = x;
160        x
161    }
162    /// Uniform in `[0, n)` (`n > 0`); a modulo reduction — the tiny bias is
163    /// irrelevant for a workload key stream.
164    fn below(&mut self, n: u64) -> u64 {
165        self.next_u64() % n
166    }
167    /// Uniform `f64` in `(0, 1]` (open at 0, closed at 1) from the top 53 bits —
168    /// the `OpenClosed01` form the Pareto inverse-CDF needs ([`sample_service_us`]):
169    /// `U = 0` would make `U^(−1/α)` infinite, so 0 is excluded. Workload sampling
170    /// only, not cryptographic.
171    fn f64_open01(&mut self) -> f64 {
172        // Top 53 bits + 1 -> integer in 1..=2^53; / 2^53 -> (0, 1].
173        let bits = (self.next_u64() >> 11) + 1;
174        bits as f64 * (1.0 / ((1u64 << 53) as f64))
175    }
176}
177
178/// Sample a value size from [`VALUE_SIZES`] weighted by [`VALUE_WEIGHTS`].
179fn sample_value_size(rng: &mut Rng) -> usize {
180    let total: u32 = VALUE_WEIGHTS.iter().sum();
181    let mut pick = (rng.below(total as u64)) as u32;
182    for (i, &w) in VALUE_WEIGHTS.iter().enumerate() {
183        if pick < w {
184            return VALUE_SIZES[i];
185        }
186        pick -= w;
187    }
188    VALUE_SIZES[VALUE_SIZES.len() - 1]
189}
190
191/// Build a freshly-allocated, byte-filled value of `size` (the fill touches every
192/// byte, the write-side bandwidth cost).
193fn make_value(size: usize) -> Box<[u8]> {
194    vec![0xABu8; size].into_boxed_slice()
195}
196
197// ---------------------------------------------------------------------------
198// Config
199// ---------------------------------------------------------------------------
200
201/// User-facing config for the [`Taobench`](crate::workload::WorkType::Taobench)
202/// workload — a bounded, evicting key-value cache with a fast hit path and a slow
203/// miss path, driven to a steady-state hit ratio.
204///
205/// All fields are integer/scalar so the type keeps `Eq + Hash` (fractional knobs
206/// are expressed as integer percents). Every field has a chainable builder
207/// setter; [`Default`] is a useful working config.
208#[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
209#[serde(rename_all = "snake_case")]
210pub struct TaobenchConfig {
211    /// CLIENT threads that issue lookups and serve hits. `0` resolves
212    /// to the allocated guest cpuset CPU count (one client per CPU).
213    pub client_threads: usize,
214    /// SLOW dispatcher threads that serve misses (sleep + fill + wake). `0`
215    /// resolves to `max(1, client_threads / 3)`, the real's fast:slow staffing
216    /// ratio.
217    pub slow_threads: usize,
218    /// Resident cache budget in MiB. The cache FIFO-evicts to stay near this many
219    /// bytes' worth of objects; larger than the guest LLC makes the value touch a
220    /// real memory-bandwidth cost.
221    pub cache_capacity_mib: usize,
222    /// Target steady-state hit ratio, in percent (`1..=99`). The key range is
223    /// sized `capacity / target_hit` so a uniform key stream hits at this rate at
224    /// equilibrium. Clamped into range at consumption.
225    pub target_hit_pct: usize,
226    /// Simulated backing-store fetch latency on a miss, in microseconds (the slow
227    /// dispatcher sleeps this long before filling). `0` keeps the slow path as a
228    /// pure thread hop with no sleep (and disables the heavy tail — a zero median
229    /// has no Pareto scale, so the fetch stays a no-op regardless of
230    /// `slow_path_p99_us`). When `slow_path_sleep_us > 0` and `slow_path_p99_us >
231    /// slow_path_sleep_us` this is the MEDIAN (p50) of a heavy-tailed service-time
232    /// distribution rather than a fixed latency (see `slow_path_p99_us`).
233    pub slow_path_sleep_us: u64,
234    /// Heavy-tailed slow-path service time: the p99 of the simulated backing-store
235    /// fetch, in microseconds. When `> slow_path_sleep_us`, each miss's fetch sleep
236    /// is drawn from a Pareto distribution whose median is `slow_path_sleep_us` and
237    /// whose 99th percentile is this value — most fetches near the median, a heavy
238    /// tail of rare slow fetches (GC pauses, cold cache, disk seeks), the realistic
239    /// power-law shape a backing store exhibits. `0` (default), any value
240    /// `<= slow_path_sleep_us`, OR `slow_path_sleep_us == 0` (a zero median has no
241    /// Pareto scale) keeps the fixed-latency behavior (every fetch sleeps exactly
242    /// `slow_path_sleep_us` — the legacy path, byte-identical). The tail is
243    /// clamped at a fixed maximum so a pathological draw cannot park a dispatcher
244    /// off-CPU unboundedly. A ktstr enhancement beyond the reference, whose
245    /// per-request slow-path service time is a FIXED sleep with no tail (the
246    /// reference's `[target/2, 2×target]` uniform jitter + idle-poll backoff is a
247    /// separate poll/idle-wait knob, not the per-request fetch), so the open-loop
248    /// serve-latency tail ([`crate::workload::TaobenchConfig::arrival_rate`])
249    /// reflects realistic
250    /// backing-store variance.
251    pub slow_path_p99_us: u64,
252    /// Open-loop arrival rate, AGGREGATE ops/sec across all client threads (the
253    /// taobench analog of schbench's `-R`). `0` (default) = CLOSED loop: each
254    /// client blocks until its request completes before issuing the next (the
255    /// legacy behavior, byte-identical). Non-zero = OPEN loop: each client has a
256    /// fixed intended-arrival SCHEDULE (`arrival_rate / client_threads` per client)
257    /// independent of completion, and serve latency is measured from that intended
258    /// time. A client still holds at most one outstanding request, so a slow
259    /// completion delays the next issue; that backlog is folded into the late
260    /// requests' serve latency (coordinated-omission correction) rather than
261    /// omitted. Divided evenly across resolved clients. (Named for the standard
262    /// queueing-theory term; the schbench analog is `requests_per_sec`, which is a
263    /// verbatim mirror of schbench's own `-R` CLI flag — a constraint this
264    /// ktstr-added field, with no reference `-R` to mirror, does not share.)
265    pub arrival_rate: usize,
266}
267
268impl Default for TaobenchConfig {
269    fn default() -> Self {
270        Self {
271            client_threads: 0,
272            slow_threads: 0,
273            cache_capacity_mib: 64,
274            target_hit_pct: 90,
275            slow_path_sleep_us: 100,
276            slow_path_p99_us: 0,
277            arrival_rate: 0,
278        }
279    }
280}
281
282impl TaobenchConfig {
283    /// Set the client thread count (`0` = one per allocated CPU).
284    #[must_use = "builder methods consume self; bind the result"]
285    pub fn client_threads(mut self, n: usize) -> Self {
286        self.client_threads = n;
287        self
288    }
289    /// Set the slow dispatcher thread count (`0` = `max(1, client_threads/3)`).
290    #[must_use = "builder methods consume self; bind the result"]
291    pub fn slow_threads(mut self, n: usize) -> Self {
292        self.slow_threads = n;
293        self
294    }
295    /// Set the resident cache budget in MiB.
296    #[must_use = "builder methods consume self; bind the result"]
297    pub fn cache_capacity_mib(mut self, mib: usize) -> Self {
298        self.cache_capacity_mib = mib;
299        self
300    }
301    /// Set the target steady-state hit ratio in percent (`1..=99`).
302    #[must_use = "builder methods consume self; bind the result"]
303    pub fn target_hit_pct(mut self, pct: usize) -> Self {
304        self.target_hit_pct = pct;
305        self
306    }
307    /// Set the simulated backing-store fetch latency on a miss, microseconds (the
308    /// MEDIAN when `slow_path_p99_us` enables the heavy tail; `0` keeps the fetch a
309    /// no-op and also disables the tail — a zero median has no Pareto scale).
310    #[must_use = "builder methods consume self; bind the result"]
311    pub fn slow_path_sleep_us(mut self, us: u64) -> Self {
312        self.slow_path_sleep_us = us;
313        self
314    }
315    /// Set the heavy-tailed slow-path service-time p99 in microseconds (`0` or
316    /// `<= slow_path_sleep_us` = fixed latency, the legacy behavior; also fixed when
317    /// `slow_path_sleep_us == 0`, a zero median having no Pareto scale). When larger
318    /// than a non-zero `slow_path_sleep_us`, each miss's fetch is drawn from a Pareto
319    /// with median `slow_path_sleep_us` and this p99.
320    #[must_use = "builder methods consume self; bind the result"]
321    pub fn slow_path_p99_us(mut self, us: u64) -> Self {
322        self.slow_path_p99_us = us;
323        self
324    }
325    /// Set the open-loop AGGREGATE arrival rate in ops/sec across all clients
326    /// (`0` = closed loop). Divided evenly across resolved client threads; serve
327    /// latency is then measured from the intended arrival (coordinated-omission).
328    #[must_use = "builder methods consume self; bind the result"]
329    pub fn arrival_rate(mut self, ops_per_sec: usize) -> Self {
330        self.arrival_rate = ops_per_sec;
331        self
332    }
333
334    /// Resolve the client thread count: the configured value, or the allocated
335    /// CPU count when `0`.
336    fn resolve_client_threads(&self, allowed_cpus: usize) -> usize {
337        if self.client_threads == 0 {
338            allowed_cpus.max(1)
339        } else {
340            self.client_threads
341        }
342    }
343
344    /// Resolve the slow dispatcher count: the configured value, or
345    /// `max(1, clients/3)` when `0`.
346    fn resolve_slow_threads(&self, clients: usize) -> usize {
347        if self.slow_threads == 0 {
348            (clients / 3).max(1)
349        } else {
350            self.slow_threads
351        }
352    }
353
354    /// Clamp the target hit ratio to a usable open interval (a 0% or ≥100% target
355    /// has no finite key range / no miss stream).
356    fn target_hit_fraction(&self) -> f64 {
357        (self.target_hit_pct.clamp(1, 99) as f64) / 100.0
358    }
359
360    /// Resolve the heavy-tailed slow-path service-time model from the `(p50, p99)`
361    /// knob: `None` = fixed `slow_path_sleep_us` (the legacy path); `Some((scale,
362    /// inv_neg_alpha))` = a Pareto whose median is `slow_path_sleep_us` and whose
363    /// p99 is `slow_path_p99_us`, sampled by [`sample_service_us`].
364    ///
365    /// `slow_path_p99_us <= slow_path_sleep_us` (incl. `0`) resolves to `None` — a
366    /// p99 at or below the median is not a tail. `slow_path_sleep_us == 0` also
367    /// resolves to `None`: a zero median has no Pareto scale, and the legacy no-op
368    /// fetch is preserved.
369    ///
370    /// Mapping: the Pareto q-quantile is `scale·(1−q)^(−1/α)`, so from the two
371    /// quantiles `p99/p50 = (0.01/0.5)^(−1/α) = 50^(1/α)`, giving
372    /// `α = ln(50) / ln(p99/p50)` and `scale = p50·0.5^(1/α)`. `p99 > p50`
373    /// guarantees `ln(p99/p50) > 0 ⇒ α > 0`, and `α ≥ ln(50)/ln(spread)` keeps the
374    /// mean finite (`α > 1`) for any spread below 50x. The stored `inv_neg_alpha =
375    /// −1/α` is the exponent [`sample_service_us`] raises `U` to.
376    fn resolve_service_pareto(&self) -> Option<(f64, f64)> {
377        let p50 = self.slow_path_sleep_us;
378        let p99 = self.slow_path_p99_us;
379        if p50 == 0 || p99 <= p50 {
380            return None;
381        }
382        let (p50f, p99f) = (p50 as f64, p99 as f64);
383        let alpha = 50.0_f64.ln() / (p99f / p50f).ln();
384        let scale = p50f * 0.5_f64.powf(1.0 / alpha);
385        Some((scale, -1.0 / alpha))
386    }
387}
388
389// ---------------------------------------------------------------------------
390// Cache
391// ---------------------------------------------------------------------------
392
393/// One cache shard: a FIFO-evicting map bounded to `cap` objects. FIFO eviction
394/// over a uniform key stream yields the same equilibrium hit ratio as LRU
395/// (resident_fraction = cap / key_range) while being O(1) and lock-cheap.
396struct Shard {
397    map: HashMap<u64, Box<[u8]>>,
398    fifo: VecDeque<u64>,
399    cap: usize,
400}
401
402impl Shard {
403    fn with_cap(cap: usize) -> Self {
404        Shard {
405            map: HashMap::new(),
406            fifo: VecDeque::new(),
407            cap,
408        }
409    }
410    /// Look up `k`; on a hit, touch its bytes and return `true`.
411    fn get_touch(&self, k: u64) -> bool {
412        match self.map.get(&k) {
413            Some(v) => {
414                touch(v);
415                true
416            }
417            None => false,
418        }
419    }
420    /// Insert a freshly-built value for `k` — called at warmup and on a miss,
421    /// where `k` is absent; the `is_none()` guard keeps it correct even if `k`
422    /// is already present. FIFO-evicts down to `cap`.
423    fn insert(&mut self, k: u64, v: Box<[u8]>) {
424        if self.map.insert(k, v).is_none() {
425            self.fifo.push_back(k);
426            while self.map.len() > self.cap {
427                match self.fifo.pop_front() {
428                    Some(old) => {
429                        self.map.remove(&old);
430                    }
431                    None => break,
432                }
433            }
434        }
435    }
436}
437
438/// A bounded, sharded, FIFO-evicting cache. Keys map to shards by the low bits of
439/// the key (`SHARDS` is a power of two).
440struct Cache {
441    shards: Vec<Mutex<Shard>>,
442}
443
444impl Cache {
445    /// Build a cache holding ≈ `total_objects` across `SHARDS` shards.
446    fn new(total_objects: usize) -> Self {
447        let per_shard = (total_objects / SHARDS).max(1);
448        let shards = (0..SHARDS)
449            .map(|_| Mutex::new(Shard::with_cap(per_shard)))
450            .collect();
451        Cache { shards }
452    }
453    fn shard(&self, k: u64) -> &Mutex<Shard> {
454        &self.shards[(k as usize) & (SHARDS - 1)]
455    }
456    /// Look up + touch `k`; `true` on a hit.
457    fn get_touch(&self, k: u64) -> bool {
458        self.shard(k)
459            .lock()
460            .expect("cache shard poisoned")
461            .get_touch(k)
462    }
463    /// Fill `k` with a freshly sized + built value (the value bytes are touched
464    /// by `make_value`); FIFO-evicts to keep the shard bounded.
465    fn fill(&self, k: u64, size: usize) {
466        let v = make_value(size);
467        self.shard(k)
468            .lock()
469            .expect("cache shard poisoned")
470            .insert(k, v);
471    }
472}
473
474// ---------------------------------------------------------------------------
475// Slow-path handoff
476// ---------------------------------------------------------------------------
477
478/// A miss handed from a client to a slow dispatcher. The client measures serve
479/// latency from its own local intended-arrival timestamp after `wait_filled`, so
480/// the request itself carries no timing — only the key + the client to wake.
481struct SlowReq {
482    key: u64,
483    client: usize,
484}
485
486/// The slow-request queue: dispatchers block here when idle; clients push misses
487/// and wake one dispatcher.
488struct SlowQueue {
489    q: Mutex<VecDeque<SlowReq>>,
490    cv: Condvar,
491}
492
493impl SlowQueue {
494    fn new() -> Self {
495        SlowQueue {
496            q: Mutex::new(VecDeque::new()),
497            cv: Condvar::new(),
498        }
499    }
500    fn push(&self, req: SlowReq) {
501        self.q.lock().expect("slow queue poisoned").push_back(req);
502        self.cv.notify_one();
503    }
504}
505
506/// A per-client response slot: the dispatcher sets `done` + notifies after the
507/// fill; the client blocks here until its outstanding miss is served. At most one
508/// outstanding request per client in BOTH arrival modes (a client blocks on
509/// `wait_filled` for a miss before issuing its next request), so the slot is
510/// reused. This single-outstanding-request serialization is exactly why open-loop
511/// serve latency must be measured from the intended arrival (coordinated-omission):
512/// a slow completion delays the next issue, and that backlog is folded into the
513/// late requests' latency instead of omitted.
514struct Slot {
515    done: Mutex<bool>,
516    cv: Condvar,
517}
518
519impl Slot {
520    fn new() -> Self {
521        Slot {
522            done: Mutex::new(false),
523            cv: Condvar::new(),
524        }
525    }
526    /// Block until the dispatcher marks this slot done, then reset it.
527    fn wait_filled(&self) {
528        let mut done = self.done.lock().expect("slot poisoned");
529        while !*done {
530            done = self.cv.wait(done).expect("slot poisoned");
531        }
532        *done = false;
533    }
534    /// Mark this slot filled and wake the waiting client.
535    fn signal(&self) {
536        *self.done.lock().expect("slot poisoned") = true;
537        self.cv.notify_one();
538    }
539}
540
541// ---------------------------------------------------------------------------
542// Stats
543// ---------------------------------------------------------------------------
544
545/// Taobench engine counters for one accounting window — a single phase
546/// epoch (the per-phase `crate::workload::PhaseSlice::taobench` carrier) or a
547/// whole worker run (the [`crate::workload::WorkerReport::taobench_whole`] /
548/// [`crate::assert::CgroupStats::taobench_whole`] aggregate). Integer-only so the
549/// enclosing `PhaseSlice` keeps `Eq`. `get_cmds` / `get_misses` are request-time;
550/// `fast_ops` / `slow_ops` are response-time (see the module docs). `Self::merge`
551/// pools two windows (Σ ops, MAX wall) and [`Self::total_ops`] is the throughput
552/// numerator; the host derives the run-level `taobench_*` Rate metrics from the
553/// pooled aggregate.
554#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
555pub struct TaobenchStats {
556    /// Lookups issued (request time).
557    pub get_cmds: u64,
558    /// Lookups that missed (request time).
559    pub get_misses: u64,
560    /// Hits served (response time).
561    pub fast_ops: u64,
562    /// Misses served via the slow path (response time).
563    pub slow_ops: u64,
564    /// Wall-clock window this stat covers, ns. Per-phase: the phase segment;
565    /// whole-run: the run window. Merged as MAX (the window is shared by the
566    /// concurrent threads/workers being pooled, not summed). NOT the throughput
567    /// denominator anymore (that is `cpu_time_ns`); kept for the standalone /
568    /// validate surface (upstream wall-qps shape) and as the window evidence
569    /// from which the wall picture reconstructs exactly (`ops / elapsed`).
570    pub elapsed_ns: u64,
571    /// Σ client-thread `CLOCK_THREAD_CPUTIME_ID` over the window, ns — the
572    /// CPU time the clients actually received, the
573    /// `taobench_*_ops_per_cpu_sec` DENOMINATOR. CPU-clock denominated so the
574    /// throughput measures the workload, not host/guest scheduling delay;
575    /// under host dilation the wall picture is `cpu_rate / D` (the sidecar's
576    /// `host_dilation`). Dispatcher (slow-path service) CPU is EXCLUDED —
577    /// dispatchers model backing-store latency (mostly sleeping), and ops
578    /// complete on client threads. Merged as SUM (per-thread CPU is
579    /// disjoint, unlike the shared wall window). `#[serde(default)]`:
580    /// pre-field sidecars deserialize with 0 (the throughput keys then read
581    /// absent, never a false rate).
582    #[serde(default)]
583    pub cpu_time_ns: u64,
584}
585
586impl TaobenchStats {
587    pub(crate) fn merge(&mut self, o: &TaobenchStats) {
588        // saturating_add: these are guest-runtime monotonic op counters pooled
589        // across concurrent workers/cgroups. Overflow is unreachable for honest
590        // data (real counts ~1e7..1e14 << u64::MAX), but a corrupt/hostile guest
591        // value would otherwise debug-panic or release-wrap to a silently-wrong
592        // qps/hit Rate; saturating is exact for every in-range value.
593        self.get_cmds = self.get_cmds.saturating_add(o.get_cmds);
594        self.get_misses = self.get_misses.saturating_add(o.get_misses);
595        self.fast_ops = self.fast_ops.saturating_add(o.fast_ops);
596        self.slow_ops = self.slow_ops.saturating_add(o.slow_ops);
597        // The wall window is shared across the pooled concurrent threads/workers
598        // (they run the same phase at the same time), so MAX, not sum.
599        self.elapsed_ns = self.elapsed_ns.max(o.elapsed_ns);
600        // Per-thread CPU is disjoint across the pooled threads/workers: SUM.
601        self.cpu_time_ns = self.cpu_time_ns.saturating_add(o.cpu_time_ns);
602    }
603    /// Completed ops (fast + slow) — the throughput numerator.
604    pub fn total_ops(&self) -> u64 {
605        self.fast_ops.saturating_add(self.slow_ops)
606    }
607}
608
609/// One PER-PHASE accounting window's taobench carrier: the integer counters
610/// ([`TaobenchStats`]) plus the open-loop serve-latency histogram. Internal
611/// (`pub(crate)`) — the taobench analog of
612/// [`crate::workload::schbench::run::SchbenchPhaseStats`] (counters + `PlatStats`
613/// histograms together), and like it, never on a public field. It rides only the
614/// per-phase carriers (`PhaseSlice::taobench` / `PhaseCgroupStats::taobench`); the
615/// WHOLE-RUN carrier is the counters-only [`TaobenchStats`] (the serve histogram
616/// is per-phase data — the whole-run serve distribution is the union of these
617/// per-phase histograms, computed where needed). `serve_lat` is EMPTY in closed
618/// loop: serve latency is measured only under open-loop arrival, so its
619/// percentiles read absent there.
620#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
621pub(crate) struct TaobenchPhaseStats {
622    /// Throughput / hit counters (request- and response-time).
623    pub(crate) counters: TaobenchStats,
624    /// Coordinated-omission serve latency (µs), measured from intended arrival.
625    /// Exposed to test authors only via the derived `taobench_serve_*_us` metric
626    /// keys (the schbench-histogram model), never as a direct field.
627    pub(crate) serve_lat: PlatStats,
628}
629
630impl TaobenchPhaseStats {
631    /// Pool another window: Σ counters ([`TaobenchStats::merge`]) and union the
632    /// serve histograms ([`PlatStats::combine`]) — the per-cgroup / cross-thread
633    /// fold, mirroring the schbench carrier merge.
634    pub(crate) fn merge(&mut self, other: &TaobenchPhaseStats) {
635        self.counters.merge(&other.counters);
636        self.serve_lat.combine(&other.serve_lat);
637    }
638}
639
640/// The engine's return: the whole-run merged COUNTERS ([`TaobenchStats`], the
641/// qps/hit aggregate) and the per-phase-epoch carriers (each counters + the
642/// serve-latency histogram). The whole-run carrier is counters-only — mirroring
643/// the wire carriers (`WorkerReport::taobench_whole` / `CgroupStats::taobench_whole`);
644/// a whole-run serve distribution, when needed (the standalone driver), is the
645/// union of the per-phase `serve_lat` histograms.
646pub(crate) struct TaobenchOutcome {
647    pub whole_run: TaobenchStats,
648    pub phases: Vec<(u32, TaobenchPhaseStats)>,
649}
650
651/// Per-thread accumulation with phase-epoch bucketing: the current epoch's
652/// counters roll into `phases` when the epoch changes (the schbench drain-on-epoch
653/// pattern), and `whole` accumulates the un-bucketed run total. Wall windows are
654/// stamped from `CLOCK_MONOTONIC` at each roll (per-phase segment) and at finalize
655/// (whole-run).
656struct ThreadAccum {
657    cur_epoch: u32,
658    cur: TaobenchPhaseStats,
659    phases: BTreeMap<u32, TaobenchPhaseStats>,
660    /// Whole-run counters only (the qps/hit aggregate). The serve histogram is
661    /// per-phase data (in `cur`/`phases`); the whole-run serve distribution, when
662    /// needed, is the union of the per-phase histograms.
663    whole: TaobenchStats,
664    /// When the current phase segment started (ns).
665    phase_start_ns: u64,
666    /// When this thread started (ns) — the whole-run window start.
667    thread_start_ns: u64,
668    /// `CLOCK_THREAD_CPUTIME_ID` at the current phase segment's start (ns) —
669    /// baselines the per-phase client-CPU delta (the CPU-qps denominator).
670    phase_start_cpu_ns: u64,
671    /// `CLOCK_THREAD_CPUTIME_ID` at thread start (ns) — baselines the
672    /// whole-run client-CPU delta.
673    thread_start_cpu_ns: u64,
674}
675
676impl ThreadAccum {
677    fn new(epoch: u32) -> Self {
678        let now = monotonic_nanos();
679        let cpu_now = crate::workload::worker::sched::thread_cpu_time_ns();
680        ThreadAccum {
681            cur_epoch: epoch,
682            cur: TaobenchPhaseStats::default(),
683            phases: BTreeMap::new(),
684            whole: TaobenchStats::default(),
685            phase_start_ns: now,
686            thread_start_ns: now,
687            phase_start_cpu_ns: cpu_now,
688            thread_start_cpu_ns: cpu_now,
689        }
690    }
691    /// Roll the current bucket into `phases` (stamping its segment wall + CPU
692    /// windows) and start a fresh one for `epoch`.
693    fn roll_to(&mut self, epoch: u32) {
694        if epoch != self.cur_epoch {
695            let now = monotonic_nanos();
696            let cpu_now = crate::workload::worker::sched::thread_cpu_time_ns();
697            let mut cur = std::mem::take(&mut self.cur);
698            cur.counters.elapsed_ns = now.saturating_sub(self.phase_start_ns);
699            cur.counters.cpu_time_ns = cpu_now.saturating_sub(self.phase_start_cpu_ns);
700            self.phases.entry(self.cur_epoch).or_default().merge(&cur);
701            self.cur_epoch = epoch;
702            self.phase_start_ns = now;
703            self.phase_start_cpu_ns = cpu_now;
704        }
705    }
706    /// Record a lookup at request time in `epoch`.
707    fn record_cmd(&mut self, epoch: u32, hit: bool) {
708        self.roll_to(epoch);
709        self.cur.counters.get_cmds += 1;
710        self.whole.get_cmds += 1;
711        if !hit {
712            self.cur.counters.get_misses += 1;
713            self.whole.get_misses += 1;
714        }
715    }
716    /// Record a completion at response time in `epoch`.
717    fn record_complete(&mut self, epoch: u32, hit: bool) {
718        self.roll_to(epoch);
719        if hit {
720            self.cur.counters.fast_ops += 1;
721            self.whole.fast_ops += 1;
722        } else {
723            self.cur.counters.slow_ops += 1;
724            self.whole.slow_ops += 1;
725        }
726    }
727    /// Record an open-loop serve latency (`ns` from the intended arrival) in
728    /// `epoch`, into the current phase segment's histogram (µs buckets). Closed
729    /// loop never calls this, so the histograms stay empty and the serve-latency
730    /// keys read absent. The whole-run serve distribution is the union of the
731    /// per-phase histograms, so only the per-phase bucket is recorded here.
732    fn record_serve_lat(&mut self, epoch: u32, ns: u64) {
733        self.roll_to(epoch);
734        let us = (ns / 1000).min(u32::MAX as u64) as u32;
735        self.cur.serve_lat.add_lat(us);
736    }
737    /// Flush the last open bucket (stamping its segment wall + CPU windows)
738    /// and stamp the whole-run wall + CPU windows.
739    fn finalize(mut self) -> Self {
740        let now = monotonic_nanos();
741        let cpu_now = crate::workload::worker::sched::thread_cpu_time_ns();
742        let mut cur = std::mem::take(&mut self.cur);
743        cur.counters.elapsed_ns = now.saturating_sub(self.phase_start_ns);
744        cur.counters.cpu_time_ns = cpu_now.saturating_sub(self.phase_start_cpu_ns);
745        self.phases.entry(self.cur_epoch).or_default().merge(&cur);
746        self.whole.elapsed_ns = now.saturating_sub(self.thread_start_ns);
747        self.whole.cpu_time_ns = cpu_now.saturating_sub(self.thread_start_cpu_ns);
748        self
749    }
750}
751
752/// Read the current phase epoch (`0` when phases are not tracked).
753fn read_epoch(phase_epoch: Option<&AtomicU32>) -> u32 {
754    phase_epoch.map(|e| e.load(Ordering::Relaxed)).unwrap_or(0)
755}
756
757// ---------------------------------------------------------------------------
758// Engine
759// ---------------------------------------------------------------------------
760
761/// Run the taobench engine until `stop` is set. `progress` counts completed ops
762/// (the live work-unit counter); `phase_epoch` buckets per-phase stats.
763pub(crate) fn run(
764    config: &TaobenchConfig,
765    stop: &AtomicBool,
766    progress: &AtomicU64,
767    phase_epoch: Option<&AtomicU32>,
768) -> TaobenchOutcome {
769    let allowed_cpus = resolve_allowed_cpus();
770    let n_clients = config.resolve_client_threads(allowed_cpus);
771    let n_slow = config.resolve_slow_threads(n_clients);
772    // Open-loop per-client inter-arrival = 1s / (arrival_rate / n_clients) =
773    // 1e9 * n_clients / arrival_rate. `0` arrival_rate ⇒ `0` interval ⇒ closed
774    // loop (issue as fast as completions allow — the legacy behavior). `.max(1)`
775    // keeps a non-zero rate strictly open-loop even at an absurd rate (so the
776    // `interval_ns != 0` mode gate in client_loop never misfires to closed loop).
777    // saturating_mul: an absurd client count (physically unreachable — it would
778    // OOM on thread spawn first) cannot wrap the numerator to a tiny interval, the
779    // same overflow-safe discipline as `TaobenchStats::merge`.
780    let interval_ns: u64 = if config.arrival_rate == 0 {
781        0
782    } else {
783        (1_000_000_000u64.saturating_mul(n_clients as u64) / config.arrival_rate as u64).max(1)
784    };
785    // Resolve the slow-path service-time model once (None = fixed legacy latency;
786    // Some = a per-fetch Pareto draw). Cheap, but kept out of the dispatcher hot
787    // path.
788    let service_pareto = config.resolve_service_pareto();
789
790    // Size the resident object count and the key range. Mean value size pins the
791    // object count for a byte budget; key_range = capacity / target_hit makes a
792    // uniform key stream hit the resident set at ≈ target_hit at equilibrium.
793    let mean_value = mean_value_size();
794    let capacity_bytes = config.cache_capacity_mib.max(1) * 1024 * 1024;
795    let total_objects = (capacity_bytes / mean_value).max(SHARDS);
796    let key_range = ((total_objects as f64) / config.target_hit_fraction()).ceil() as u64;
797
798    let cache = Cache::new(total_objects);
799    // Warm the cache to capacity so the hit ratio starts at the target rather
800    // than climbing from empty (the real warms before measuring). Keys
801    // [0, total_objects) are the initial resident set; the client key stream over
802    // [0, key_range) then hits them at ≈ target_hit and the eviction<->refill
803    // equilibrium holds it there.
804    {
805        let mut warm = Rng::new(0xC0FFEE);
806        for k in 0..total_objects as u64 {
807            cache.fill(k, sample_value_size(&mut warm));
808        }
809    }
810
811    let slow_q = SlowQueue::new();
812    let slots: Vec<Slot> = (0..n_clients).map(|_| Slot::new()).collect();
813    let disp_stop = AtomicBool::new(false);
814
815    let client_accums: Vec<ThreadAccum> = std::thread::scope(|s| {
816        // Dispatchers first so a client miss is served immediately.
817        let dispatchers: Vec<_> = (0..n_slow)
818            .map(|i| {
819                let cache = &cache;
820                let slow_q = &slow_q;
821                let slots = &slots;
822                let disp_stop = &disp_stop;
823                s.spawn(move || {
824                    dispatcher_loop(i, config, service_pareto, cache, slow_q, slots, disp_stop)
825                })
826            })
827            .collect();
828
829        let clients: Vec<_> = (0..n_clients)
830            .map(|i| {
831                let cache = &cache;
832                let slow_q = &slow_q;
833                let slot = &slots[i];
834                s.spawn(move || {
835                    client_loop(
836                        i,
837                        stop,
838                        progress,
839                        phase_epoch,
840                        cache,
841                        slow_q,
842                        slot,
843                        key_range,
844                        interval_ns,
845                    )
846                })
847            })
848            .collect();
849
850        // Join the clients first: once every client has returned (each only
851        // exits at the top of its loop, after its outstanding request — if any —
852        // has been served), NO further misses can be enqueued. Only then signal
853        // the dispatchers, so a dispatcher can never exit while a client is still
854        // blocked on an unserved miss.
855        let accums: Vec<ThreadAccum> = clients
856            .into_iter()
857            .map(|c| c.join().expect("taobench client panicked"))
858            .collect();
859        disp_stop.store(true, Ordering::Release);
860        slow_q.cv.notify_all();
861        for d in dispatchers {
862            d.join().expect("taobench dispatcher panicked");
863        }
864        accums
865    });
866
867    // Reduce: merge per-epoch and whole-run across clients. Each ThreadAccum
868    // stamped its per-phase segment walls + its whole-run window in finalize().
869    let mut all_phases: BTreeMap<u32, TaobenchPhaseStats> = BTreeMap::new();
870    let mut whole = TaobenchStats::default();
871    for accum in client_accums {
872        for (e, s) in accum.phases {
873            all_phases.entry(e).or_default().merge(&s);
874        }
875        whole.merge(&accum.whole);
876    }
877
878    TaobenchOutcome {
879        whole_run: whole,
880        phases: all_phases.into_iter().collect(),
881    }
882}
883
884/// One client: pick a key, look it up, serve a hit inline (touch + fast op) or
885/// hand a miss to a dispatcher and block until filled (slow op). `interval_ns ==
886/// 0` is CLOSED loop (issue the next request as soon as the prior completes — the
887/// legacy behavior). `interval_ns > 0` is OPEN loop: requests are due on a fixed
888/// `interval_ns` cadence independent of completion (the first one interval after
889/// the loop start `t0`), and serve latency is measured from the INTENDED arrival
890/// (coordinated-omission) so a backlog from slow service is folded into the late
891/// requests' latency instead of omitted.
892#[allow(clippy::too_many_arguments)]
893fn client_loop(
894    id: usize,
895    stop: &AtomicBool,
896    progress: &AtomicU64,
897    phase_epoch: Option<&AtomicU32>,
898    cache: &Cache,
899    slow_q: &SlowQueue,
900    slot: &Slot,
901    key_range: u64,
902    interval_ns: u64,
903) -> ThreadAccum {
904    let mut rng = Rng::new(0x5EED_0000 ^ (id as u64).wrapping_mul(0x9E37_79B1));
905    let mut accum = ThreadAccum::new(read_epoch(phase_epoch));
906    let open_loop = interval_ns != 0;
907    // Open-loop schedule cursor, advanced one `interval` before each request: the
908    // first request is due at `t0 + interval`, then every `interval` thereafter.
909    let mut intended = monotonic_nanos();
910
911    'client: while !stop.load(Ordering::Acquire) {
912        if open_loop {
913            // Pace to the next scheduled arrival; if behind (overload), the wait
914            // collapses to zero and the gap is captured as serve latency below.
915            // Each sleep is capped at STOP_POLL_QUANTUM_NS so a shutdown set during
916            // a long inter-arrival gap is observed within the quantum (not after a
917            // full interval) — exiting promptly via `break 'client` to `finalize`,
918            // bounding client-join teardown regardless of `arrival_rate`.
919            intended = intended.saturating_add(interval_ns);
920            loop {
921                if stop.load(Ordering::Acquire) {
922                    break 'client;
923                }
924                let now = monotonic_nanos();
925                if now >= intended {
926                    break;
927                }
928                let nap = (intended - now).min(STOP_POLL_QUANTUM_NS);
929                std::thread::sleep(std::time::Duration::from_nanos(nap));
930            }
931        }
932        let epoch_cmd = read_epoch(phase_epoch);
933        let key = rng.below(key_range);
934        let hit = cache.get_touch(key);
935        accum.record_cmd(epoch_cmd, hit);
936
937        if hit {
938            // Hit completes inline; response epoch == request epoch.
939            accum.record_complete(epoch_cmd, true);
940            if open_loop {
941                let lat = monotonic_nanos().saturating_sub(intended);
942                accum.record_serve_lat(epoch_cmd, lat);
943            }
944        } else {
945            // Miss: hand to a dispatcher and block until it fills + wakes us.
946            // The dispatcher always serves us before `disp_stop` is set (which
947            // happens only after every client has joined), so this never blocks
948            // past shutdown.
949            slow_q.push(SlowReq { key, client: id });
950            slot.wait_filled();
951            let resp_epoch = read_epoch(phase_epoch);
952            accum.record_complete(resp_epoch, false);
953            if open_loop {
954                // Completion timestamp taken by the CLIENT after wait_filled, so
955                // serve latency covers the full slow_q queueing + dispatcher
956                // service + wakeup, all measured from the intended arrival.
957                let lat = monotonic_nanos().saturating_sub(intended);
958                accum.record_serve_lat(resp_epoch, lat);
959            }
960        }
961        progress.fetch_add(1, Ordering::Relaxed);
962    }
963    accum.finalize()
964}
965
966/// One slow dispatcher: pull a miss, perform the simulated backing-store fetch,
967/// fill the cache, and wake the waiting client. Drains any queued misses before
968/// exiting on `disp_stop`. `service_pareto` is the resolved heavy-tail model
969/// (`None` = fixed `slow_path_sleep_us`); the per-fetch draw is deterministic from
970/// this dispatcher's seeded `rng` (the same one that picks the fill value size).
971fn dispatcher_loop(
972    id: usize,
973    config: &TaobenchConfig,
974    service_pareto: Option<(f64, f64)>,
975    cache: &Cache,
976    slow_q: &SlowQueue,
977    slots: &[Slot],
978    disp_stop: &AtomicBool,
979) {
980    let mut rng = Rng::new(0xD15A_0000 ^ (id as u64).wrapping_mul(0x9E37_79B1));
981    loop {
982        let req = {
983            let mut q = slow_q.q.lock().expect("slow queue poisoned");
984            loop {
985                if let Some(r) = q.pop_front() {
986                    break Some(r);
987                }
988                if disp_stop.load(Ordering::Acquire) {
989                    break None;
990                }
991                q = slow_q.cv.wait(q).expect("slow queue poisoned");
992            }
993        };
994        let Some(req) = req else { break };
995
996        // Heavy-tailed Pareto draw when configured, else the fixed latency. The
997        // draw precedes the value-size draw, so both come off this dispatcher's
998        // deterministic seeded stream.
999        let usec = match service_pareto {
1000            Some((scale, inv_neg_alpha)) => sample_service_us(&mut rng, scale, inv_neg_alpha),
1001            None => config.slow_path_sleep_us,
1002        };
1003        backing_store_fetch(usec);
1004        cache.fill(req.key, sample_value_size(&mut rng));
1005        slots[req.client].signal();
1006    }
1007}
1008
1009// ---------------------------------------------------------------------------
1010// Topology + size helpers
1011// ---------------------------------------------------------------------------
1012
1013/// The mean of the value-size distribution (for sizing the object count).
1014fn mean_value_size() -> usize {
1015    let total_w: u64 = VALUE_WEIGHTS.iter().map(|&w| w as u64).sum();
1016    let weighted: u64 = VALUE_SIZES
1017        .iter()
1018        .zip(VALUE_WEIGHTS.iter())
1019        .map(|(&s, &w)| s as u64 * w as u64)
1020        .sum();
1021    (weighted / total_w).max(1) as usize
1022}
1023
1024/// Resolve the number of CPUs the worker is allowed to run on (its affinity
1025/// mask), falling back to the online CPU count. Mirrors the schbench engine's
1026/// cpuset-scoped resolution.
1027fn resolve_allowed_cpus() -> usize {
1028    // SAFETY: `sched_getaffinity` writes the calling thread's CPU mask into the
1029    // provided cpu_set; reads nothing else.
1030    unsafe {
1031        let mut set: libc::cpu_set_t = core::mem::zeroed();
1032        if libc::sched_getaffinity(0, core::mem::size_of::<libc::cpu_set_t>(), &mut set) == 0 {
1033            let n = libc::CPU_COUNT(&set);
1034            if n > 0 {
1035                return n as usize;
1036            }
1037        }
1038    }
1039    let n = unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) };
1040    if n > 0 { n as usize } else { 1 }
1041}
1042
1043// ---------------------------------------------------------------------------
1044// Host-side standalone runner (validation driver entry)
1045// ---------------------------------------------------------------------------
1046
1047/// Host-side standalone run of the taobench engine for `run_secs`, returning a
1048/// summary report — the analog of schbench's `run_standalone`, backing the
1049/// `ktstr-taobench-validate` driver for the side-by-side comparison against the
1050/// reference taobench. NOT used in-VM (the scenario engine drives `run` there).
1051///
1052/// The engine warms the cache synchronously before the client threads start; the
1053/// report's `elapsed_secs` is the engine-measured CLIENT window (post-warmup), so
1054/// the qps figures are steady-state, not diluted by warmup.
1055pub fn run_standalone(config: &TaobenchConfig, run_secs: u64) -> TaobenchStandaloneReport {
1056    let stop = AtomicBool::new(false);
1057    let progress = AtomicU64::new(0);
1058    let allowed_cpus = resolve_allowed_cpus();
1059    let nr_client_threads = config.resolve_client_threads(allowed_cpus);
1060    let nr_slow_threads = config.resolve_slow_threads(nr_client_threads);
1061    let outcome = std::thread::scope(|s| {
1062        let h = s.spawn(|| run(config, &stop, &progress, None));
1063        // The benchmark run window — a workload-defined duration (the same model
1064        // as the scenario engine's hold step), not a synchronization wait.
1065        std::thread::sleep(std::time::Duration::from_secs(run_secs));
1066        stop.store(true, Ordering::Release);
1067        h.join().expect("taobench standalone run panicked")
1068    });
1069    // Whole-run serve-latency distribution = union of the per-phase histograms
1070    // (the whole-run carrier is counters-only). Empty in closed loop, so the
1071    // report's serve percentiles read absent there.
1072    let mut serve = PlatStats::default();
1073    for (_epoch, p) in &outcome.phases {
1074        serve.combine(&p.serve_lat);
1075    }
1076    TaobenchStandaloneReport::from_run(
1077        &outcome.whole_run,
1078        &serve,
1079        nr_client_threads,
1080        nr_slow_threads,
1081    )
1082}
1083
1084/// Summary of a [`run_standalone`] run — the headline taobench metrics in the
1085/// shape the reference taobench server reports (`fast_qps` / `hit_rate` /
1086/// `slow_qps`) plus the derived `total_qps` (= fast + slow) and `hit_ratio`
1087/// (= fast / total). Under open-loop arrival (`arrival_rate > 0`) it also carries
1088/// the coordinated-omission serve-latency percentiles; these are `None` in closed
1089/// loop (no intended-arrival schedule, so no serve latency is measured).
1090///
1091/// DENOMINATION: the qps fields here are WALL-clock (`ops / elapsed`) on
1092/// purpose — this is the standalone/validate surface whose output shape must
1093/// stay byte-identical to the reference taobench report. The ktstr sidecar's
1094/// `taobench_*_ops_per_cpu_sec` metrics are CPU-second denominated instead
1095/// (see [`TaobenchStats::cpu_time_ns`]).
1096#[derive(Debug, Clone, Copy, PartialEq)]
1097pub struct TaobenchStandaloneReport {
1098    /// (fast + slow) ops per second over the measured window.
1099    pub total_qps: f64,
1100    /// Hits served per second.
1101    pub fast_qps: f64,
1102    /// Misses served (slow path) per second.
1103    pub slow_qps: f64,
1104    /// Response-time hit ratio: fast / (fast + slow).
1105    pub hit_ratio: f64,
1106    /// Command-time hit rate: 1 - get_misses / get_cmds.
1107    pub hit_rate: f64,
1108    /// Completed ops (fast + slow).
1109    pub total_ops: u64,
1110    /// Hits served.
1111    pub fast_ops: u64,
1112    /// Misses served via the slow path.
1113    pub slow_ops: u64,
1114    /// The engine-measured client window, seconds.
1115    pub elapsed_secs: f64,
1116    /// Resolved client thread count.
1117    pub nr_client_threads: usize,
1118    /// Resolved slow dispatcher count.
1119    pub nr_slow_threads: usize,
1120    /// Open-loop coordinated-omission serve-latency percentiles (µs), measured
1121    /// from intended arrival. `None` in closed loop / when no serve samples were
1122    /// recorded (matching the metric-key ABSENT discipline).
1123    pub serve_p50_us: Option<u32>,
1124    /// Open-loop serve-latency p99 (µs) — the headline coordinated-omission tail.
1125    pub serve_p99_us: Option<u32>,
1126    /// Open-loop serve-latency p99.9 (µs).
1127    pub serve_p999_us: Option<u32>,
1128    /// Open-loop serve-latency minimum sample (µs).
1129    pub serve_min_us: Option<u32>,
1130    /// Open-loop serve-latency maximum sample (µs).
1131    pub serve_max_us: Option<u32>,
1132}
1133
1134impl TaobenchStandaloneReport {
1135    fn from_run(
1136        w: &TaobenchStats,
1137        serve: &PlatStats,
1138        nr_client_threads: usize,
1139        nr_slow_threads: usize,
1140    ) -> Self {
1141        use crate::workload::schbench::plat::Pct;
1142        let secs = (w.elapsed_ns as f64 / 1e9).max(f64::MIN_POSITIVE);
1143        let total = w.total_ops();
1144        // Serve-latency percentiles are present iff open-loop arrival recorded
1145        // samples; closed loop leaves the histogram empty, so all fields read
1146        // None (the metric-key ABSENT discipline).
1147        let serve_q = (serve.sample_count() > 0).then(|| serve.percentiles());
1148        TaobenchStandaloneReport {
1149            total_qps: total as f64 / secs,
1150            fast_qps: w.fast_ops as f64 / secs,
1151            slow_qps: w.slow_ops as f64 / secs,
1152            hit_ratio: if total > 0 {
1153                w.fast_ops as f64 / total as f64
1154            } else {
1155                0.0
1156            },
1157            hit_rate: if w.get_cmds > 0 {
1158                1.0 - (w.get_misses as f64 / w.get_cmds as f64)
1159            } else {
1160                0.0
1161            },
1162            total_ops: total,
1163            fast_ops: w.fast_ops,
1164            slow_ops: w.slow_ops,
1165            elapsed_secs: secs,
1166            nr_client_threads,
1167            nr_slow_threads,
1168            serve_p50_us: serve_q.as_ref().map(|q| q.value_at(Pct::P50)),
1169            serve_p99_us: serve_q.as_ref().map(|q| q.value_at(Pct::P99)),
1170            serve_p999_us: serve_q.as_ref().map(|q| q.value_at(Pct::P999)),
1171            serve_min_us: serve_q.as_ref().map(|q| q.min),
1172            serve_max_us: serve_q.as_ref().map(|q| q.max),
1173        }
1174    }
1175}
1176
1177#[cfg(test)]
1178mod tests {
1179    use super::*;
1180
1181    #[test]
1182    fn value_size_distribution_mean_is_small_object_heavy() {
1183        // The representative distribution is small-object-heavy (mean well under
1184        // 1 KiB) with a tail to 64 KiB.
1185        let m = mean_value_size();
1186        assert!(
1187            (200..=500).contains(&m),
1188            "mean value size {m} B is small-object-heavy"
1189        );
1190        assert_eq!(*VALUE_SIZES.last().unwrap(), 65536, "tail reaches 64 KiB");
1191    }
1192
1193    #[test]
1194    fn taobench_stats_merge_and_total_ops_saturate_on_overflow() {
1195        // Pooling guest-runtime op counters must saturate, not wrap: a corrupt
1196        // u64::MAX component must never produce a silently-wrong (wrapped-small)
1197        // qps/hit Rate. Saturating is exact for all in-range data.
1198        let mut a = TaobenchStats {
1199            get_cmds: u64::MAX,
1200            get_misses: u64::MAX,
1201            fast_ops: u64::MAX,
1202            slow_ops: 1,
1203            elapsed_ns: 1000,
1204            cpu_time_ns: u64::MAX,
1205        };
1206        let b = TaobenchStats {
1207            get_cmds: 5,
1208            get_misses: 5,
1209            fast_ops: 5,
1210            slow_ops: 5,
1211            elapsed_ns: 9000,
1212            cpu_time_ns: 5,
1213        };
1214        a.merge(&b);
1215        assert_eq!(a.get_cmds, u64::MAX, "saturates, not wraps to 4");
1216        assert_eq!(a.get_misses, u64::MAX);
1217        assert_eq!(a.fast_ops, u64::MAX);
1218        assert_eq!(a.slow_ops, 6);
1219        assert_eq!(a.elapsed_ns, 9000, "wall window is MAX, not summed");
1220        // total_ops = fast_ops + slow_ops saturates (u64::MAX + 6 -> u64::MAX).
1221        assert_eq!(a.total_ops(), u64::MAX, "total_ops saturates");
1222    }
1223
1224    #[test]
1225    fn engine_serves_ops_and_hit_ratio_settles_near_target_not_one() {
1226        // A short run on a small cache: the hit ratio must settle near the target
1227        // (the eviction<->refill equilibrium), NOT drift to 1.0 (which a
1228        // self-healing cache with no eviction would do).
1229        let cfg = TaobenchConfig::default()
1230            .client_threads(4)
1231            .slow_threads(2)
1232            .cache_capacity_mib(8)
1233            .target_hit_pct(90)
1234            .slow_path_sleep_us(10);
1235        let stop = AtomicBool::new(false);
1236        let progress = AtomicU64::new(0);
1237
1238        std::thread::scope(|s| {
1239            let h = s.spawn(|| run(&cfg, &stop, &progress, None));
1240            // Spin until the engine has served enough ops to reach equilibrium.
1241            while progress.load(Ordering::Relaxed) < 100_000 {
1242                std::hint::spin_loop();
1243            }
1244            stop.store(true, Ordering::Release);
1245            let out = h.join().expect("engine panicked");
1246
1247            let total = out.whole_run.total_ops();
1248            assert!(total > 0, "engine served ops");
1249            assert!(out.whole_run.elapsed_ns > 0, "the run window was measured");
1250
1251            let hit = out.whole_run.fast_ops as f64 / total as f64;
1252            assert!(
1253                (0.80..=0.97).contains(&hit),
1254                "hit ratio {hit} settles near target 0.90 (eviction equilibrium), not 1.0"
1255            );
1256            assert!(
1257                out.whole_run.slow_ops > 0,
1258                "the slow/miss path is exercised"
1259            );
1260        });
1261    }
1262
1263    #[test]
1264    fn taobench_config_serde_roundtrips() {
1265        // The new serialized type roundtrips unchanged.
1266        // Every field set to a non-default value to exercise the full serde surface.
1267        let cfg = TaobenchConfig::default()
1268            .client_threads(8)
1269            .slow_threads(3)
1270            .cache_capacity_mib(128)
1271            .target_hit_pct(85)
1272            .slow_path_sleep_us(250)
1273            .slow_path_p99_us(2500)
1274            .arrival_rate(50_000);
1275        let json = serde_json::to_string(&cfg).expect("TaobenchConfig must serialize");
1276        let back: TaobenchConfig =
1277            serde_json::from_str(&json).expect("TaobenchConfig must deserialize");
1278        assert_eq!(cfg, back, "config roundtrips unchanged");
1279    }
1280
1281    #[test]
1282    fn worktype_taobench_registration_and_serde() {
1283        use crate::workload::WorkType;
1284        let wt = WorkType::taobench(
1285            TaobenchConfig::default()
1286                .client_threads(4)
1287                .target_hit_pct(95),
1288        );
1289        assert_eq!(wt.name(), "Taobench");
1290        // from_name yields the default-config variant.
1291        assert_eq!(
1292            WorkType::from_name("Taobench"),
1293            Some(WorkType::Taobench {
1294                config: TaobenchConfig::default()
1295            })
1296        );
1297        // The variant serde-roundtrips, carrying its config.
1298        let json = serde_json::to_string(&wt).expect("WorkType::Taobench must serialize");
1299        let back: WorkType =
1300            serde_json::from_str(&json).expect("WorkType::Taobench must deserialize");
1301        assert_eq!(wt, back);
1302    }
1303
1304    #[test]
1305    fn taobench_config_reachable_via_prelude() {
1306        // Regression-pin the prelude placement: test authors construct the config
1307        // via `use ktstr::prelude::*`. Dropping TaobenchConfig from the prelude
1308        // would fail this compile. Also exercises the Eq derive.
1309        let cfg: crate::prelude::TaobenchConfig = crate::prelude::TaobenchConfig::default();
1310        assert_eq!(cfg, TaobenchConfig::default());
1311    }
1312
1313    #[test]
1314    fn taobench_stats_serde_roundtrips() {
1315        // TaobenchStats is a pub serialized type that rides real wires — a field
1316        // on WorkerReport (the worker→host postcard payload) and CgroupStats (the
1317        // sidecar JSON) — so a field-rename or serde-attr drift is a silent
1318        // data-loss risk this pins. Every field distinct +
1319        // non-zero to exercise the full serde surface.
1320        let s = TaobenchStats {
1321            get_cmds: 1000,
1322            get_misses: 150,
1323            fast_ops: 850,
1324            slow_ops: 150,
1325            elapsed_ns: 9_000_000_000,
1326            cpu_time_ns: 4_200_000_000,
1327        };
1328        let json = serde_json::to_string(&s).expect("TaobenchStats must serialize");
1329        let back: TaobenchStats =
1330            serde_json::from_str(&json).expect("TaobenchStats must deserialize");
1331        assert_eq!(s, back, "TaobenchStats roundtrips unchanged");
1332    }
1333
1334    #[test]
1335    fn taobench_stats_reachable_via_prelude() {
1336        // Regression-pin the prelude placement: TaobenchStats is read off the
1337        // preluded WorkerReport/CgroupStats `taobench_whole` fields, so it must be
1338        // nameable via `use ktstr::prelude::*`. Dropping it from the prelude would
1339        // fail this compile. Also exercises Default + Eq.
1340        let s: crate::prelude::TaobenchStats = crate::prelude::TaobenchStats::default();
1341        assert_eq!(s, TaobenchStats::default());
1342    }
1343
1344    #[test]
1345    fn taobench_config_arrival_rate_default_and_setter() {
1346        // Default is closed loop (no open-loop schedule); the setter sets the
1347        // aggregate ops/sec field.
1348        assert_eq!(
1349            TaobenchConfig::default().arrival_rate,
1350            0,
1351            "default arrival_rate is 0 (closed loop)"
1352        );
1353        let cfg = TaobenchConfig::default().arrival_rate(100_000);
1354        assert_eq!(cfg.arrival_rate, 100_000, "setter sets arrival_rate");
1355    }
1356
1357    #[test]
1358    fn taobench_phase_stats_merge_sums_counters_and_unions_serve_lat() {
1359        // The carrier merge pools counters (Σ ops, MAX wall — TaobenchStats::merge)
1360        // AND unions the serve histograms (PlatStats::combine) — the per-cgroup /
1361        // cross-thread fold.
1362        let mut a = TaobenchPhaseStats {
1363            counters: TaobenchStats {
1364                get_cmds: 10,
1365                get_misses: 2,
1366                fast_ops: 8,
1367                slow_ops: 2,
1368                elapsed_ns: 1000,
1369                cpu_time_ns: 700,
1370            },
1371            ..Default::default()
1372        };
1373        a.serve_lat.add_lat(5);
1374        a.serve_lat.add_lat(50);
1375        let mut b = TaobenchPhaseStats {
1376            counters: TaobenchStats {
1377                get_cmds: 6,
1378                get_misses: 1,
1379                fast_ops: 5,
1380                slow_ops: 1,
1381                elapsed_ns: 4000,
1382                cpu_time_ns: 300,
1383            },
1384            ..Default::default()
1385        };
1386        b.serve_lat.add_lat(500);
1387        a.merge(&b);
1388        assert_eq!(a.counters.get_cmds, 16, "Σ get_cmds");
1389        assert_eq!(a.counters.fast_ops, 13, "Σ fast_ops");
1390        assert_eq!(a.counters.slow_ops, 3, "Σ slow_ops");
1391        assert_eq!(a.counters.elapsed_ns, 4000, "wall is MAX, not summed");
1392        assert_eq!(
1393            a.counters.cpu_time_ns, 1000,
1394            "cpu is SUM (disjoint per-thread)"
1395        );
1396        assert_eq!(
1397            a.serve_lat.sample_count(),
1398            3,
1399            "serve histograms union (2 + 1 samples)"
1400        );
1401    }
1402
1403    #[test]
1404    fn taobench_phase_stats_serde_roundtrips() {
1405        // TaobenchPhaseStats rides the WorkerReport postcard + CgroupStats sidecar
1406        // JSON wires; both the counters and the serve histogram must roundtrip.
1407        let mut s = TaobenchPhaseStats {
1408            counters: TaobenchStats {
1409                get_cmds: 1000,
1410                get_misses: 150,
1411                fast_ops: 850,
1412                slow_ops: 150,
1413                elapsed_ns: 9_000_000_000,
1414                cpu_time_ns: 4_200_000_000,
1415            },
1416            ..Default::default()
1417        };
1418        s.serve_lat.add_lat(12);
1419        s.serve_lat.add_lat(340);
1420        s.serve_lat.add_lat(99_999);
1421        let json = serde_json::to_string(&s).expect("TaobenchPhaseStats must serialize");
1422        let back: TaobenchPhaseStats =
1423            serde_json::from_str(&json).expect("TaobenchPhaseStats must deserialize");
1424        assert_eq!(
1425            s, back,
1426            "TaobenchPhaseStats roundtrips unchanged (counters + serve_lat)"
1427        );
1428    }
1429
1430    #[test]
1431    fn open_loop_stamps_serve_latency_on_every_completion() {
1432        // Open loop (non-zero arrival_rate) measures coordinated-omission serve
1433        // latency: every completion — a hit (served inline) AND a miss (served via
1434        // the slow path) — records exactly one sample from its intended arrival, so
1435        // the histogram count equals total_ops. An absurdly high rate keeps the
1436        // pacing in permanent overload (the per-arrival sleep collapses to zero), so
1437        // the test runs at full speed while still exercising stamping on both arms.
1438        let cfg = TaobenchConfig::default()
1439            .client_threads(4)
1440            .slow_threads(2)
1441            .cache_capacity_mib(8)
1442            .target_hit_pct(90)
1443            .slow_path_sleep_us(10)
1444            .arrival_rate(1_000_000_000);
1445        let stop = AtomicBool::new(false);
1446        let progress = AtomicU64::new(0);
1447        let out = std::thread::scope(|s| {
1448            let h = s.spawn(|| run(&cfg, &stop, &progress, None));
1449            while progress.load(Ordering::Relaxed) < 50_000 {
1450                std::hint::spin_loop();
1451            }
1452            stop.store(true, Ordering::Release);
1453            h.join().expect("engine panicked")
1454        });
1455        let total = out.whole_run.total_ops();
1456        assert!(total > 0, "engine served ops");
1457        assert!(out.whole_run.slow_ops > 0, "the miss arm is exercised");
1458        // The whole-run serve distribution is the union of the per-phase
1459        // histograms (the whole-run carrier is counters-only). Its count equals
1460        // total_ops: every completion (hit inline + miss after wait_filled)
1461        // records exactly one serve-latency sample.
1462        let mut serve = PlatStats::default();
1463        for (_epoch, p) in &out.phases {
1464            serve.combine(&p.serve_lat);
1465        }
1466        assert_eq!(
1467            serve.sample_count(),
1468            total,
1469            "every completion (hit + miss) records one serve-latency sample"
1470        );
1471    }
1472
1473    #[test]
1474    fn closed_loop_records_no_serve_latency() {
1475        // arrival_rate == 0 is closed loop: there is no intended-arrival schedule,
1476        // so serve latency is undefined and the histogram stays EMPTY (the
1477        // taobench_serve_*_us keys then read absent, distinct from a measured 0).
1478        // The heavy tail is ALSO enabled (slow_path_p99_us > slow_path_sleep_us) to
1479        // pin that enabling the tail never leaks a serve sample without an open-loop
1480        // schedule — the tail only affects fetch latency, not whether serve latency
1481        // is measured.
1482        let cfg = TaobenchConfig::default()
1483            .client_threads(4)
1484            .slow_threads(2)
1485            .cache_capacity_mib(8)
1486            .target_hit_pct(90)
1487            .slow_path_sleep_us(10)
1488            .slow_path_p99_us(1000); // tail enabled; arrival_rate defaults to 0 (closed loop)
1489        let stop = AtomicBool::new(false);
1490        let progress = AtomicU64::new(0);
1491        let out = std::thread::scope(|s| {
1492            let h = s.spawn(|| run(&cfg, &stop, &progress, None));
1493            while progress.load(Ordering::Relaxed) < 20_000 {
1494                std::hint::spin_loop();
1495            }
1496            stop.store(true, Ordering::Release);
1497            h.join().expect("engine panicked")
1498        });
1499        assert!(out.whole_run.total_ops() > 0, "ops completed");
1500        for (epoch, p) in &out.phases {
1501            assert_eq!(
1502                p.serve_lat.sample_count(),
1503                0,
1504                "closed loop records no per-phase serve latency (epoch {epoch})"
1505            );
1506        }
1507    }
1508
1509    #[test]
1510    fn open_loop_per_phase_carrier_holds_serve_latency() {
1511        // The per-phase carrier (PhaseSlice.taobench) carries the serve histogram
1512        // (the whole-run carrier is counters-only). With a single static epoch
1513        // every completion's sample lands in that epoch's bucket, so the per-phase
1514        // serve count equals the whole-run completion count (total_ops).
1515        let cfg = TaobenchConfig::default()
1516            .client_threads(4)
1517            .slow_threads(2)
1518            .cache_capacity_mib(8)
1519            .target_hit_pct(90)
1520            .slow_path_sleep_us(10)
1521            .arrival_rate(1_000_000_000);
1522        let stop = AtomicBool::new(false);
1523        let progress = AtomicU64::new(0);
1524        let epoch = AtomicU32::new(7);
1525        let out = std::thread::scope(|s| {
1526            let h = s.spawn(|| run(&cfg, &stop, &progress, Some(&epoch)));
1527            while progress.load(Ordering::Relaxed) < 30_000 {
1528                std::hint::spin_loop();
1529            }
1530            stop.store(true, Ordering::Release);
1531            h.join().expect("engine panicked")
1532        });
1533        let phase7 = out
1534            .phases
1535            .iter()
1536            .find(|(e, _)| *e == 7)
1537            .map(|(_, p)| p)
1538            .expect("epoch-7 phase carrier present");
1539        assert!(
1540            phase7.serve_lat.sample_count() > 0,
1541            "per-phase serve latency recorded"
1542        );
1543        assert_eq!(
1544            phase7.serve_lat.sample_count(),
1545            out.whole_run.total_ops(),
1546            "single-epoch run: every completion stamps one per-phase serve sample"
1547        );
1548    }
1549
1550    #[test]
1551    fn open_loop_pacing_sleep_is_bounded_for_prompt_shutdown() {
1552        // A degenerate low arrival_rate makes the inter-arrival interval ~1s, but a
1553        // client waiting for its next scheduled arrival must observe `stop` within
1554        // STOP_POLL_QUANTUM_NS (50 ms) and exit, so the join completes well under
1555        // one interval. An unbounded pacing sleep would block the join for the full
1556        // ~1s. The 500 ms bound is 10x the quantum (robust to scheduling jitter)
1557        // and half the interval (so it cleanly distinguishes bounded from
1558        // unbounded).
1559        let cfg = TaobenchConfig::default()
1560            .client_threads(1)
1561            .slow_threads(1)
1562            .cache_capacity_mib(4)
1563            .target_hit_pct(90)
1564            .slow_path_sleep_us(0)
1565            .arrival_rate(1); // interval_ns = 1e9 * 1 / 1 = 1s
1566        let stop = AtomicBool::new(false);
1567        let progress = AtomicU64::new(0);
1568        let start = std::time::Instant::now();
1569        std::thread::scope(|s| {
1570            let h = s.spawn(|| run(&cfg, &stop, &progress, None));
1571            // Let the client reach its first (long) pacing sleep, then shut down.
1572            std::thread::sleep(std::time::Duration::from_millis(20));
1573            stop.store(true, Ordering::Release);
1574            let _ = h.join().expect("engine panicked");
1575        });
1576        let elapsed = start.elapsed();
1577        assert!(
1578            elapsed < std::time::Duration::from_millis(500),
1579            "open-loop client joined in {elapsed:?}; the pacing sleep must be \
1580             bounded (interval is ~1s — an unbounded sleep would block the join)"
1581        );
1582    }
1583
1584    #[test]
1585    fn taobench_config_slow_path_p99_default_and_setter() {
1586        // Default keeps the legacy fixed-latency path (no tail); the setter sets the
1587        // p99 tail target.
1588        assert_eq!(
1589            TaobenchConfig::default().slow_path_p99_us,
1590            0,
1591            "default slow_path_p99_us is 0 (fixed latency)"
1592        );
1593        let cfg = TaobenchConfig::default().slow_path_p99_us(5000);
1594        assert_eq!(cfg.slow_path_p99_us, 5000, "setter sets slow_path_p99_us");
1595    }
1596
1597    #[test]
1598    fn taobench_service_pareto_mapping_and_legacy_gate() {
1599        // The (p50, p99) knob maps to a Pareto whose analytic quantiles round-trip:
1600        // alpha = ln(50)/ln(p99/p50); scale = p50 * 0.5^(1/alpha).
1601        let cfg = TaobenchConfig::default()
1602            .slow_path_sleep_us(100)
1603            .slow_path_p99_us(1000);
1604        let (scale, inv_neg_alpha) = cfg.resolve_service_pareto().expect("tail enabled");
1605        let alpha = -1.0 / inv_neg_alpha;
1606        assert!(
1607            (alpha - 50.0_f64.ln() / 10.0_f64.ln()).abs() < 1e-9,
1608            "alpha = ln(50)/ln(10), got {alpha}",
1609        );
1610        // The Pareto q-quantile scale*(1-q)^(-1/alpha) round-trips the config: the
1611        // median quantile is p50, the 99th percentile is p99.
1612        let median = scale * 0.5_f64.powf(-1.0 / alpha);
1613        let p99 = scale * 0.01_f64.powf(-1.0 / alpha);
1614        assert!(
1615            (median - 100.0).abs() < 1e-6,
1616            "median quantile = p50 (100), got {median}",
1617        );
1618        assert!(
1619            (p99 - 1000.0).abs() < 1e-6,
1620            "p99 quantile = configured (1000), got {p99}",
1621        );
1622        // Legacy gate: p99 == 0, p99 < p50, p99 == p50, and p50 == 0 all resolve to
1623        // the fixed path (None).
1624        let base = TaobenchConfig::default().slow_path_sleep_us(100);
1625        assert!(
1626            base.clone()
1627                .slow_path_p99_us(0)
1628                .resolve_service_pareto()
1629                .is_none(),
1630            "p99 == 0 is fixed",
1631        );
1632        assert!(
1633            base.clone()
1634                .slow_path_p99_us(50)
1635                .resolve_service_pareto()
1636                .is_none(),
1637            "p99 < p50 is fixed",
1638        );
1639        assert!(
1640            base.clone()
1641                .slow_path_p99_us(100)
1642                .resolve_service_pareto()
1643                .is_none(),
1644            "p99 == p50 is fixed",
1645        );
1646        assert!(
1647            TaobenchConfig::default()
1648                .slow_path_sleep_us(0)
1649                .slow_path_p99_us(1000)
1650                .resolve_service_pareto()
1651                .is_none(),
1652            "p50 == 0 has no Pareto scale",
1653        );
1654    }
1655
1656    #[test]
1657    fn taobench_service_sampler_matches_configured_quantiles() {
1658        // Drawing many samples from the resolved Pareto through the actual
1659        // measurement path (PlatStats) yields a histogram whose p50/p99 land near
1660        // the configured (100, 1000) µs. Deterministic (fixed seed); the bands
1661        // absorb PlatStats log-bucketing + sampling variance.
1662        use crate::workload::schbench::plat::Pct;
1663        let cfg = TaobenchConfig::default()
1664            .slow_path_sleep_us(100)
1665            .slow_path_p99_us(1000);
1666        let (scale, inv_neg_alpha) = cfg.resolve_service_pareto().unwrap();
1667        let mut rng = Rng::new(0x5A3C_1234);
1668        let mut plat = PlatStats::default();
1669        for _ in 0..200_000 {
1670            let us = sample_service_us(&mut rng, scale, inv_neg_alpha);
1671            plat.add_lat(us.min(u32::MAX as u64) as u32);
1672        }
1673        let q = plat.percentiles();
1674        let p50 = q.value_at(Pct::P50);
1675        let p99 = q.value_at(Pct::P99);
1676        assert!(
1677            (90..=115).contains(&p50),
1678            "empirical p50 {p50} ≈ configured 100µs (tight band: rejects a sampler \
1679             bit-shift / lost-scale bug, absorbs only log-bucketing + seed noise)",
1680        );
1681        assert!(
1682            (850..=1200).contains(&p99),
1683            "empirical p99 {p99} ≈ configured 1000µs (tight band: rejects a ~±50% \
1684             sampler error, absorbs only log-bucketing + seed noise)",
1685        );
1686    }
1687
1688    #[test]
1689    fn taobench_service_sampler_clamps_pathological_tail() {
1690        // A pathological draw (huge scale, so any U gives a value far above the cap)
1691        // is clamped to MAX_SERVICE_US rather than parking a dispatcher unboundedly.
1692        let mut rng = Rng::new(7);
1693        let v = sample_service_us(&mut rng, 1e12, -0.5);
1694        assert_eq!(v, MAX_SERVICE_US, "huge draw clamps to MAX_SERVICE_US");
1695    }
1696
1697    #[test]
1698    fn taobench_service_sampler_is_deterministic_from_seed() {
1699        // Same seed -> same service-time draw sequence (the determinism the engine
1700        // relies on for reproducible runs).
1701        let cfg = TaobenchConfig::default()
1702            .slow_path_sleep_us(100)
1703            .slow_path_p99_us(1000);
1704        let (scale, inv_neg_alpha) = cfg.resolve_service_pareto().unwrap();
1705        let mut a = Rng::new(42);
1706        let mut b = Rng::new(42);
1707        for _ in 0..1000 {
1708            assert_eq!(
1709                sample_service_us(&mut a, scale, inv_neg_alpha),
1710                sample_service_us(&mut b, scale, inv_neg_alpha),
1711            );
1712        }
1713    }
1714
1715    #[test]
1716    fn rng_f64_open01_is_in_open_closed_unit_interval() {
1717        // f64_open01 must yield U in (0, 1]: 0 is excluded (U^(−1/α) would be
1718        // infinite in the Pareto inverse-CDF), 1 is included (the distribution
1719        // floor). Pin the boundary contract directly over many fixed-seed draws.
1720        let mut rng = Rng::new(0xF00D_BEEF);
1721        for _ in 0..1_000_000 {
1722            let u = rng.f64_open01();
1723            assert!(
1724                u > 0.0 && u <= 1.0,
1725                "f64_open01 yielded {u}, outside (0, 1]"
1726            );
1727        }
1728    }
1729
1730    // The heavy-tail slow-path service model — the Pareto that
1731    // `resolve_service_pareto` derives from the `slow_path_sleep_us` median +
1732    // `slow_path_p99_us` 99th-percentile knobs, sampled by `sample_service_us` —
1733    // reproduces those quantiles and a heavy tail the fixed path lacks.
1734    //
1735    // Deterministic: a fixed-seed PRNG driving the pure sampler, no threads and
1736    // no wall-clock timing, so it pins the heavy-tail LOGIC without the
1737    // host-scheduling jitter a real-timing run carries (a prior version compared
1738    // wall-clock serve MAX across host threads and flaked on loaded CI runners,
1739    // where the control's jitter spike collapsed the ratio). The end-to-end
1740    // engine path — the slow path drawing from this sampler under a live
1741    // open-loop arrival process — is covered by the coordinated in-VM
1742    // `taobench_open_loop_runs_in_vm` e2e (pinned vCPUs, no host jitter).
1743    #[test]
1744    fn heavy_tail_service_model_reproduces_configured_quantiles() {
1745        // A p99 above the median resolves to a Pareto; at/below it (or a zero
1746        // median) stays on the fixed-latency path.
1747        let heavy = TaobenchConfig::default()
1748            .slow_path_sleep_us(100)
1749            .slow_path_p99_us(2000);
1750        let (scale, inv_neg_alpha) = heavy
1751            .resolve_service_pareto()
1752            .expect("p99 2000µs above median 100µs resolves to a Pareto tail");
1753        for (p50, p99, why) in [
1754            (100u64, 0u64, "p99 == 0 keeps the fixed path"),
1755            (100, 100, "p99 == median is not a tail"),
1756            (0, 2000, "zero median has no Pareto scale"),
1757        ] {
1758            assert!(
1759                TaobenchConfig::default()
1760                    .slow_path_sleep_us(p50)
1761                    .slow_path_p99_us(p99)
1762                    .resolve_service_pareto()
1763                    .is_none(),
1764                "{why}",
1765            );
1766        }
1767
1768        // Fixed-seed draws reproduce the configured quantiles + a heavy tail.
1769        let mut rng = Rng::new(0x00C0_FFEE);
1770        let mut samples: Vec<u64> = (0..200_000)
1771            .map(|_| sample_service_us(&mut rng, scale, inv_neg_alpha))
1772            .collect();
1773        samples.sort_unstable();
1774        let q = |p: f64| samples[((samples.len() as f64 * p) as usize).min(samples.len() - 1)];
1775        let (p50, p99, max) = (q(0.50), q(0.99), *samples.last().unwrap());
1776        // Median ~100µs and p99 ~2000µs (the configured knobs), within a
1777        // generous band only a broken Pareto mapping would miss.
1778        assert!(
1779            (50..=200).contains(&p50),
1780            "empirical median {p50}µs should track the configured 100µs",
1781        );
1782        assert!(
1783            (1000..=4000).contains(&p99),
1784            "empirical p99 {p99}µs should track the configured 2000µs",
1785        );
1786        assert!(
1787            max > p99.saturating_mul(2),
1788            "max {max}µs must dwarf p99 {p99}µs — the Pareto tail the fixed path lacks",
1789        );
1790    }
1791}