ktstr/workload/taobench/run.rs
1//! The taobench engine: a bounded, sharded, evicting key-value cache served by a
2//! client population (closed-loop by default, or open-loop fixed-rate arrival),
3//! with a fast in-cache path (hit) and a slow backing-store-miss path (a
4//! worker-defined sleep on a dispatcher thread). It is
5//! entirely in-process — no sockets, no TLS, no subprocess — and re-expresses the
6//! taobench ACCESS PATTERN in ktstr primitives so its qps / hit-ratio flow
7//! through the metric API. See `validation.md` for the per-aspect real-vs-port
8//! fidelity comparison.
9//!
10//! ## Model
11//!
12//! - A `Cache` of `SHARDS` independently-locked shards, each a FIFO-evicting
13//! map bounded to a per-shard object cap. Total capacity is `cache_capacity_mib`
14//! worth of objects; the key range is sized `capacity / target_hit` so a
15//! uniform-random key stream hits the resident set with probability
16//! ≈ `target_hit` at equilibrium. Eviction is the load-bearing mechanism: with
17//! no eviction a self-healing cache drifts to a 1.0 hit ratio; a bounded cache
18//! whose key range exceeds its capacity holds a steady-state miss stream.
19//! - `client_threads` CLIENT threads pick a key, look it up, and on a HIT touch
20//! the stored value bytes (the cache read-bandwidth cost) and count a fast op;
21//! on a MISS hand the key to a slow dispatcher and block until it is filled,
22//! then count a slow op.
23//! - Arrival: `arrival_rate == 0` (default) is CLOSED loop — each client issues
24//! its next request as soon as the prior completes. `arrival_rate > 0` is OPEN
25//! loop — each client has a fixed intended-arrival SCHEDULE
26//! (`arrival_rate / client_threads` per client) independent of completion, and
27//! serve latency is measured from that intended time. A client still holds at
28//! most one outstanding request (it blocks on a miss before issuing the next),
29//! so a slow completion delays the next issue; that backlog is folded into the
30//! late requests' serve latency (coordinated-omission correction) instead of
31//! being omitted. The serve-latency histogram is per-phase data (empty in
32//! closed loop).
33//! - `slow_threads` DISPATCHER threads serve misses: sleep the simulated
34//! backing-store fetch (a worker-defined cost, the same model schbench's
35//! think-sleep uses, not a synchronization wait), insert a freshly sized+touched
36//! value, and wake the waiting client. The fetch latency is a fixed
37//! `slow_path_sleep_us` by default, or — when `slow_path_p99_us` is set above it
38//! — a per-fetch heavy-tailed Pareto draw with median `slow_path_sleep_us` and
39//! 99th percentile `slow_path_p99_us` (most fetches near the median, a rare slow
40//! tail), so the open-loop serve-latency tail reflects realistic backing-store
41//! variance.
42//!
43//! ## Counters (request-time vs response-time)
44//!
45//! `get_cmds` / `get_misses` are counted at LOOKUP (request) time; `fast_ops` /
46//! `slow_ops` at COMPLETION (response) time. In a closed loop they are equal once
47//! every in-flight request drains, but a request that straddles a phase boundary
48//! lands its lookup in one phase and its completion in the next, so a per-phase
49//! command-time hit_rate (`1 - get_misses/get_cmds`) differs slightly from the
50//! response-time hit_ratio (`fast_ops/(fast_ops+slow_ops)`) — the same skew the
51//! real reports between its interval hit_rate and its final hit_ratio.
52
53use crate::workload::schbench::plat::PlatStats;
54use core::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
55use std::collections::HashMap;
56use std::collections::{BTreeMap, VecDeque};
57use std::sync::{Condvar, Mutex};
58
59/// Number of independently-locked cache shards (power of two). Sized to keep
60/// per-shard lock contention low for typical guest thread counts while bounding
61/// the lock-array footprint; the cache capacity is split evenly across shards.
62const SHARDS: usize = 256;
63
64/// Representative value-size distribution: a small-object-heavy, long-tailed
65/// histogram (bytes → relative weight) approximating the empirical object-size
66/// profile of the real workload (mean ≈ 332 B, tail to 64 KiB). This is ktstr's
67/// own approximation, not a copy of any external size table.
68const VALUE_SIZES: [usize; 8] = [64, 128, 256, 512, 1024, 4096, 16384, 65536];
69const VALUE_WEIGHTS: [u32; 8] = [450, 300, 130, 70, 30, 12, 3, 1];
70
71/// Cap on a single open-loop pacing sleep (ns). A client waiting for its next
72/// scheduled arrival re-checks `stop` at least this often, so a shutdown is
73/// observed within this bound rather than after a full inter-arrival interval —
74/// which at a degenerate low `arrival_rate` (`interval_ns = 1e9 * n_clients /
75/// arrival_rate`) can exceed the scenario cleanup budget and risk a watchdog kill.
76/// At realistic open-loop rates the per-arrival interval is well under this, so the
77/// pacing sleep is a single uncapped wait — the cap only engages at pathological
78/// low rates.
79const STOP_POLL_QUANTUM_NS: u64 = 50_000_000;
80
81/// Read `CLOCK_MONOTONIC` as nanoseconds (monotonic, not wall-clock), matching
82/// the schbench engine's clock source.
83fn monotonic_nanos() -> u64 {
84 // SAFETY: `clock_gettime` writes a `timespec` through the out-pointer and
85 // reads nothing else; CLOCK_MONOTONIC is always available on Linux.
86 let mut ts: libc::timespec = unsafe { core::mem::zeroed() };
87 let rc = unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut ts) };
88 assert_eq!(rc, 0, "clock_gettime(CLOCK_MONOTONIC) failed");
89 (ts.tv_sec as u64) * 1_000_000_000 + ts.tv_nsec as u64
90}
91
92/// The simulated backing-store fetch on the slow path. Worker-defined cost (like
93/// schbench's think-sleep), not a synchronization wait: `std::thread::sleep` maps
94/// to `clock_nanosleep` on Linux. `0` is a no-op (the dispatcher still does the
95/// fill + wakeup, so the slow path remains a distinct thread hop). The caller
96/// passes either the fixed `slow_path_sleep_us` or, when the heavy tail is enabled,
97/// a per-fetch [`sample_service_us`] Pareto draw — this function just sleeps `usec`.
98fn backing_store_fetch(usec: u64) {
99 if usec > 0 {
100 std::thread::sleep(std::time::Duration::from_micros(usec));
101 }
102}
103
104/// Upper bound on a single heavy-tailed slow-path fetch (µs). Pareto is unbounded;
105/// a pathological tail draw must not park a dispatcher off-CPU for an unbounded
106/// time — the dispatcher must complete the fetch + fill + wake for the blocked
107/// client (it cannot be interrupted on shutdown without breaking that contract), so
108/// this caps each fetch at 2 s (far above any realistic backing-store p99.9). The
109/// worst-case client-join/teardown is NOT a single fetch: at shutdown up to
110/// `n_clients` misses are outstanding and each dispatcher drains its queue share
111/// SERIALLY, so the last client can wait up to `ceil(n_clients / n_slow)` fetches —
112/// with the default staffing (`n_slow ≈ n_clients/3`) ~3·MAX_SERVICE_US. Size the
113/// scenario watchdog / cleanup budget accordingly when configuring a deep tail with
114/// many clients per dispatcher. This caps only the per-FETCH off-CPU time, not a
115/// whole serve-latency sample (fetch + open-loop queueing/coordinated-omission
116/// backlog, which can exceed it); the serve sample is kept inside the `PlatStats`
117/// u32 range by `record_serve_lat`'s own `(ns/1000).min(u32::MAX)` clamp,
118/// independent of this cap. Only the fixed legacy path is uncapped (the user sets
119/// that latency directly).
120const MAX_SERVICE_US: u64 = 2_000_000;
121
122/// Draw one heavy-tailed slow-path service time (µs) from the Pareto resolved by
123/// [`TaobenchConfig::resolve_service_pareto`]: `scale · U^(−1/α)` with `U ∈ (0, 1]`
124/// ([`Rng::f64_open01`]). `U = 1` yields the scale (the floor, below the median);
125/// smaller `U` yields the tail. Clamped to [`MAX_SERVICE_US`]. `inv_neg_alpha`
126/// is the precomputed `−1/α` exponent.
127fn sample_service_us(rng: &mut Rng, scale: f64, inv_neg_alpha: f64) -> u64 {
128 let us = scale * rng.f64_open01().powf(inv_neg_alpha);
129 (us as u64).min(MAX_SERVICE_US)
130}
131
132/// Touch every byte of a served/filled value so the read cannot be elided — the
133/// cache memory-bandwidth cost that makes this a cache workload rather than a
134/// control-flow micro-benchmark (schbench's `black_box`-guarded memset is the
135/// precedent). Returns the (black-boxed) checksum.
136fn touch(bytes: &[u8]) -> u64 {
137 let mut acc = 0u64;
138 for &b in bytes {
139 acc = acc.wrapping_add(b as u64);
140 }
141 std::hint::black_box(acc)
142}
143
144/// A small, fast, per-thread xorshift64 PRNG (workload key/size sampling only —
145/// not cryptographic). Seeded distinctly per thread so threads do not march in
146/// lockstep.
147struct Rng(u64);
148
149impl Rng {
150 fn new(seed: u64) -> Self {
151 // Avoid the xorshift fixed point at 0.
152 Rng(seed ^ 0x9E37_79B9_7F4A_7C15)
153 }
154 fn next_u64(&mut self) -> u64 {
155 let mut x = self.0;
156 x ^= x << 13;
157 x ^= x >> 7;
158 x ^= x << 17;
159 self.0 = x;
160 x
161 }
162 /// Uniform in `[0, n)` (`n > 0`); a modulo reduction — the tiny bias is
163 /// irrelevant for a workload key stream.
164 fn below(&mut self, n: u64) -> u64 {
165 self.next_u64() % n
166 }
167 /// Uniform `f64` in `(0, 1]` (open at 0, closed at 1) from the top 53 bits —
168 /// the `OpenClosed01` form the Pareto inverse-CDF needs ([`sample_service_us`]):
169 /// `U = 0` would make `U^(−1/α)` infinite, so 0 is excluded. Workload sampling
170 /// only, not cryptographic.
171 fn f64_open01(&mut self) -> f64 {
172 // Top 53 bits + 1 -> integer in 1..=2^53; / 2^53 -> (0, 1].
173 let bits = (self.next_u64() >> 11) + 1;
174 bits as f64 * (1.0 / ((1u64 << 53) as f64))
175 }
176}
177
178/// Sample a value size from [`VALUE_SIZES`] weighted by [`VALUE_WEIGHTS`].
179fn sample_value_size(rng: &mut Rng) -> usize {
180 let total: u32 = VALUE_WEIGHTS.iter().sum();
181 let mut pick = (rng.below(total as u64)) as u32;
182 for (i, &w) in VALUE_WEIGHTS.iter().enumerate() {
183 if pick < w {
184 return VALUE_SIZES[i];
185 }
186 pick -= w;
187 }
188 VALUE_SIZES[VALUE_SIZES.len() - 1]
189}
190
191/// Build a freshly-allocated, byte-filled value of `size` (the fill touches every
192/// byte, the write-side bandwidth cost).
193fn make_value(size: usize) -> Box<[u8]> {
194 vec![0xABu8; size].into_boxed_slice()
195}
196
197// ---------------------------------------------------------------------------
198// Config
199// ---------------------------------------------------------------------------
200
201/// User-facing config for the [`Taobench`](crate::workload::WorkType::Taobench)
202/// workload — a bounded, evicting key-value cache with a fast hit path and a slow
203/// miss path, driven to a steady-state hit ratio.
204///
205/// All fields are integer/scalar so the type keeps `Eq + Hash` (fractional knobs
206/// are expressed as integer percents). Every field has a chainable builder
207/// setter; [`Default`] is a useful working config.
208#[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
209#[serde(rename_all = "snake_case")]
210pub struct TaobenchConfig {
211 /// CLIENT threads that issue lookups and serve hits. `0` resolves
212 /// to the allocated guest cpuset CPU count (one client per CPU).
213 pub client_threads: usize,
214 /// SLOW dispatcher threads that serve misses (sleep + fill + wake). `0`
215 /// resolves to `max(1, client_threads / 3)`, the real's fast:slow staffing
216 /// ratio.
217 pub slow_threads: usize,
218 /// Resident cache budget in MiB. The cache FIFO-evicts to stay near this many
219 /// bytes' worth of objects; larger than the guest LLC makes the value touch a
220 /// real memory-bandwidth cost.
221 pub cache_capacity_mib: usize,
222 /// Target steady-state hit ratio, in percent (`1..=99`). The key range is
223 /// sized `capacity / target_hit` so a uniform key stream hits at this rate at
224 /// equilibrium. Clamped into range at consumption.
225 pub target_hit_pct: usize,
226 /// Simulated backing-store fetch latency on a miss, in microseconds (the slow
227 /// dispatcher sleeps this long before filling). `0` keeps the slow path as a
228 /// pure thread hop with no sleep (and disables the heavy tail — a zero median
229 /// has no Pareto scale, so the fetch stays a no-op regardless of
230 /// `slow_path_p99_us`). When `slow_path_sleep_us > 0` and `slow_path_p99_us >
231 /// slow_path_sleep_us` this is the MEDIAN (p50) of a heavy-tailed service-time
232 /// distribution rather than a fixed latency (see `slow_path_p99_us`).
233 pub slow_path_sleep_us: u64,
234 /// Heavy-tailed slow-path service time: the p99 of the simulated backing-store
235 /// fetch, in microseconds. When `> slow_path_sleep_us`, each miss's fetch sleep
236 /// is drawn from a Pareto distribution whose median is `slow_path_sleep_us` and
237 /// whose 99th percentile is this value — most fetches near the median, a heavy
238 /// tail of rare slow fetches (GC pauses, cold cache, disk seeks), the realistic
239 /// power-law shape a backing store exhibits. `0` (default), any value
240 /// `<= slow_path_sleep_us`, OR `slow_path_sleep_us == 0` (a zero median has no
241 /// Pareto scale) keeps the fixed-latency behavior (every fetch sleeps exactly
242 /// `slow_path_sleep_us` — the legacy path, byte-identical). The tail is
243 /// clamped at a fixed maximum so a pathological draw cannot park a dispatcher
244 /// off-CPU unboundedly. A ktstr enhancement beyond the reference, whose
245 /// per-request slow-path service time is a FIXED sleep with no tail (the
246 /// reference's `[target/2, 2×target]` uniform jitter + idle-poll backoff is a
247 /// separate poll/idle-wait knob, not the per-request fetch), so the open-loop
248 /// serve-latency tail ([`crate::workload::TaobenchConfig::arrival_rate`])
249 /// reflects realistic
250 /// backing-store variance.
251 pub slow_path_p99_us: u64,
252 /// Open-loop arrival rate, AGGREGATE ops/sec across all client threads (the
253 /// taobench analog of schbench's `-R`). `0` (default) = CLOSED loop: each
254 /// client blocks until its request completes before issuing the next (the
255 /// legacy behavior, byte-identical). Non-zero = OPEN loop: each client has a
256 /// fixed intended-arrival SCHEDULE (`arrival_rate / client_threads` per client)
257 /// independent of completion, and serve latency is measured from that intended
258 /// time. A client still holds at most one outstanding request, so a slow
259 /// completion delays the next issue; that backlog is folded into the late
260 /// requests' serve latency (coordinated-omission correction) rather than
261 /// omitted. Divided evenly across resolved clients. (Named for the standard
262 /// queueing-theory term; the schbench analog is `requests_per_sec`, which is a
263 /// verbatim mirror of schbench's own `-R` CLI flag — a constraint this
264 /// ktstr-added field, with no reference `-R` to mirror, does not share.)
265 pub arrival_rate: usize,
266}
267
268impl Default for TaobenchConfig {
269 fn default() -> Self {
270 Self {
271 client_threads: 0,
272 slow_threads: 0,
273 cache_capacity_mib: 64,
274 target_hit_pct: 90,
275 slow_path_sleep_us: 100,
276 slow_path_p99_us: 0,
277 arrival_rate: 0,
278 }
279 }
280}
281
282impl TaobenchConfig {
283 /// Set the client thread count (`0` = one per allocated CPU).
284 #[must_use = "builder methods consume self; bind the result"]
285 pub fn client_threads(mut self, n: usize) -> Self {
286 self.client_threads = n;
287 self
288 }
289 /// Set the slow dispatcher thread count (`0` = `max(1, client_threads/3)`).
290 #[must_use = "builder methods consume self; bind the result"]
291 pub fn slow_threads(mut self, n: usize) -> Self {
292 self.slow_threads = n;
293 self
294 }
295 /// Set the resident cache budget in MiB.
296 #[must_use = "builder methods consume self; bind the result"]
297 pub fn cache_capacity_mib(mut self, mib: usize) -> Self {
298 self.cache_capacity_mib = mib;
299 self
300 }
301 /// Set the target steady-state hit ratio in percent (`1..=99`).
302 #[must_use = "builder methods consume self; bind the result"]
303 pub fn target_hit_pct(mut self, pct: usize) -> Self {
304 self.target_hit_pct = pct;
305 self
306 }
307 /// Set the simulated backing-store fetch latency on a miss, microseconds (the
308 /// MEDIAN when `slow_path_p99_us` enables the heavy tail; `0` keeps the fetch a
309 /// no-op and also disables the tail — a zero median has no Pareto scale).
310 #[must_use = "builder methods consume self; bind the result"]
311 pub fn slow_path_sleep_us(mut self, us: u64) -> Self {
312 self.slow_path_sleep_us = us;
313 self
314 }
315 /// Set the heavy-tailed slow-path service-time p99 in microseconds (`0` or
316 /// `<= slow_path_sleep_us` = fixed latency, the legacy behavior; also fixed when
317 /// `slow_path_sleep_us == 0`, a zero median having no Pareto scale). When larger
318 /// than a non-zero `slow_path_sleep_us`, each miss's fetch is drawn from a Pareto
319 /// with median `slow_path_sleep_us` and this p99.
320 #[must_use = "builder methods consume self; bind the result"]
321 pub fn slow_path_p99_us(mut self, us: u64) -> Self {
322 self.slow_path_p99_us = us;
323 self
324 }
325 /// Set the open-loop AGGREGATE arrival rate in ops/sec across all clients
326 /// (`0` = closed loop). Divided evenly across resolved client threads; serve
327 /// latency is then measured from the intended arrival (coordinated-omission).
328 #[must_use = "builder methods consume self; bind the result"]
329 pub fn arrival_rate(mut self, ops_per_sec: usize) -> Self {
330 self.arrival_rate = ops_per_sec;
331 self
332 }
333
334 /// Resolve the client thread count: the configured value, or the allocated
335 /// CPU count when `0`.
336 fn resolve_client_threads(&self, allowed_cpus: usize) -> usize {
337 if self.client_threads == 0 {
338 allowed_cpus.max(1)
339 } else {
340 self.client_threads
341 }
342 }
343
344 /// Resolve the slow dispatcher count: the configured value, or
345 /// `max(1, clients/3)` when `0`.
346 fn resolve_slow_threads(&self, clients: usize) -> usize {
347 if self.slow_threads == 0 {
348 (clients / 3).max(1)
349 } else {
350 self.slow_threads
351 }
352 }
353
354 /// Clamp the target hit ratio to a usable open interval (a 0% or ≥100% target
355 /// has no finite key range / no miss stream).
356 fn target_hit_fraction(&self) -> f64 {
357 (self.target_hit_pct.clamp(1, 99) as f64) / 100.0
358 }
359
360 /// Resolve the heavy-tailed slow-path service-time model from the `(p50, p99)`
361 /// knob: `None` = fixed `slow_path_sleep_us` (the legacy path); `Some((scale,
362 /// inv_neg_alpha))` = a Pareto whose median is `slow_path_sleep_us` and whose
363 /// p99 is `slow_path_p99_us`, sampled by [`sample_service_us`].
364 ///
365 /// `slow_path_p99_us <= slow_path_sleep_us` (incl. `0`) resolves to `None` — a
366 /// p99 at or below the median is not a tail. `slow_path_sleep_us == 0` also
367 /// resolves to `None`: a zero median has no Pareto scale, and the legacy no-op
368 /// fetch is preserved.
369 ///
370 /// Mapping: the Pareto q-quantile is `scale·(1−q)^(−1/α)`, so from the two
371 /// quantiles `p99/p50 = (0.01/0.5)^(−1/α) = 50^(1/α)`, giving
372 /// `α = ln(50) / ln(p99/p50)` and `scale = p50·0.5^(1/α)`. `p99 > p50`
373 /// guarantees `ln(p99/p50) > 0 ⇒ α > 0`, and `α ≥ ln(50)/ln(spread)` keeps the
374 /// mean finite (`α > 1`) for any spread below 50x. The stored `inv_neg_alpha =
375 /// −1/α` is the exponent [`sample_service_us`] raises `U` to.
376 fn resolve_service_pareto(&self) -> Option<(f64, f64)> {
377 let p50 = self.slow_path_sleep_us;
378 let p99 = self.slow_path_p99_us;
379 if p50 == 0 || p99 <= p50 {
380 return None;
381 }
382 let (p50f, p99f) = (p50 as f64, p99 as f64);
383 let alpha = 50.0_f64.ln() / (p99f / p50f).ln();
384 let scale = p50f * 0.5_f64.powf(1.0 / alpha);
385 Some((scale, -1.0 / alpha))
386 }
387}
388
389// ---------------------------------------------------------------------------
390// Cache
391// ---------------------------------------------------------------------------
392
393/// One cache shard: a FIFO-evicting map bounded to `cap` objects. FIFO eviction
394/// over a uniform key stream yields the same equilibrium hit ratio as LRU
395/// (resident_fraction = cap / key_range) while being O(1) and lock-cheap.
396struct Shard {
397 map: HashMap<u64, Box<[u8]>>,
398 fifo: VecDeque<u64>,
399 cap: usize,
400}
401
402impl Shard {
403 fn with_cap(cap: usize) -> Self {
404 Shard {
405 map: HashMap::new(),
406 fifo: VecDeque::new(),
407 cap,
408 }
409 }
410 /// Look up `k`; on a hit, touch its bytes and return `true`.
411 fn get_touch(&self, k: u64) -> bool {
412 match self.map.get(&k) {
413 Some(v) => {
414 touch(v);
415 true
416 }
417 None => false,
418 }
419 }
420 /// Insert a freshly-built value for `k` — called at warmup and on a miss,
421 /// where `k` is absent; the `is_none()` guard keeps it correct even if `k`
422 /// is already present. FIFO-evicts down to `cap`.
423 fn insert(&mut self, k: u64, v: Box<[u8]>) {
424 if self.map.insert(k, v).is_none() {
425 self.fifo.push_back(k);
426 while self.map.len() > self.cap {
427 match self.fifo.pop_front() {
428 Some(old) => {
429 self.map.remove(&old);
430 }
431 None => break,
432 }
433 }
434 }
435 }
436}
437
438/// A bounded, sharded, FIFO-evicting cache. Keys map to shards by the low bits of
439/// the key (`SHARDS` is a power of two).
440struct Cache {
441 shards: Vec<Mutex<Shard>>,
442}
443
444impl Cache {
445 /// Build a cache holding ≈ `total_objects` across `SHARDS` shards.
446 fn new(total_objects: usize) -> Self {
447 let per_shard = (total_objects / SHARDS).max(1);
448 let shards = (0..SHARDS)
449 .map(|_| Mutex::new(Shard::with_cap(per_shard)))
450 .collect();
451 Cache { shards }
452 }
453 fn shard(&self, k: u64) -> &Mutex<Shard> {
454 &self.shards[(k as usize) & (SHARDS - 1)]
455 }
456 /// Look up + touch `k`; `true` on a hit.
457 fn get_touch(&self, k: u64) -> bool {
458 self.shard(k)
459 .lock()
460 .expect("cache shard poisoned")
461 .get_touch(k)
462 }
463 /// Fill `k` with a freshly sized + built value (the value bytes are touched
464 /// by `make_value`); FIFO-evicts to keep the shard bounded.
465 fn fill(&self, k: u64, size: usize) {
466 let v = make_value(size);
467 self.shard(k)
468 .lock()
469 .expect("cache shard poisoned")
470 .insert(k, v);
471 }
472}
473
474// ---------------------------------------------------------------------------
475// Slow-path handoff
476// ---------------------------------------------------------------------------
477
478/// A miss handed from a client to a slow dispatcher. The client measures serve
479/// latency from its own local intended-arrival timestamp after `wait_filled`, so
480/// the request itself carries no timing — only the key + the client to wake.
481struct SlowReq {
482 key: u64,
483 client: usize,
484}
485
486/// The slow-request queue: dispatchers block here when idle; clients push misses
487/// and wake one dispatcher.
488struct SlowQueue {
489 q: Mutex<VecDeque<SlowReq>>,
490 cv: Condvar,
491}
492
493impl SlowQueue {
494 fn new() -> Self {
495 SlowQueue {
496 q: Mutex::new(VecDeque::new()),
497 cv: Condvar::new(),
498 }
499 }
500 fn push(&self, req: SlowReq) {
501 self.q.lock().expect("slow queue poisoned").push_back(req);
502 self.cv.notify_one();
503 }
504}
505
506/// A per-client response slot: the dispatcher sets `done` + notifies after the
507/// fill; the client blocks here until its outstanding miss is served. At most one
508/// outstanding request per client in BOTH arrival modes (a client blocks on
509/// `wait_filled` for a miss before issuing its next request), so the slot is
510/// reused. This single-outstanding-request serialization is exactly why open-loop
511/// serve latency must be measured from the intended arrival (coordinated-omission):
512/// a slow completion delays the next issue, and that backlog is folded into the
513/// late requests' latency instead of omitted.
514struct Slot {
515 done: Mutex<bool>,
516 cv: Condvar,
517}
518
519impl Slot {
520 fn new() -> Self {
521 Slot {
522 done: Mutex::new(false),
523 cv: Condvar::new(),
524 }
525 }
526 /// Block until the dispatcher marks this slot done, then reset it.
527 fn wait_filled(&self) {
528 let mut done = self.done.lock().expect("slot poisoned");
529 while !*done {
530 done = self.cv.wait(done).expect("slot poisoned");
531 }
532 *done = false;
533 }
534 /// Mark this slot filled and wake the waiting client.
535 fn signal(&self) {
536 *self.done.lock().expect("slot poisoned") = true;
537 self.cv.notify_one();
538 }
539}
540
541// ---------------------------------------------------------------------------
542// Stats
543// ---------------------------------------------------------------------------
544
545/// Taobench engine counters for one accounting window — a single phase
546/// epoch (the per-phase `crate::workload::PhaseSlice::taobench` carrier) or a
547/// whole worker run (the [`crate::workload::WorkerReport::taobench_whole`] /
548/// [`crate::assert::CgroupStats::taobench_whole`] aggregate). Integer-only so the
549/// enclosing `PhaseSlice` keeps `Eq`. `get_cmds` / `get_misses` are request-time;
550/// `fast_ops` / `slow_ops` are response-time (see the module docs). `Self::merge`
551/// pools two windows (Σ ops, MAX wall) and [`Self::total_ops`] is the throughput
552/// numerator; the host derives the run-level `taobench_*` Rate metrics from the
553/// pooled aggregate.
554#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
555pub struct TaobenchStats {
556 /// Lookups issued (request time).
557 pub get_cmds: u64,
558 /// Lookups that missed (request time).
559 pub get_misses: u64,
560 /// Hits served (response time).
561 pub fast_ops: u64,
562 /// Misses served via the slow path (response time).
563 pub slow_ops: u64,
564 /// Wall-clock window this stat covers, ns. Per-phase: the phase segment;
565 /// whole-run: the run window. Merged as MAX (the window is shared by the
566 /// concurrent threads/workers being pooled, not summed). NOT the throughput
567 /// denominator anymore (that is `cpu_time_ns`); kept for the standalone /
568 /// validate surface (upstream wall-qps shape) and as the window evidence
569 /// from which the wall picture reconstructs exactly (`ops / elapsed`).
570 pub elapsed_ns: u64,
571 /// Σ client-thread `CLOCK_THREAD_CPUTIME_ID` over the window, ns — the
572 /// CPU time the clients actually received, the
573 /// `taobench_*_ops_per_cpu_sec` DENOMINATOR. CPU-clock denominated so the
574 /// throughput measures the workload, not host/guest scheduling delay;
575 /// under host dilation the wall picture is `cpu_rate / D` (the sidecar's
576 /// `host_dilation`). Dispatcher (slow-path service) CPU is EXCLUDED —
577 /// dispatchers model backing-store latency (mostly sleeping), and ops
578 /// complete on client threads. Merged as SUM (per-thread CPU is
579 /// disjoint, unlike the shared wall window). `#[serde(default)]`:
580 /// pre-field sidecars deserialize with 0 (the throughput keys then read
581 /// absent, never a false rate).
582 #[serde(default)]
583 pub cpu_time_ns: u64,
584}
585
586impl TaobenchStats {
587 pub(crate) fn merge(&mut self, o: &TaobenchStats) {
588 // saturating_add: these are guest-runtime monotonic op counters pooled
589 // across concurrent workers/cgroups. Overflow is unreachable for honest
590 // data (real counts ~1e7..1e14 << u64::MAX), but a corrupt/hostile guest
591 // value would otherwise debug-panic or release-wrap to a silently-wrong
592 // qps/hit Rate; saturating is exact for every in-range value.
593 self.get_cmds = self.get_cmds.saturating_add(o.get_cmds);
594 self.get_misses = self.get_misses.saturating_add(o.get_misses);
595 self.fast_ops = self.fast_ops.saturating_add(o.fast_ops);
596 self.slow_ops = self.slow_ops.saturating_add(o.slow_ops);
597 // The wall window is shared across the pooled concurrent threads/workers
598 // (they run the same phase at the same time), so MAX, not sum.
599 self.elapsed_ns = self.elapsed_ns.max(o.elapsed_ns);
600 // Per-thread CPU is disjoint across the pooled threads/workers: SUM.
601 self.cpu_time_ns = self.cpu_time_ns.saturating_add(o.cpu_time_ns);
602 }
603 /// Completed ops (fast + slow) — the throughput numerator.
604 pub fn total_ops(&self) -> u64 {
605 self.fast_ops.saturating_add(self.slow_ops)
606 }
607}
608
609/// One PER-PHASE accounting window's taobench carrier: the integer counters
610/// ([`TaobenchStats`]) plus the open-loop serve-latency histogram. Internal
611/// (`pub(crate)`) — the taobench analog of
612/// [`crate::workload::schbench::run::SchbenchPhaseStats`] (counters + `PlatStats`
613/// histograms together), and like it, never on a public field. It rides only the
614/// per-phase carriers (`PhaseSlice::taobench` / `PhaseCgroupStats::taobench`); the
615/// WHOLE-RUN carrier is the counters-only [`TaobenchStats`] (the serve histogram
616/// is per-phase data — the whole-run serve distribution is the union of these
617/// per-phase histograms, computed where needed). `serve_lat` is EMPTY in closed
618/// loop: serve latency is measured only under open-loop arrival, so its
619/// percentiles read absent there.
620#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
621pub(crate) struct TaobenchPhaseStats {
622 /// Throughput / hit counters (request- and response-time).
623 pub(crate) counters: TaobenchStats,
624 /// Coordinated-omission serve latency (µs), measured from intended arrival.
625 /// Exposed to test authors only via the derived `taobench_serve_*_us` metric
626 /// keys (the schbench-histogram model), never as a direct field.
627 pub(crate) serve_lat: PlatStats,
628}
629
630impl TaobenchPhaseStats {
631 /// Pool another window: Σ counters ([`TaobenchStats::merge`]) and union the
632 /// serve histograms ([`PlatStats::combine`]) — the per-cgroup / cross-thread
633 /// fold, mirroring the schbench carrier merge.
634 pub(crate) fn merge(&mut self, other: &TaobenchPhaseStats) {
635 self.counters.merge(&other.counters);
636 self.serve_lat.combine(&other.serve_lat);
637 }
638}
639
640/// The engine's return: the whole-run merged COUNTERS ([`TaobenchStats`], the
641/// qps/hit aggregate) and the per-phase-epoch carriers (each counters + the
642/// serve-latency histogram). The whole-run carrier is counters-only — mirroring
643/// the wire carriers (`WorkerReport::taobench_whole` / `CgroupStats::taobench_whole`);
644/// a whole-run serve distribution, when needed (the standalone driver), is the
645/// union of the per-phase `serve_lat` histograms.
646pub(crate) struct TaobenchOutcome {
647 pub whole_run: TaobenchStats,
648 pub phases: Vec<(u32, TaobenchPhaseStats)>,
649}
650
651/// Per-thread accumulation with phase-epoch bucketing: the current epoch's
652/// counters roll into `phases` when the epoch changes (the schbench drain-on-epoch
653/// pattern), and `whole` accumulates the un-bucketed run total. Wall windows are
654/// stamped from `CLOCK_MONOTONIC` at each roll (per-phase segment) and at finalize
655/// (whole-run).
656struct ThreadAccum {
657 cur_epoch: u32,
658 cur: TaobenchPhaseStats,
659 phases: BTreeMap<u32, TaobenchPhaseStats>,
660 /// Whole-run counters only (the qps/hit aggregate). The serve histogram is
661 /// per-phase data (in `cur`/`phases`); the whole-run serve distribution, when
662 /// needed, is the union of the per-phase histograms.
663 whole: TaobenchStats,
664 /// When the current phase segment started (ns).
665 phase_start_ns: u64,
666 /// When this thread started (ns) — the whole-run window start.
667 thread_start_ns: u64,
668 /// `CLOCK_THREAD_CPUTIME_ID` at the current phase segment's start (ns) —
669 /// baselines the per-phase client-CPU delta (the CPU-qps denominator).
670 phase_start_cpu_ns: u64,
671 /// `CLOCK_THREAD_CPUTIME_ID` at thread start (ns) — baselines the
672 /// whole-run client-CPU delta.
673 thread_start_cpu_ns: u64,
674}
675
676impl ThreadAccum {
677 fn new(epoch: u32) -> Self {
678 let now = monotonic_nanos();
679 let cpu_now = crate::workload::worker::sched::thread_cpu_time_ns();
680 ThreadAccum {
681 cur_epoch: epoch,
682 cur: TaobenchPhaseStats::default(),
683 phases: BTreeMap::new(),
684 whole: TaobenchStats::default(),
685 phase_start_ns: now,
686 thread_start_ns: now,
687 phase_start_cpu_ns: cpu_now,
688 thread_start_cpu_ns: cpu_now,
689 }
690 }
691 /// Roll the current bucket into `phases` (stamping its segment wall + CPU
692 /// windows) and start a fresh one for `epoch`.
693 fn roll_to(&mut self, epoch: u32) {
694 if epoch != self.cur_epoch {
695 let now = monotonic_nanos();
696 let cpu_now = crate::workload::worker::sched::thread_cpu_time_ns();
697 let mut cur = std::mem::take(&mut self.cur);
698 cur.counters.elapsed_ns = now.saturating_sub(self.phase_start_ns);
699 cur.counters.cpu_time_ns = cpu_now.saturating_sub(self.phase_start_cpu_ns);
700 self.phases.entry(self.cur_epoch).or_default().merge(&cur);
701 self.cur_epoch = epoch;
702 self.phase_start_ns = now;
703 self.phase_start_cpu_ns = cpu_now;
704 }
705 }
706 /// Record a lookup at request time in `epoch`.
707 fn record_cmd(&mut self, epoch: u32, hit: bool) {
708 self.roll_to(epoch);
709 self.cur.counters.get_cmds += 1;
710 self.whole.get_cmds += 1;
711 if !hit {
712 self.cur.counters.get_misses += 1;
713 self.whole.get_misses += 1;
714 }
715 }
716 /// Record a completion at response time in `epoch`.
717 fn record_complete(&mut self, epoch: u32, hit: bool) {
718 self.roll_to(epoch);
719 if hit {
720 self.cur.counters.fast_ops += 1;
721 self.whole.fast_ops += 1;
722 } else {
723 self.cur.counters.slow_ops += 1;
724 self.whole.slow_ops += 1;
725 }
726 }
727 /// Record an open-loop serve latency (`ns` from the intended arrival) in
728 /// `epoch`, into the current phase segment's histogram (µs buckets). Closed
729 /// loop never calls this, so the histograms stay empty and the serve-latency
730 /// keys read absent. The whole-run serve distribution is the union of the
731 /// per-phase histograms, so only the per-phase bucket is recorded here.
732 fn record_serve_lat(&mut self, epoch: u32, ns: u64) {
733 self.roll_to(epoch);
734 let us = (ns / 1000).min(u32::MAX as u64) as u32;
735 self.cur.serve_lat.add_lat(us);
736 }
737 /// Flush the last open bucket (stamping its segment wall + CPU windows)
738 /// and stamp the whole-run wall + CPU windows.
739 fn finalize(mut self) -> Self {
740 let now = monotonic_nanos();
741 let cpu_now = crate::workload::worker::sched::thread_cpu_time_ns();
742 let mut cur = std::mem::take(&mut self.cur);
743 cur.counters.elapsed_ns = now.saturating_sub(self.phase_start_ns);
744 cur.counters.cpu_time_ns = cpu_now.saturating_sub(self.phase_start_cpu_ns);
745 self.phases.entry(self.cur_epoch).or_default().merge(&cur);
746 self.whole.elapsed_ns = now.saturating_sub(self.thread_start_ns);
747 self.whole.cpu_time_ns = cpu_now.saturating_sub(self.thread_start_cpu_ns);
748 self
749 }
750}
751
752/// Read the current phase epoch (`0` when phases are not tracked).
753fn read_epoch(phase_epoch: Option<&AtomicU32>) -> u32 {
754 phase_epoch.map(|e| e.load(Ordering::Relaxed)).unwrap_or(0)
755}
756
757// ---------------------------------------------------------------------------
758// Engine
759// ---------------------------------------------------------------------------
760
761/// Run the taobench engine until `stop` is set. `progress` counts completed ops
762/// (the live work-unit counter); `phase_epoch` buckets per-phase stats.
763pub(crate) fn run(
764 config: &TaobenchConfig,
765 stop: &AtomicBool,
766 progress: &AtomicU64,
767 phase_epoch: Option<&AtomicU32>,
768) -> TaobenchOutcome {
769 let allowed_cpus = resolve_allowed_cpus();
770 let n_clients = config.resolve_client_threads(allowed_cpus);
771 let n_slow = config.resolve_slow_threads(n_clients);
772 // Open-loop per-client inter-arrival = 1s / (arrival_rate / n_clients) =
773 // 1e9 * n_clients / arrival_rate. `0` arrival_rate ⇒ `0` interval ⇒ closed
774 // loop (issue as fast as completions allow — the legacy behavior). `.max(1)`
775 // keeps a non-zero rate strictly open-loop even at an absurd rate (so the
776 // `interval_ns != 0` mode gate in client_loop never misfires to closed loop).
777 // saturating_mul: an absurd client count (physically unreachable — it would
778 // OOM on thread spawn first) cannot wrap the numerator to a tiny interval, the
779 // same overflow-safe discipline as `TaobenchStats::merge`.
780 let interval_ns: u64 = if config.arrival_rate == 0 {
781 0
782 } else {
783 (1_000_000_000u64.saturating_mul(n_clients as u64) / config.arrival_rate as u64).max(1)
784 };
785 // Resolve the slow-path service-time model once (None = fixed legacy latency;
786 // Some = a per-fetch Pareto draw). Cheap, but kept out of the dispatcher hot
787 // path.
788 let service_pareto = config.resolve_service_pareto();
789
790 // Size the resident object count and the key range. Mean value size pins the
791 // object count for a byte budget; key_range = capacity / target_hit makes a
792 // uniform key stream hit the resident set at ≈ target_hit at equilibrium.
793 let mean_value = mean_value_size();
794 let capacity_bytes = config.cache_capacity_mib.max(1) * 1024 * 1024;
795 let total_objects = (capacity_bytes / mean_value).max(SHARDS);
796 let key_range = ((total_objects as f64) / config.target_hit_fraction()).ceil() as u64;
797
798 let cache = Cache::new(total_objects);
799 // Warm the cache to capacity so the hit ratio starts at the target rather
800 // than climbing from empty (the real warms before measuring). Keys
801 // [0, total_objects) are the initial resident set; the client key stream over
802 // [0, key_range) then hits them at ≈ target_hit and the eviction<->refill
803 // equilibrium holds it there.
804 {
805 let mut warm = Rng::new(0xC0FFEE);
806 for k in 0..total_objects as u64 {
807 cache.fill(k, sample_value_size(&mut warm));
808 }
809 }
810
811 let slow_q = SlowQueue::new();
812 let slots: Vec<Slot> = (0..n_clients).map(|_| Slot::new()).collect();
813 let disp_stop = AtomicBool::new(false);
814
815 let client_accums: Vec<ThreadAccum> = std::thread::scope(|s| {
816 // Dispatchers first so a client miss is served immediately.
817 let dispatchers: Vec<_> = (0..n_slow)
818 .map(|i| {
819 let cache = &cache;
820 let slow_q = &slow_q;
821 let slots = &slots;
822 let disp_stop = &disp_stop;
823 s.spawn(move || {
824 dispatcher_loop(i, config, service_pareto, cache, slow_q, slots, disp_stop)
825 })
826 })
827 .collect();
828
829 let clients: Vec<_> = (0..n_clients)
830 .map(|i| {
831 let cache = &cache;
832 let slow_q = &slow_q;
833 let slot = &slots[i];
834 s.spawn(move || {
835 client_loop(
836 i,
837 stop,
838 progress,
839 phase_epoch,
840 cache,
841 slow_q,
842 slot,
843 key_range,
844 interval_ns,
845 )
846 })
847 })
848 .collect();
849
850 // Join the clients first: once every client has returned (each only
851 // exits at the top of its loop, after its outstanding request — if any —
852 // has been served), NO further misses can be enqueued. Only then signal
853 // the dispatchers, so a dispatcher can never exit while a client is still
854 // blocked on an unserved miss.
855 let accums: Vec<ThreadAccum> = clients
856 .into_iter()
857 .map(|c| c.join().expect("taobench client panicked"))
858 .collect();
859 disp_stop.store(true, Ordering::Release);
860 slow_q.cv.notify_all();
861 for d in dispatchers {
862 d.join().expect("taobench dispatcher panicked");
863 }
864 accums
865 });
866
867 // Reduce: merge per-epoch and whole-run across clients. Each ThreadAccum
868 // stamped its per-phase segment walls + its whole-run window in finalize().
869 let mut all_phases: BTreeMap<u32, TaobenchPhaseStats> = BTreeMap::new();
870 let mut whole = TaobenchStats::default();
871 for accum in client_accums {
872 for (e, s) in accum.phases {
873 all_phases.entry(e).or_default().merge(&s);
874 }
875 whole.merge(&accum.whole);
876 }
877
878 TaobenchOutcome {
879 whole_run: whole,
880 phases: all_phases.into_iter().collect(),
881 }
882}
883
884/// One client: pick a key, look it up, serve a hit inline (touch + fast op) or
885/// hand a miss to a dispatcher and block until filled (slow op). `interval_ns ==
886/// 0` is CLOSED loop (issue the next request as soon as the prior completes — the
887/// legacy behavior). `interval_ns > 0` is OPEN loop: requests are due on a fixed
888/// `interval_ns` cadence independent of completion (the first one interval after
889/// the loop start `t0`), and serve latency is measured from the INTENDED arrival
890/// (coordinated-omission) so a backlog from slow service is folded into the late
891/// requests' latency instead of omitted.
892#[allow(clippy::too_many_arguments)]
893fn client_loop(
894 id: usize,
895 stop: &AtomicBool,
896 progress: &AtomicU64,
897 phase_epoch: Option<&AtomicU32>,
898 cache: &Cache,
899 slow_q: &SlowQueue,
900 slot: &Slot,
901 key_range: u64,
902 interval_ns: u64,
903) -> ThreadAccum {
904 let mut rng = Rng::new(0x5EED_0000 ^ (id as u64).wrapping_mul(0x9E37_79B1));
905 let mut accum = ThreadAccum::new(read_epoch(phase_epoch));
906 let open_loop = interval_ns != 0;
907 // Open-loop schedule cursor, advanced one `interval` before each request: the
908 // first request is due at `t0 + interval`, then every `interval` thereafter.
909 let mut intended = monotonic_nanos();
910
911 'client: while !stop.load(Ordering::Acquire) {
912 if open_loop {
913 // Pace to the next scheduled arrival; if behind (overload), the wait
914 // collapses to zero and the gap is captured as serve latency below.
915 // Each sleep is capped at STOP_POLL_QUANTUM_NS so a shutdown set during
916 // a long inter-arrival gap is observed within the quantum (not after a
917 // full interval) — exiting promptly via `break 'client` to `finalize`,
918 // bounding client-join teardown regardless of `arrival_rate`.
919 intended = intended.saturating_add(interval_ns);
920 loop {
921 if stop.load(Ordering::Acquire) {
922 break 'client;
923 }
924 let now = monotonic_nanos();
925 if now >= intended {
926 break;
927 }
928 let nap = (intended - now).min(STOP_POLL_QUANTUM_NS);
929 std::thread::sleep(std::time::Duration::from_nanos(nap));
930 }
931 }
932 let epoch_cmd = read_epoch(phase_epoch);
933 let key = rng.below(key_range);
934 let hit = cache.get_touch(key);
935 accum.record_cmd(epoch_cmd, hit);
936
937 if hit {
938 // Hit completes inline; response epoch == request epoch.
939 accum.record_complete(epoch_cmd, true);
940 if open_loop {
941 let lat = monotonic_nanos().saturating_sub(intended);
942 accum.record_serve_lat(epoch_cmd, lat);
943 }
944 } else {
945 // Miss: hand to a dispatcher and block until it fills + wakes us.
946 // The dispatcher always serves us before `disp_stop` is set (which
947 // happens only after every client has joined), so this never blocks
948 // past shutdown.
949 slow_q.push(SlowReq { key, client: id });
950 slot.wait_filled();
951 let resp_epoch = read_epoch(phase_epoch);
952 accum.record_complete(resp_epoch, false);
953 if open_loop {
954 // Completion timestamp taken by the CLIENT after wait_filled, so
955 // serve latency covers the full slow_q queueing + dispatcher
956 // service + wakeup, all measured from the intended arrival.
957 let lat = monotonic_nanos().saturating_sub(intended);
958 accum.record_serve_lat(resp_epoch, lat);
959 }
960 }
961 progress.fetch_add(1, Ordering::Relaxed);
962 }
963 accum.finalize()
964}
965
966/// One slow dispatcher: pull a miss, perform the simulated backing-store fetch,
967/// fill the cache, and wake the waiting client. Drains any queued misses before
968/// exiting on `disp_stop`. `service_pareto` is the resolved heavy-tail model
969/// (`None` = fixed `slow_path_sleep_us`); the per-fetch draw is deterministic from
970/// this dispatcher's seeded `rng` (the same one that picks the fill value size).
971fn dispatcher_loop(
972 id: usize,
973 config: &TaobenchConfig,
974 service_pareto: Option<(f64, f64)>,
975 cache: &Cache,
976 slow_q: &SlowQueue,
977 slots: &[Slot],
978 disp_stop: &AtomicBool,
979) {
980 let mut rng = Rng::new(0xD15A_0000 ^ (id as u64).wrapping_mul(0x9E37_79B1));
981 loop {
982 let req = {
983 let mut q = slow_q.q.lock().expect("slow queue poisoned");
984 loop {
985 if let Some(r) = q.pop_front() {
986 break Some(r);
987 }
988 if disp_stop.load(Ordering::Acquire) {
989 break None;
990 }
991 q = slow_q.cv.wait(q).expect("slow queue poisoned");
992 }
993 };
994 let Some(req) = req else { break };
995
996 // Heavy-tailed Pareto draw when configured, else the fixed latency. The
997 // draw precedes the value-size draw, so both come off this dispatcher's
998 // deterministic seeded stream.
999 let usec = match service_pareto {
1000 Some((scale, inv_neg_alpha)) => sample_service_us(&mut rng, scale, inv_neg_alpha),
1001 None => config.slow_path_sleep_us,
1002 };
1003 backing_store_fetch(usec);
1004 cache.fill(req.key, sample_value_size(&mut rng));
1005 slots[req.client].signal();
1006 }
1007}
1008
1009// ---------------------------------------------------------------------------
1010// Topology + size helpers
1011// ---------------------------------------------------------------------------
1012
1013/// The mean of the value-size distribution (for sizing the object count).
1014fn mean_value_size() -> usize {
1015 let total_w: u64 = VALUE_WEIGHTS.iter().map(|&w| w as u64).sum();
1016 let weighted: u64 = VALUE_SIZES
1017 .iter()
1018 .zip(VALUE_WEIGHTS.iter())
1019 .map(|(&s, &w)| s as u64 * w as u64)
1020 .sum();
1021 (weighted / total_w).max(1) as usize
1022}
1023
1024/// Resolve the number of CPUs the worker is allowed to run on (its affinity
1025/// mask), falling back to the online CPU count. Mirrors the schbench engine's
1026/// cpuset-scoped resolution.
1027fn resolve_allowed_cpus() -> usize {
1028 // SAFETY: `sched_getaffinity` writes the calling thread's CPU mask into the
1029 // provided cpu_set; reads nothing else.
1030 unsafe {
1031 let mut set: libc::cpu_set_t = core::mem::zeroed();
1032 if libc::sched_getaffinity(0, core::mem::size_of::<libc::cpu_set_t>(), &mut set) == 0 {
1033 let n = libc::CPU_COUNT(&set);
1034 if n > 0 {
1035 return n as usize;
1036 }
1037 }
1038 }
1039 let n = unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) };
1040 if n > 0 { n as usize } else { 1 }
1041}
1042
1043// ---------------------------------------------------------------------------
1044// Host-side standalone runner (validation driver entry)
1045// ---------------------------------------------------------------------------
1046
1047/// Host-side standalone run of the taobench engine for `run_secs`, returning a
1048/// summary report — the analog of schbench's `run_standalone`, backing the
1049/// `ktstr-taobench-validate` driver for the side-by-side comparison against the
1050/// reference taobench. NOT used in-VM (the scenario engine drives `run` there).
1051///
1052/// The engine warms the cache synchronously before the client threads start; the
1053/// report's `elapsed_secs` is the engine-measured CLIENT window (post-warmup), so
1054/// the qps figures are steady-state, not diluted by warmup.
1055pub fn run_standalone(config: &TaobenchConfig, run_secs: u64) -> TaobenchStandaloneReport {
1056 let stop = AtomicBool::new(false);
1057 let progress = AtomicU64::new(0);
1058 let allowed_cpus = resolve_allowed_cpus();
1059 let nr_client_threads = config.resolve_client_threads(allowed_cpus);
1060 let nr_slow_threads = config.resolve_slow_threads(nr_client_threads);
1061 let outcome = std::thread::scope(|s| {
1062 let h = s.spawn(|| run(config, &stop, &progress, None));
1063 // The benchmark run window — a workload-defined duration (the same model
1064 // as the scenario engine's hold step), not a synchronization wait.
1065 std::thread::sleep(std::time::Duration::from_secs(run_secs));
1066 stop.store(true, Ordering::Release);
1067 h.join().expect("taobench standalone run panicked")
1068 });
1069 // Whole-run serve-latency distribution = union of the per-phase histograms
1070 // (the whole-run carrier is counters-only). Empty in closed loop, so the
1071 // report's serve percentiles read absent there.
1072 let mut serve = PlatStats::default();
1073 for (_epoch, p) in &outcome.phases {
1074 serve.combine(&p.serve_lat);
1075 }
1076 TaobenchStandaloneReport::from_run(
1077 &outcome.whole_run,
1078 &serve,
1079 nr_client_threads,
1080 nr_slow_threads,
1081 )
1082}
1083
1084/// Summary of a [`run_standalone`] run — the headline taobench metrics in the
1085/// shape the reference taobench server reports (`fast_qps` / `hit_rate` /
1086/// `slow_qps`) plus the derived `total_qps` (= fast + slow) and `hit_ratio`
1087/// (= fast / total). Under open-loop arrival (`arrival_rate > 0`) it also carries
1088/// the coordinated-omission serve-latency percentiles; these are `None` in closed
1089/// loop (no intended-arrival schedule, so no serve latency is measured).
1090///
1091/// DENOMINATION: the qps fields here are WALL-clock (`ops / elapsed`) on
1092/// purpose — this is the standalone/validate surface whose output shape must
1093/// stay byte-identical to the reference taobench report. The ktstr sidecar's
1094/// `taobench_*_ops_per_cpu_sec` metrics are CPU-second denominated instead
1095/// (see [`TaobenchStats::cpu_time_ns`]).
1096#[derive(Debug, Clone, Copy, PartialEq)]
1097pub struct TaobenchStandaloneReport {
1098 /// (fast + slow) ops per second over the measured window.
1099 pub total_qps: f64,
1100 /// Hits served per second.
1101 pub fast_qps: f64,
1102 /// Misses served (slow path) per second.
1103 pub slow_qps: f64,
1104 /// Response-time hit ratio: fast / (fast + slow).
1105 pub hit_ratio: f64,
1106 /// Command-time hit rate: 1 - get_misses / get_cmds.
1107 pub hit_rate: f64,
1108 /// Completed ops (fast + slow).
1109 pub total_ops: u64,
1110 /// Hits served.
1111 pub fast_ops: u64,
1112 /// Misses served via the slow path.
1113 pub slow_ops: u64,
1114 /// The engine-measured client window, seconds.
1115 pub elapsed_secs: f64,
1116 /// Resolved client thread count.
1117 pub nr_client_threads: usize,
1118 /// Resolved slow dispatcher count.
1119 pub nr_slow_threads: usize,
1120 /// Open-loop coordinated-omission serve-latency percentiles (µs), measured
1121 /// from intended arrival. `None` in closed loop / when no serve samples were
1122 /// recorded (matching the metric-key ABSENT discipline).
1123 pub serve_p50_us: Option<u32>,
1124 /// Open-loop serve-latency p99 (µs) — the headline coordinated-omission tail.
1125 pub serve_p99_us: Option<u32>,
1126 /// Open-loop serve-latency p99.9 (µs).
1127 pub serve_p999_us: Option<u32>,
1128 /// Open-loop serve-latency minimum sample (µs).
1129 pub serve_min_us: Option<u32>,
1130 /// Open-loop serve-latency maximum sample (µs).
1131 pub serve_max_us: Option<u32>,
1132}
1133
1134impl TaobenchStandaloneReport {
1135 fn from_run(
1136 w: &TaobenchStats,
1137 serve: &PlatStats,
1138 nr_client_threads: usize,
1139 nr_slow_threads: usize,
1140 ) -> Self {
1141 use crate::workload::schbench::plat::Pct;
1142 let secs = (w.elapsed_ns as f64 / 1e9).max(f64::MIN_POSITIVE);
1143 let total = w.total_ops();
1144 // Serve-latency percentiles are present iff open-loop arrival recorded
1145 // samples; closed loop leaves the histogram empty, so all fields read
1146 // None (the metric-key ABSENT discipline).
1147 let serve_q = (serve.sample_count() > 0).then(|| serve.percentiles());
1148 TaobenchStandaloneReport {
1149 total_qps: total as f64 / secs,
1150 fast_qps: w.fast_ops as f64 / secs,
1151 slow_qps: w.slow_ops as f64 / secs,
1152 hit_ratio: if total > 0 {
1153 w.fast_ops as f64 / total as f64
1154 } else {
1155 0.0
1156 },
1157 hit_rate: if w.get_cmds > 0 {
1158 1.0 - (w.get_misses as f64 / w.get_cmds as f64)
1159 } else {
1160 0.0
1161 },
1162 total_ops: total,
1163 fast_ops: w.fast_ops,
1164 slow_ops: w.slow_ops,
1165 elapsed_secs: secs,
1166 nr_client_threads,
1167 nr_slow_threads,
1168 serve_p50_us: serve_q.as_ref().map(|q| q.value_at(Pct::P50)),
1169 serve_p99_us: serve_q.as_ref().map(|q| q.value_at(Pct::P99)),
1170 serve_p999_us: serve_q.as_ref().map(|q| q.value_at(Pct::P999)),
1171 serve_min_us: serve_q.as_ref().map(|q| q.min),
1172 serve_max_us: serve_q.as_ref().map(|q| q.max),
1173 }
1174 }
1175}
1176
1177#[cfg(test)]
1178mod tests {
1179 use super::*;
1180
1181 #[test]
1182 fn value_size_distribution_mean_is_small_object_heavy() {
1183 // The representative distribution is small-object-heavy (mean well under
1184 // 1 KiB) with a tail to 64 KiB.
1185 let m = mean_value_size();
1186 assert!(
1187 (200..=500).contains(&m),
1188 "mean value size {m} B is small-object-heavy"
1189 );
1190 assert_eq!(*VALUE_SIZES.last().unwrap(), 65536, "tail reaches 64 KiB");
1191 }
1192
1193 #[test]
1194 fn taobench_stats_merge_and_total_ops_saturate_on_overflow() {
1195 // Pooling guest-runtime op counters must saturate, not wrap: a corrupt
1196 // u64::MAX component must never produce a silently-wrong (wrapped-small)
1197 // qps/hit Rate. Saturating is exact for all in-range data.
1198 let mut a = TaobenchStats {
1199 get_cmds: u64::MAX,
1200 get_misses: u64::MAX,
1201 fast_ops: u64::MAX,
1202 slow_ops: 1,
1203 elapsed_ns: 1000,
1204 cpu_time_ns: u64::MAX,
1205 };
1206 let b = TaobenchStats {
1207 get_cmds: 5,
1208 get_misses: 5,
1209 fast_ops: 5,
1210 slow_ops: 5,
1211 elapsed_ns: 9000,
1212 cpu_time_ns: 5,
1213 };
1214 a.merge(&b);
1215 assert_eq!(a.get_cmds, u64::MAX, "saturates, not wraps to 4");
1216 assert_eq!(a.get_misses, u64::MAX);
1217 assert_eq!(a.fast_ops, u64::MAX);
1218 assert_eq!(a.slow_ops, 6);
1219 assert_eq!(a.elapsed_ns, 9000, "wall window is MAX, not summed");
1220 // total_ops = fast_ops + slow_ops saturates (u64::MAX + 6 -> u64::MAX).
1221 assert_eq!(a.total_ops(), u64::MAX, "total_ops saturates");
1222 }
1223
1224 #[test]
1225 fn engine_serves_ops_and_hit_ratio_settles_near_target_not_one() {
1226 // A short run on a small cache: the hit ratio must settle near the target
1227 // (the eviction<->refill equilibrium), NOT drift to 1.0 (which a
1228 // self-healing cache with no eviction would do).
1229 let cfg = TaobenchConfig::default()
1230 .client_threads(4)
1231 .slow_threads(2)
1232 .cache_capacity_mib(8)
1233 .target_hit_pct(90)
1234 .slow_path_sleep_us(10);
1235 let stop = AtomicBool::new(false);
1236 let progress = AtomicU64::new(0);
1237
1238 std::thread::scope(|s| {
1239 let h = s.spawn(|| run(&cfg, &stop, &progress, None));
1240 // Spin until the engine has served enough ops to reach equilibrium.
1241 while progress.load(Ordering::Relaxed) < 100_000 {
1242 std::hint::spin_loop();
1243 }
1244 stop.store(true, Ordering::Release);
1245 let out = h.join().expect("engine panicked");
1246
1247 let total = out.whole_run.total_ops();
1248 assert!(total > 0, "engine served ops");
1249 assert!(out.whole_run.elapsed_ns > 0, "the run window was measured");
1250
1251 let hit = out.whole_run.fast_ops as f64 / total as f64;
1252 assert!(
1253 (0.80..=0.97).contains(&hit),
1254 "hit ratio {hit} settles near target 0.90 (eviction equilibrium), not 1.0"
1255 );
1256 assert!(
1257 out.whole_run.slow_ops > 0,
1258 "the slow/miss path is exercised"
1259 );
1260 });
1261 }
1262
1263 #[test]
1264 fn taobench_config_serde_roundtrips() {
1265 // The new serialized type roundtrips unchanged.
1266 // Every field set to a non-default value to exercise the full serde surface.
1267 let cfg = TaobenchConfig::default()
1268 .client_threads(8)
1269 .slow_threads(3)
1270 .cache_capacity_mib(128)
1271 .target_hit_pct(85)
1272 .slow_path_sleep_us(250)
1273 .slow_path_p99_us(2500)
1274 .arrival_rate(50_000);
1275 let json = serde_json::to_string(&cfg).expect("TaobenchConfig must serialize");
1276 let back: TaobenchConfig =
1277 serde_json::from_str(&json).expect("TaobenchConfig must deserialize");
1278 assert_eq!(cfg, back, "config roundtrips unchanged");
1279 }
1280
1281 #[test]
1282 fn worktype_taobench_registration_and_serde() {
1283 use crate::workload::WorkType;
1284 let wt = WorkType::taobench(
1285 TaobenchConfig::default()
1286 .client_threads(4)
1287 .target_hit_pct(95),
1288 );
1289 assert_eq!(wt.name(), "Taobench");
1290 // from_name yields the default-config variant.
1291 assert_eq!(
1292 WorkType::from_name("Taobench"),
1293 Some(WorkType::Taobench {
1294 config: TaobenchConfig::default()
1295 })
1296 );
1297 // The variant serde-roundtrips, carrying its config.
1298 let json = serde_json::to_string(&wt).expect("WorkType::Taobench must serialize");
1299 let back: WorkType =
1300 serde_json::from_str(&json).expect("WorkType::Taobench must deserialize");
1301 assert_eq!(wt, back);
1302 }
1303
1304 #[test]
1305 fn taobench_config_reachable_via_prelude() {
1306 // Regression-pin the prelude placement: test authors construct the config
1307 // via `use ktstr::prelude::*`. Dropping TaobenchConfig from the prelude
1308 // would fail this compile. Also exercises the Eq derive.
1309 let cfg: crate::prelude::TaobenchConfig = crate::prelude::TaobenchConfig::default();
1310 assert_eq!(cfg, TaobenchConfig::default());
1311 }
1312
1313 #[test]
1314 fn taobench_stats_serde_roundtrips() {
1315 // TaobenchStats is a pub serialized type that rides real wires — a field
1316 // on WorkerReport (the worker→host postcard payload) and CgroupStats (the
1317 // sidecar JSON) — so a field-rename or serde-attr drift is a silent
1318 // data-loss risk this pins. Every field distinct +
1319 // non-zero to exercise the full serde surface.
1320 let s = TaobenchStats {
1321 get_cmds: 1000,
1322 get_misses: 150,
1323 fast_ops: 850,
1324 slow_ops: 150,
1325 elapsed_ns: 9_000_000_000,
1326 cpu_time_ns: 4_200_000_000,
1327 };
1328 let json = serde_json::to_string(&s).expect("TaobenchStats must serialize");
1329 let back: TaobenchStats =
1330 serde_json::from_str(&json).expect("TaobenchStats must deserialize");
1331 assert_eq!(s, back, "TaobenchStats roundtrips unchanged");
1332 }
1333
1334 #[test]
1335 fn taobench_stats_reachable_via_prelude() {
1336 // Regression-pin the prelude placement: TaobenchStats is read off the
1337 // preluded WorkerReport/CgroupStats `taobench_whole` fields, so it must be
1338 // nameable via `use ktstr::prelude::*`. Dropping it from the prelude would
1339 // fail this compile. Also exercises Default + Eq.
1340 let s: crate::prelude::TaobenchStats = crate::prelude::TaobenchStats::default();
1341 assert_eq!(s, TaobenchStats::default());
1342 }
1343
1344 #[test]
1345 fn taobench_config_arrival_rate_default_and_setter() {
1346 // Default is closed loop (no open-loop schedule); the setter sets the
1347 // aggregate ops/sec field.
1348 assert_eq!(
1349 TaobenchConfig::default().arrival_rate,
1350 0,
1351 "default arrival_rate is 0 (closed loop)"
1352 );
1353 let cfg = TaobenchConfig::default().arrival_rate(100_000);
1354 assert_eq!(cfg.arrival_rate, 100_000, "setter sets arrival_rate");
1355 }
1356
1357 #[test]
1358 fn taobench_phase_stats_merge_sums_counters_and_unions_serve_lat() {
1359 // The carrier merge pools counters (Σ ops, MAX wall — TaobenchStats::merge)
1360 // AND unions the serve histograms (PlatStats::combine) — the per-cgroup /
1361 // cross-thread fold.
1362 let mut a = TaobenchPhaseStats {
1363 counters: TaobenchStats {
1364 get_cmds: 10,
1365 get_misses: 2,
1366 fast_ops: 8,
1367 slow_ops: 2,
1368 elapsed_ns: 1000,
1369 cpu_time_ns: 700,
1370 },
1371 ..Default::default()
1372 };
1373 a.serve_lat.add_lat(5);
1374 a.serve_lat.add_lat(50);
1375 let mut b = TaobenchPhaseStats {
1376 counters: TaobenchStats {
1377 get_cmds: 6,
1378 get_misses: 1,
1379 fast_ops: 5,
1380 slow_ops: 1,
1381 elapsed_ns: 4000,
1382 cpu_time_ns: 300,
1383 },
1384 ..Default::default()
1385 };
1386 b.serve_lat.add_lat(500);
1387 a.merge(&b);
1388 assert_eq!(a.counters.get_cmds, 16, "Σ get_cmds");
1389 assert_eq!(a.counters.fast_ops, 13, "Σ fast_ops");
1390 assert_eq!(a.counters.slow_ops, 3, "Σ slow_ops");
1391 assert_eq!(a.counters.elapsed_ns, 4000, "wall is MAX, not summed");
1392 assert_eq!(
1393 a.counters.cpu_time_ns, 1000,
1394 "cpu is SUM (disjoint per-thread)"
1395 );
1396 assert_eq!(
1397 a.serve_lat.sample_count(),
1398 3,
1399 "serve histograms union (2 + 1 samples)"
1400 );
1401 }
1402
1403 #[test]
1404 fn taobench_phase_stats_serde_roundtrips() {
1405 // TaobenchPhaseStats rides the WorkerReport postcard + CgroupStats sidecar
1406 // JSON wires; both the counters and the serve histogram must roundtrip.
1407 let mut s = TaobenchPhaseStats {
1408 counters: TaobenchStats {
1409 get_cmds: 1000,
1410 get_misses: 150,
1411 fast_ops: 850,
1412 slow_ops: 150,
1413 elapsed_ns: 9_000_000_000,
1414 cpu_time_ns: 4_200_000_000,
1415 },
1416 ..Default::default()
1417 };
1418 s.serve_lat.add_lat(12);
1419 s.serve_lat.add_lat(340);
1420 s.serve_lat.add_lat(99_999);
1421 let json = serde_json::to_string(&s).expect("TaobenchPhaseStats must serialize");
1422 let back: TaobenchPhaseStats =
1423 serde_json::from_str(&json).expect("TaobenchPhaseStats must deserialize");
1424 assert_eq!(
1425 s, back,
1426 "TaobenchPhaseStats roundtrips unchanged (counters + serve_lat)"
1427 );
1428 }
1429
1430 #[test]
1431 fn open_loop_stamps_serve_latency_on_every_completion() {
1432 // Open loop (non-zero arrival_rate) measures coordinated-omission serve
1433 // latency: every completion — a hit (served inline) AND a miss (served via
1434 // the slow path) — records exactly one sample from its intended arrival, so
1435 // the histogram count equals total_ops. An absurdly high rate keeps the
1436 // pacing in permanent overload (the per-arrival sleep collapses to zero), so
1437 // the test runs at full speed while still exercising stamping on both arms.
1438 let cfg = TaobenchConfig::default()
1439 .client_threads(4)
1440 .slow_threads(2)
1441 .cache_capacity_mib(8)
1442 .target_hit_pct(90)
1443 .slow_path_sleep_us(10)
1444 .arrival_rate(1_000_000_000);
1445 let stop = AtomicBool::new(false);
1446 let progress = AtomicU64::new(0);
1447 let out = std::thread::scope(|s| {
1448 let h = s.spawn(|| run(&cfg, &stop, &progress, None));
1449 while progress.load(Ordering::Relaxed) < 50_000 {
1450 std::hint::spin_loop();
1451 }
1452 stop.store(true, Ordering::Release);
1453 h.join().expect("engine panicked")
1454 });
1455 let total = out.whole_run.total_ops();
1456 assert!(total > 0, "engine served ops");
1457 assert!(out.whole_run.slow_ops > 0, "the miss arm is exercised");
1458 // The whole-run serve distribution is the union of the per-phase
1459 // histograms (the whole-run carrier is counters-only). Its count equals
1460 // total_ops: every completion (hit inline + miss after wait_filled)
1461 // records exactly one serve-latency sample.
1462 let mut serve = PlatStats::default();
1463 for (_epoch, p) in &out.phases {
1464 serve.combine(&p.serve_lat);
1465 }
1466 assert_eq!(
1467 serve.sample_count(),
1468 total,
1469 "every completion (hit + miss) records one serve-latency sample"
1470 );
1471 }
1472
1473 #[test]
1474 fn closed_loop_records_no_serve_latency() {
1475 // arrival_rate == 0 is closed loop: there is no intended-arrival schedule,
1476 // so serve latency is undefined and the histogram stays EMPTY (the
1477 // taobench_serve_*_us keys then read absent, distinct from a measured 0).
1478 // The heavy tail is ALSO enabled (slow_path_p99_us > slow_path_sleep_us) to
1479 // pin that enabling the tail never leaks a serve sample without an open-loop
1480 // schedule — the tail only affects fetch latency, not whether serve latency
1481 // is measured.
1482 let cfg = TaobenchConfig::default()
1483 .client_threads(4)
1484 .slow_threads(2)
1485 .cache_capacity_mib(8)
1486 .target_hit_pct(90)
1487 .slow_path_sleep_us(10)
1488 .slow_path_p99_us(1000); // tail enabled; arrival_rate defaults to 0 (closed loop)
1489 let stop = AtomicBool::new(false);
1490 let progress = AtomicU64::new(0);
1491 let out = std::thread::scope(|s| {
1492 let h = s.spawn(|| run(&cfg, &stop, &progress, None));
1493 while progress.load(Ordering::Relaxed) < 20_000 {
1494 std::hint::spin_loop();
1495 }
1496 stop.store(true, Ordering::Release);
1497 h.join().expect("engine panicked")
1498 });
1499 assert!(out.whole_run.total_ops() > 0, "ops completed");
1500 for (epoch, p) in &out.phases {
1501 assert_eq!(
1502 p.serve_lat.sample_count(),
1503 0,
1504 "closed loop records no per-phase serve latency (epoch {epoch})"
1505 );
1506 }
1507 }
1508
1509 #[test]
1510 fn open_loop_per_phase_carrier_holds_serve_latency() {
1511 // The per-phase carrier (PhaseSlice.taobench) carries the serve histogram
1512 // (the whole-run carrier is counters-only). With a single static epoch
1513 // every completion's sample lands in that epoch's bucket, so the per-phase
1514 // serve count equals the whole-run completion count (total_ops).
1515 let cfg = TaobenchConfig::default()
1516 .client_threads(4)
1517 .slow_threads(2)
1518 .cache_capacity_mib(8)
1519 .target_hit_pct(90)
1520 .slow_path_sleep_us(10)
1521 .arrival_rate(1_000_000_000);
1522 let stop = AtomicBool::new(false);
1523 let progress = AtomicU64::new(0);
1524 let epoch = AtomicU32::new(7);
1525 let out = std::thread::scope(|s| {
1526 let h = s.spawn(|| run(&cfg, &stop, &progress, Some(&epoch)));
1527 while progress.load(Ordering::Relaxed) < 30_000 {
1528 std::hint::spin_loop();
1529 }
1530 stop.store(true, Ordering::Release);
1531 h.join().expect("engine panicked")
1532 });
1533 let phase7 = out
1534 .phases
1535 .iter()
1536 .find(|(e, _)| *e == 7)
1537 .map(|(_, p)| p)
1538 .expect("epoch-7 phase carrier present");
1539 assert!(
1540 phase7.serve_lat.sample_count() > 0,
1541 "per-phase serve latency recorded"
1542 );
1543 assert_eq!(
1544 phase7.serve_lat.sample_count(),
1545 out.whole_run.total_ops(),
1546 "single-epoch run: every completion stamps one per-phase serve sample"
1547 );
1548 }
1549
1550 #[test]
1551 fn open_loop_pacing_sleep_is_bounded_for_prompt_shutdown() {
1552 // A degenerate low arrival_rate makes the inter-arrival interval ~1s, but a
1553 // client waiting for its next scheduled arrival must observe `stop` within
1554 // STOP_POLL_QUANTUM_NS (50 ms) and exit, so the join completes well under
1555 // one interval. An unbounded pacing sleep would block the join for the full
1556 // ~1s. The 500 ms bound is 10x the quantum (robust to scheduling jitter)
1557 // and half the interval (so it cleanly distinguishes bounded from
1558 // unbounded).
1559 let cfg = TaobenchConfig::default()
1560 .client_threads(1)
1561 .slow_threads(1)
1562 .cache_capacity_mib(4)
1563 .target_hit_pct(90)
1564 .slow_path_sleep_us(0)
1565 .arrival_rate(1); // interval_ns = 1e9 * 1 / 1 = 1s
1566 let stop = AtomicBool::new(false);
1567 let progress = AtomicU64::new(0);
1568 let start = std::time::Instant::now();
1569 std::thread::scope(|s| {
1570 let h = s.spawn(|| run(&cfg, &stop, &progress, None));
1571 // Let the client reach its first (long) pacing sleep, then shut down.
1572 std::thread::sleep(std::time::Duration::from_millis(20));
1573 stop.store(true, Ordering::Release);
1574 let _ = h.join().expect("engine panicked");
1575 });
1576 let elapsed = start.elapsed();
1577 assert!(
1578 elapsed < std::time::Duration::from_millis(500),
1579 "open-loop client joined in {elapsed:?}; the pacing sleep must be \
1580 bounded (interval is ~1s — an unbounded sleep would block the join)"
1581 );
1582 }
1583
1584 #[test]
1585 fn taobench_config_slow_path_p99_default_and_setter() {
1586 // Default keeps the legacy fixed-latency path (no tail); the setter sets the
1587 // p99 tail target.
1588 assert_eq!(
1589 TaobenchConfig::default().slow_path_p99_us,
1590 0,
1591 "default slow_path_p99_us is 0 (fixed latency)"
1592 );
1593 let cfg = TaobenchConfig::default().slow_path_p99_us(5000);
1594 assert_eq!(cfg.slow_path_p99_us, 5000, "setter sets slow_path_p99_us");
1595 }
1596
1597 #[test]
1598 fn taobench_service_pareto_mapping_and_legacy_gate() {
1599 // The (p50, p99) knob maps to a Pareto whose analytic quantiles round-trip:
1600 // alpha = ln(50)/ln(p99/p50); scale = p50 * 0.5^(1/alpha).
1601 let cfg = TaobenchConfig::default()
1602 .slow_path_sleep_us(100)
1603 .slow_path_p99_us(1000);
1604 let (scale, inv_neg_alpha) = cfg.resolve_service_pareto().expect("tail enabled");
1605 let alpha = -1.0 / inv_neg_alpha;
1606 assert!(
1607 (alpha - 50.0_f64.ln() / 10.0_f64.ln()).abs() < 1e-9,
1608 "alpha = ln(50)/ln(10), got {alpha}",
1609 );
1610 // The Pareto q-quantile scale*(1-q)^(-1/alpha) round-trips the config: the
1611 // median quantile is p50, the 99th percentile is p99.
1612 let median = scale * 0.5_f64.powf(-1.0 / alpha);
1613 let p99 = scale * 0.01_f64.powf(-1.0 / alpha);
1614 assert!(
1615 (median - 100.0).abs() < 1e-6,
1616 "median quantile = p50 (100), got {median}",
1617 );
1618 assert!(
1619 (p99 - 1000.0).abs() < 1e-6,
1620 "p99 quantile = configured (1000), got {p99}",
1621 );
1622 // Legacy gate: p99 == 0, p99 < p50, p99 == p50, and p50 == 0 all resolve to
1623 // the fixed path (None).
1624 let base = TaobenchConfig::default().slow_path_sleep_us(100);
1625 assert!(
1626 base.clone()
1627 .slow_path_p99_us(0)
1628 .resolve_service_pareto()
1629 .is_none(),
1630 "p99 == 0 is fixed",
1631 );
1632 assert!(
1633 base.clone()
1634 .slow_path_p99_us(50)
1635 .resolve_service_pareto()
1636 .is_none(),
1637 "p99 < p50 is fixed",
1638 );
1639 assert!(
1640 base.clone()
1641 .slow_path_p99_us(100)
1642 .resolve_service_pareto()
1643 .is_none(),
1644 "p99 == p50 is fixed",
1645 );
1646 assert!(
1647 TaobenchConfig::default()
1648 .slow_path_sleep_us(0)
1649 .slow_path_p99_us(1000)
1650 .resolve_service_pareto()
1651 .is_none(),
1652 "p50 == 0 has no Pareto scale",
1653 );
1654 }
1655
1656 #[test]
1657 fn taobench_service_sampler_matches_configured_quantiles() {
1658 // Drawing many samples from the resolved Pareto through the actual
1659 // measurement path (PlatStats) yields a histogram whose p50/p99 land near
1660 // the configured (100, 1000) µs. Deterministic (fixed seed); the bands
1661 // absorb PlatStats log-bucketing + sampling variance.
1662 use crate::workload::schbench::plat::Pct;
1663 let cfg = TaobenchConfig::default()
1664 .slow_path_sleep_us(100)
1665 .slow_path_p99_us(1000);
1666 let (scale, inv_neg_alpha) = cfg.resolve_service_pareto().unwrap();
1667 let mut rng = Rng::new(0x5A3C_1234);
1668 let mut plat = PlatStats::default();
1669 for _ in 0..200_000 {
1670 let us = sample_service_us(&mut rng, scale, inv_neg_alpha);
1671 plat.add_lat(us.min(u32::MAX as u64) as u32);
1672 }
1673 let q = plat.percentiles();
1674 let p50 = q.value_at(Pct::P50);
1675 let p99 = q.value_at(Pct::P99);
1676 assert!(
1677 (90..=115).contains(&p50),
1678 "empirical p50 {p50} ≈ configured 100µs (tight band: rejects a sampler \
1679 bit-shift / lost-scale bug, absorbs only log-bucketing + seed noise)",
1680 );
1681 assert!(
1682 (850..=1200).contains(&p99),
1683 "empirical p99 {p99} ≈ configured 1000µs (tight band: rejects a ~±50% \
1684 sampler error, absorbs only log-bucketing + seed noise)",
1685 );
1686 }
1687
1688 #[test]
1689 fn taobench_service_sampler_clamps_pathological_tail() {
1690 // A pathological draw (huge scale, so any U gives a value far above the cap)
1691 // is clamped to MAX_SERVICE_US rather than parking a dispatcher unboundedly.
1692 let mut rng = Rng::new(7);
1693 let v = sample_service_us(&mut rng, 1e12, -0.5);
1694 assert_eq!(v, MAX_SERVICE_US, "huge draw clamps to MAX_SERVICE_US");
1695 }
1696
1697 #[test]
1698 fn taobench_service_sampler_is_deterministic_from_seed() {
1699 // Same seed -> same service-time draw sequence (the determinism the engine
1700 // relies on for reproducible runs).
1701 let cfg = TaobenchConfig::default()
1702 .slow_path_sleep_us(100)
1703 .slow_path_p99_us(1000);
1704 let (scale, inv_neg_alpha) = cfg.resolve_service_pareto().unwrap();
1705 let mut a = Rng::new(42);
1706 let mut b = Rng::new(42);
1707 for _ in 0..1000 {
1708 assert_eq!(
1709 sample_service_us(&mut a, scale, inv_neg_alpha),
1710 sample_service_us(&mut b, scale, inv_neg_alpha),
1711 );
1712 }
1713 }
1714
1715 #[test]
1716 fn rng_f64_open01_is_in_open_closed_unit_interval() {
1717 // f64_open01 must yield U in (0, 1]: 0 is excluded (U^(−1/α) would be
1718 // infinite in the Pareto inverse-CDF), 1 is included (the distribution
1719 // floor). Pin the boundary contract directly over many fixed-seed draws.
1720 let mut rng = Rng::new(0xF00D_BEEF);
1721 for _ in 0..1_000_000 {
1722 let u = rng.f64_open01();
1723 assert!(
1724 u > 0.0 && u <= 1.0,
1725 "f64_open01 yielded {u}, outside (0, 1]"
1726 );
1727 }
1728 }
1729
1730 // The heavy-tail slow-path service model — the Pareto that
1731 // `resolve_service_pareto` derives from the `slow_path_sleep_us` median +
1732 // `slow_path_p99_us` 99th-percentile knobs, sampled by `sample_service_us` —
1733 // reproduces those quantiles and a heavy tail the fixed path lacks.
1734 //
1735 // Deterministic: a fixed-seed PRNG driving the pure sampler, no threads and
1736 // no wall-clock timing, so it pins the heavy-tail LOGIC without the
1737 // host-scheduling jitter a real-timing run carries (a prior version compared
1738 // wall-clock serve MAX across host threads and flaked on loaded CI runners,
1739 // where the control's jitter spike collapsed the ratio). The end-to-end
1740 // engine path — the slow path drawing from this sampler under a live
1741 // open-loop arrival process — is covered by the coordinated in-VM
1742 // `taobench_open_loop_runs_in_vm` e2e (pinned vCPUs, no host jitter).
1743 #[test]
1744 fn heavy_tail_service_model_reproduces_configured_quantiles() {
1745 // A p99 above the median resolves to a Pareto; at/below it (or a zero
1746 // median) stays on the fixed-latency path.
1747 let heavy = TaobenchConfig::default()
1748 .slow_path_sleep_us(100)
1749 .slow_path_p99_us(2000);
1750 let (scale, inv_neg_alpha) = heavy
1751 .resolve_service_pareto()
1752 .expect("p99 2000µs above median 100µs resolves to a Pareto tail");
1753 for (p50, p99, why) in [
1754 (100u64, 0u64, "p99 == 0 keeps the fixed path"),
1755 (100, 100, "p99 == median is not a tail"),
1756 (0, 2000, "zero median has no Pareto scale"),
1757 ] {
1758 assert!(
1759 TaobenchConfig::default()
1760 .slow_path_sleep_us(p50)
1761 .slow_path_p99_us(p99)
1762 .resolve_service_pareto()
1763 .is_none(),
1764 "{why}",
1765 );
1766 }
1767
1768 // Fixed-seed draws reproduce the configured quantiles + a heavy tail.
1769 let mut rng = Rng::new(0x00C0_FFEE);
1770 let mut samples: Vec<u64> = (0..200_000)
1771 .map(|_| sample_service_us(&mut rng, scale, inv_neg_alpha))
1772 .collect();
1773 samples.sort_unstable();
1774 let q = |p: f64| samples[((samples.len() as f64 * p) as usize).min(samples.len() - 1)];
1775 let (p50, p99, max) = (q(0.50), q(0.99), *samples.last().unwrap());
1776 // Median ~100µs and p99 ~2000µs (the configured knobs), within a
1777 // generous band only a broken Pareto mapping would miss.
1778 assert!(
1779 (50..=200).contains(&p50),
1780 "empirical median {p50}µs should track the configured 100µs",
1781 );
1782 assert!(
1783 (1000..=4000).contains(&p99),
1784 "empirical p99 {p99}µs should track the configured 2000µs",
1785 );
1786 assert!(
1787 max > p99.saturating_mul(2),
1788 "max {max}µs must dwarf p99 {p99}µs — the Pareto tail the fixed path lacks",
1789 );
1790 }
1791}