ktstr/test_support/entry_validate.rs
1//! `KtstrTestEntry::validate` and its phase helpers, split out of
2//! `entry.rs` to keep that file under its grandfathered file-size
3//! ceiling while the validator stays decomposed into sub-200-line
4//! phases. These are inherent methods on
5//! [`crate::test_support::KtstrTestEntry`]; its fields are `pub`, so a
6//! sibling-module impl has full access.
7
8impl crate::test_support::KtstrTestEntry {
9 /// Reject values that would boot a broken VM or leave assertions
10 /// vacuously passing. The `#[ktstr_test]` proc macro enforces the
11 /// same constraints at compile time for attribute-built entries;
12 /// this method covers directly-constructed entries (library
13 /// callers building `KtstrTestEntry` values to push into
14 /// [`KTSTR_TESTS`](crate::test_support::KTSTR_TESTS) programmatically).
15 ///
16 /// Rules:
17 /// - `name` must be non-empty (empty names collapse into each
18 /// other in nextest output and in sidecar lookups).
19 /// - `name` must not contain `/` or `\` (path separators embed in
20 /// sidecar filenames and nextest test IDs; a separator would
21 /// create a synthetic subdirectory in sidecar output and
22 /// mangle `cargo nextest run -E 'test(name)'` filtering).
23 /// - `memory_mib` must be `> 0` (a VM with zero memory cannot boot).
24 /// - `duration` must be `> 0` (a zero-duration run never exercises
25 /// the scheduler and produces no telemetry).
26 pub fn validate(&self) -> anyhow::Result<()> {
27 self.validate_basics_and_staging()?;
28 self.validate_mode_flags()?;
29 self.validate_snapshots()?;
30 self.validate_config_and_topology()?;
31 self.validate_perf_delta_assertions()?;
32 Ok(())
33 }
34
35 /// Validate name shape, memory/duration sizing, payload/host_only
36 /// device conflicts, and staged_schedulers name uniqueness.
37 fn validate_basics_and_staging(&self) -> anyhow::Result<()> {
38 if self.name.is_empty() {
39 anyhow::bail!(
40 "KtstrTestEntry.name must be non-empty (empty names \
41 collide in nextest output and sidecar lookups)"
42 );
43 }
44 if self.name.contains('/') || self.name.contains('\\') {
45 anyhow::bail!(
46 "KtstrTestEntry '{}' name must not contain path \
47 separators ('/' or '\\') — they embed in sidecar \
48 filenames and nextest test IDs, creating synthetic \
49 subdirectories in sidecar output and mangling \
50 nextest -E 'test(name)' filtering",
51 self.name,
52 );
53 }
54 if self.memory_mib == 0 {
55 anyhow::bail!(
56 "KtstrTestEntry '{}'.memory_mib must be > 0 (a VM with \
57 zero memory cannot boot)",
58 self.name,
59 );
60 }
61 if self.duration.is_zero() {
62 anyhow::bail!(
63 "KtstrTestEntry '{}'.duration must be > 0 (a zero-duration \
64 run never exercises the scheduler and produces no data \
65 for assertions)",
66 self.name,
67 );
68 }
69 if let Some(p) = self.payload
70 && p.is_scheduler()
71 {
72 anyhow::bail!(
73 "KtstrTestEntry '{}'.payload must be PayloadKind::Binary, \
74 not Scheduler-kind (schedulers belong in the `scheduler` \
75 slot; the `payload` slot is for userspace binaries \
76 composed under the scheduler)",
77 self.name,
78 );
79 }
80 if self.host_only && self.disk.is_some() {
81 anyhow::bail!(
82 "KtstrTestEntry '{}'.host_only=true with disk=Some(..) — \
83 host_only skips the VM boot that owns the virtio-blk \
84 device lifecycle, so the disk would never be attached. \
85 Drop one of host_only or disk.",
86 self.name,
87 );
88 }
89 if self.host_only && !self.networks.is_empty() {
90 anyhow::bail!(
91 "KtstrTestEntry '{}'.host_only=true with networks=[..] — \
92 host_only skips the VM boot that owns the virtio-net \
93 device lifecycle, so the NICs would never be attached. \
94 Drop one of host_only or networks.",
95 self.name,
96 );
97 }
98 // staged_schedulers names must (a) pass the per-name shape
99 // checks (non-empty, no path separators, no NUL bytes, no
100 // leading dot, not a reserved framework slot — see
101 // [`crate::test_support::staged::validate_staged_scheduler_name`])
102 // and (b) be unique within the set AND disjoint from the
103 // boot scheduler's `name`. A collision on either axis would
104 // land two distinct schedulers at the same guest path —
105 // silent overwrite, the second-staged binary clobbering the
106 // first OR shadowing a boot-time framework slot. The
107 // boot-name seed catches the "stage all the schedulers I
108 // might use" misuse (author includes the boot scheduler in
109 // the staged set thinking it's required there too). Bails
110 // here at validate time so the error surfaces ahead of any
111 // VM boot or initramfs construction.
112 let mut seen_names: std::collections::BTreeSet<&'static str> =
113 std::collections::BTreeSet::new();
114 seen_names.insert(self.scheduler.name);
115 let staged_who = format!("KtstrTestEntry '{}'.staged_schedulers", self.name);
116 for staged in self.staged_schedulers {
117 crate::test_support::staged::validate_staged_scheduler_name(&staged_who, staged.name)?;
118 if !seen_names.insert(staged.name) {
119 if staged.name == self.scheduler.name {
120 anyhow::bail!(
121 "KtstrTestEntry '{}'.staged_schedulers cannot include \
122 the boot scheduler '{}' — the boot slot already \
123 stages it. Staged entries are the ADDITIONAL \
124 candidates the test will swap TO via \
125 Op::AttachScheduler / Op::ReplaceScheduler.",
126 self.name,
127 staged.name,
128 );
129 }
130 anyhow::bail!(
131 "KtstrTestEntry '{}'.staged_schedulers has duplicate \
132 Scheduler.name '{}'; each staged scheduler must have \
133 a unique name (the name maps 1:1 to the guest-side \
134 staging path)",
135 self.name,
136 staged.name,
137 );
138 }
139 }
140 Ok(())
141 }
142
143 /// Validate host_only/scheduler, performance/no-perf, and
144 /// cpu_budget mode-flag combinations and the scx_bpf_error matcher
145 /// gate.
146 fn validate_mode_flags(&self) -> anyhow::Result<()> {
147 // Defense-in-depth for the programmatic-construction path
148 // (struct-literal `KtstrTestEntry { .. }` in integration tests,
149 // gauntlet-rewritten entries). The macro at
150 // ktstr-macros/src/lib.rs rejects `host_only = true` paired with
151 // any `scheduler = ...` attribute at compile time, but
152 // programmatic construction bypasses that gate. Match against
153 // `SchedulerSpec::Eevdf` (the value-level marker for the
154 // no-scx-scheduler placeholder) so a struct literal that sets
155 // `scheduler: &SOME_REAL_SCHED` under host_only is caught while
156 // the default `scheduler: &Scheduler::EEVDF` (whose binary is
157 // `SchedulerSpec::Eevdf`) is accepted. The variant-based check
158 // is spec-safe — unlike a pointer-identity check against
159 // `&Scheduler::EEVDF`, which depends on rustc/LLVM's const-
160 // deduplication of `&CONST_EXPR` materializations.
161 if self.host_only
162 && !matches!(
163 self.scheduler.binary,
164 crate::test_support::SchedulerSpec::Eevdf
165 )
166 {
167 anyhow::bail!(
168 "KtstrTestEntry '{}'.host_only=true with scheduler=&{:?} — \
169 host_only skips the VM boot that owns the scheduler \
170 lifecycle, so the declared scheduler would never attach. \
171 Drop one of host_only or scheduler; the host's \
172 currently-active scheduler (default EEVDF when none is \
173 loaded) runs the test under host_only.",
174 self.name,
175 self.scheduler.name,
176 );
177 }
178 if self.performance_mode && self.no_perf_mode {
179 anyhow::bail!(
180 "KtstrTestEntry '{}'.performance_mode=true with \
181 no_perf_mode=true — the two flags are contradictory \
182 (\"I want pinning\" vs. \"I explicitly don't want \
183 pinning\"). Drop one of them.",
184 self.name,
185 );
186 }
187 // `cpu_budget` of zero cannot run a VM. The builder would
188 // otherwise clamp it to 1 (builder.rs effective_cap), silently
189 // running with a budget the author never asked for. Reject
190 // explicitly — mirrors the macro's compile-time reject and the
191 // memory_mib / cleanup_budget_ms zero-rejects in
192 // validate_cross_attr.
193 if self.cpu_budget == Some(0) {
194 anyhow::bail!(
195 "KtstrTestEntry '{}'.cpu_budget=Some(0) — a zero host-CPU \
196 budget cannot run a VM. Use a positive budget, or drop \
197 cpu_budget to auto-size the no-perf mask to the vCPU count.",
198 self.name,
199 );
200 }
201 // `cpu_budget` is consulted only on the no_perf_mode path
202 // (builder.rs sizes the shared vCPU-thread mask from it). A
203 // budget set without no_perf_mode is a silent no-op — the VM
204 // runs with the default mask and the requested overcommit never
205 // happens, so a contention test would quietly run un-contended.
206 // Reject at validate time (nextest discovery) for the
207 // programmatic-construction path; ktstr-macros enforces the same
208 // gate at compile time for the `#[ktstr_test]` path.
209 if self.cpu_budget.is_some() && !self.no_perf_mode {
210 anyhow::bail!(
211 "KtstrTestEntry '{}'.cpu_budget={:?} with no_perf_mode=false \
212 — cpu_budget sizes the no-perf vCPU-thread mask and is \
213 ignored unless no_perf_mode is set (under performance_mode \
214 vCPUs are pinned 1:1). Set no_perf_mode=true or drop \
215 cpu_budget.",
216 self.name,
217 self.cpu_budget,
218 );
219 }
220 if (self.assert.expect_scx_bpf_error_contains.is_some()
221 || self.assert.expect_scx_bpf_error_matches.is_some())
222 && !self.expect_err
223 {
224 anyhow::bail!(
225 "KtstrTestEntry '{}' sets an scx_bpf_error matcher \
226 (expect_scx_bpf_error_contains or expect_scx_bpf_error_matches) \
227 without expect_err = true — a reproducer matcher narrows \
228 which failure counts as the expected bug and only \
229 applies to expected-error tests. Set expect_err = true \
230 or drop the matcher.",
231 self.name,
232 );
233 }
234 if self.survives_storm && self.expect_err {
235 anyhow::bail!(
236 "KtstrTestEntry '{}' sets both survives_storm and expect_err — \
237 contradictory: survives_storm asserts the run passes with the \
238 scheduler alive, expect_err asserts the run fails. Pick one.",
239 self.name,
240 );
241 }
242 if self.survives_storm && self.expect_auto_repro {
243 anyhow::bail!(
244 "KtstrTestEntry '{}' sets both survives_storm and \
245 expect_auto_repro — contradictory: survives_storm forces a \
246 scheduler-death failure to EXIT_FAIL, expect_auto_repro \
247 inverts a crash-with-repro failure to PASS. Pick one.",
248 self.name,
249 );
250 }
251 if self.survives_storm && !self.scheduler.has_active_scheduling() {
252 anyhow::bail!(
253 "KtstrTestEntry '{}' sets survives_storm with no active scx \
254 scheduler — the kernel default has no scheduler to eject, so \
255 survival is vacuous. Declare a scheduler = ... or drop \
256 survives_storm.",
257 self.name,
258 );
259 }
260 Ok(())
261 }
262
263 /// Validate num_snapshots against the storage cap, host_only, and
264 /// the minimum periodic-capture interval.
265 fn validate_snapshots(&self) -> anyhow::Result<()> {
266 // Periodic snapshots route through SnapshotBridge::store, which
267 // FIFO-evicts at MAX_STORED_SNAPSHOTS. Allowing num_snapshots
268 // past the cap would silently lose the earliest samples — a
269 // periodic run with N=128 today would only retain
270 // periodic_064..periodic_127 in the bridge.
271 let max = crate::scenario::snapshot::MAX_STORED_SNAPSHOTS as u32;
272 if self.num_snapshots > max {
273 anyhow::bail!(
274 "KtstrTestEntry '{}'.num_snapshots={} exceeds \
275 MAX_STORED_SNAPSHOTS={} — the bridge would FIFO-evict \
276 the earliest periodic samples. Lower the count or split \
277 into multiple test entries.",
278 self.name,
279 self.num_snapshots,
280 max,
281 );
282 }
283 if self.num_snapshots > 0 {
284 // host_only skips the VM boot that owns the freeze
285 // coordinator's run-loop. Without that loop there is no
286 // thread to stamp `scenario_start_ns`, no thread to fire
287 // `freeze_and_dispatch(FreezeMode::Capture { gate_on_exit_kind: false })` at each boundary, and no
288 // `SnapshotBridge` plumbed onto a `VmResult` for the
289 // test author to drain post-run. The combination is
290 // unsatisfiable; reject at validate time so a
291 // misconfigured entry surfaces during nextest discovery
292 // rather than as silently-empty bridge results.
293 if self.host_only {
294 anyhow::bail!(
295 "KtstrTestEntry '{}'.host_only=true with \
296 num_snapshots={} > 0 — host_only skips the VM \
297 boot that owns the freeze coordinator's \
298 periodic-capture loop, so no snapshot would \
299 ever fire. Drop one of host_only or \
300 num_snapshots.",
301 self.name,
302 self.num_snapshots,
303 );
304 }
305 // Refuse interval shorter than the minimum useful capture
306 // cadence. Each boundary fire freezes every vCPU, walks
307 // BPF maps, serialises the dump, and writes to the
308 // bridge — under the FREEZE_RENDEZVOUS_TIMEOUT (30 s)
309 // hard ceiling but commonly tens of milliseconds on a
310 // healthy guest. An interval shorter than ~100 ms would
311 // back-to-back the captures with no actual workload
312 // progress between them, defeating the periodic-sampling
313 // purpose. Compute the interval in nanoseconds in u128
314 // to avoid overflow on long durations: the formula
315 // mirrors the run-loop's
316 // `compute_periodic_boundaries_ns` (10 % pre-buffer,
317 // 80 % usable span, divided into N+1 equal intervals).
318 let usable_span_ns = self
319 .duration
320 .as_nanos()
321 .saturating_sub(2u128.saturating_mul(self.duration.as_nanos() / 10));
322 let interval_ns = usable_span_ns / (self.num_snapshots as u128 + 1);
323 const MIN_INTERVAL_NS: u128 = 100 * 1_000_000; // 100 ms
324 if interval_ns < MIN_INTERVAL_NS {
325 anyhow::bail!(
326 "KtstrTestEntry '{}'.num_snapshots={} with \
327 duration={:?} produces a periodic interval of \
328 {} ns ({} ms) — below the 100 ms minimum the \
329 freeze-and-capture path can sustain without \
330 back-to-back firing. Either reduce num_snapshots \
331 or extend duration so 0.8·duration / (N+1) >= 100 ms.",
332 self.name,
333 self.num_snapshots,
334 self.duration,
335 interval_ns,
336 interval_ns / 1_000_000,
337 );
338 }
339 }
340 Ok(())
341 }
342
343 /// Validate scheduler config_file_def/config_content pairing,
344 /// workload-slot payload kinds, and entry/scheduler topology
345 /// constraints.
346 fn validate_config_and_topology(&self) -> anyhow::Result<()> {
347 // Pair `scheduler.config_file_def` with `config_content`. The
348 // `#[ktstr_test]` macro emits a `const _: () = assert!(...)`
349 // block that catches the same mismatch at compile time for
350 // attribute-built entries; this branch covers programmatic
351 // construction (callers building `KtstrTestEntry` values
352 // directly) and surfaces the misconfiguration before VM boot
353 // rather than as a silent missing-`--config` flag.
354 let scheduler_has_def = self.scheduler.config_file_def.is_some();
355 let entry_has_content = self.config_content.is_some();
356 if scheduler_has_def && !entry_has_content {
357 anyhow::bail!(
358 "KtstrTestEntry '{}'.scheduler '{}' declares \
359 `config_file_def` but the entry does not supply \
360 `config_content`; the scheduler binary expects an \
361 inline config and would launch without `--config`. \
362 Set `config = ...` on `#[ktstr_test]` or assign \
363 `config_content` directly.",
364 self.name,
365 self.scheduler.name,
366 );
367 }
368 if !scheduler_has_def && entry_has_content {
369 anyhow::bail!(
370 "KtstrTestEntry '{}'.config_content is set but the \
371 scheduler '{}' does not declare `config_file_def`; \
372 the content would be silently dropped at dispatch. \
373 Remove `config = ...` or add \
374 `config_file_def(arg_template, guest_path)` to the \
375 scheduler.",
376 self.name,
377 self.scheduler.name,
378 );
379 }
380 // Mirror the payload-slot gate for every workload entry. The
381 // `workloads` slot is for userspace binaries composed with
382 // the primary payload under the scheduler; a scheduler-kind
383 // Payload here would be silently ignored at spawn time. The
384 // narrow typo path post-`declare_scheduler!` rollout is
385 // pasting [`Payload::KERNEL_DEFAULT`] (the only Scheduler-kind
386 // Payload still in the prelude) into a `workloads = [...]`
387 // attribute instead of the `scheduler = ...` slot.
388 for (idx, w) in self.workloads.iter().enumerate() {
389 if w.is_scheduler() {
390 anyhow::bail!(
391 "KtstrTestEntry '{}'.workloads[{idx}] (name='{}') must be \
392 PayloadKind::Binary, not Scheduler-kind (schedulers belong \
393 in the `scheduler` slot; the `workloads` slot is for \
394 userspace binaries composed under the scheduler)",
395 self.name,
396 w.name,
397 );
398 }
399 }
400 // Reject inverted topology ranges before they silently filter
401 // every gauntlet preset to zero matches. The per-entry
402 // constraints gate which gauntlet presets the test author wants
403 // to exercise; an inverted bound (e.g. min_numa_nodes=5 with
404 // max_numa_nodes=Some(2)) would yield false on every preset.
405 self.constraints
406 .validate()
407 .map_err(|e| anyhow::anyhow!("KtstrTestEntry '{}'.constraints: {e}", self.name))?;
408 // Same for the scheduler-level constraints, which apply on top
409 // of the per-entry ones. A scheduler whose declared topology
410 // requirements are themselves inverted has the same silent-
411 // filter pathology regardless of what test entries declare.
412 self.scheduler.constraints.validate().map_err(|e| {
413 anyhow::anyhow!(
414 "KtstrTestEntry '{}'.scheduler '{}'.constraints: {e}",
415 self.name,
416 self.scheduler.name
417 )
418 })?;
419 Ok(())
420 }
421
422 /// Validate declared `perf_delta_assertions`. Each assertion is a
423 /// per-test regression gate consulted only under
424 /// `perf-delta --noise-adjust` (the scalar `perf-delta` path warns that
425 /// declared gates were skipped) — inert during a normal `ktstr test` run,
426 /// where the in-VM evaluation never reads it. Two rules:
427 ///
428 /// - Every asserted `metric` must resolve in the metric registry
429 /// (`crate::stats::metric_def`). A typo'd metric name would never
430 /// match a captured field, so the gate would silently never fire:
431 /// the author believes a regression is guarded when nothing is.
432 /// Reject at validate time so the typo surfaces during nextest
433 /// discovery rather than as a phantom-passing perf-delta run.
434 /// - Declaring any assertion requires `performance_mode`. The gate
435 /// tightens the noise threshold on a metric; under no-perf mode the
436 /// vCPU threads share an oversubscribed host-CPU mask, so the
437 /// metric carries scheduling noise the tightened gate would misread
438 /// (false regressions, or a real regression masked by the narrowed
439 /// band). A perf gate is only meaningful on a pinned run. Mirrors
440 /// the `cpu_budget ⇒ no_perf_mode` flag-dependency above.
441 fn validate_perf_delta_assertions(&self) -> anyhow::Result<()> {
442 if self.perf_delta_assertions.is_empty() {
443 return Ok(());
444 }
445 if !self.performance_mode {
446 anyhow::bail!(
447 "KtstrTestEntry '{}' declares perf_delta_assertions but \
448 performance_mode=false — a declared regression gate \
449 tightens the noise threshold on a metric, which is only \
450 meaningful on a pinned (performance_mode) run. Under \
451 no-perf mode the metric carries host-CPU-oversubscription \
452 noise the gate would misread. Set performance_mode=true \
453 or drop the assertions.",
454 self.name,
455 );
456 }
457 let mut seen: std::collections::BTreeSet<(&'static str, Option<u16>)> =
458 std::collections::BTreeSet::new();
459 for a in self.perf_delta_assertions {
460 let metric = a.metric();
461 if crate::stats::metric_def(metric).is_none() {
462 anyhow::bail!(
463 "KtstrTestEntry '{}'.perf_delta_assertions references \
464 unknown metric '{}' — it does not resolve in the \
465 metric registry, so the gate would never match a \
466 captured field and silently never fire. Check the \
467 name against the registry (src/stats/metric.rs \
468 METRICS).",
469 self.name,
470 metric,
471 );
472 }
473 // A render-suppressed rate COMPONENT (the internal numerator/
474 // denominator of a derived rate — e.g. total_phase_iterations)
475 // keeps its registry slot for the cross-run re-pool but is never
476 // emitted as a compare finding, so a gate on it can NEVER fire: a
477 // guaranteed-dead gate that would pass this check on name alone.
478 // Reject so the author declares the user-facing rate (e.g.
479 // iteration_rate) rather than its suppressed component.
480 if crate::stats::is_render_suppressed_component(metric) {
481 anyhow::bail!(
482 "KtstrTestEntry '{}'.perf_delta_assertions references \
483 render-suppressed rate component '{}' — it is an internal \
484 numerator/denominator that never surfaces as a compare \
485 finding, so the gate could never fire. Declare the \
486 user-facing rate metric instead.",
487 self.name,
488 metric,
489 );
490 }
491 // A non-finite or negative threshold silently inverts or disables
492 // the gate in classify_noise (abs_gate/rel_gate are compared with
493 // `>=`; NaN makes every comparison false, a negative floor makes
494 // every move material). Reject both overrides.
495 if let Some(pct) = a.max_regression_pct()
496 && (!pct.is_finite() || pct < 0.0)
497 {
498 anyhow::bail!(
499 "KtstrTestEntry '{}'.perf_delta_assertions gate on '{}' has \
500 max_regression_pct={} — must be finite and >= 0 (a negative \
501 or NaN threshold silently disables or inverts the gate).",
502 self.name,
503 metric,
504 pct,
505 );
506 }
507 if let Some(min) = a.min_abs()
508 && (!min.is_finite() || min < 0.0)
509 {
510 anyhow::bail!(
511 "KtstrTestEntry '{}'.perf_delta_assertions gate on '{}' has \
512 min_abs={} — must be finite and >= 0 (a negative or NaN floor \
513 silently disables or inverts the gate).",
514 self.name,
515 metric,
516 min,
517 );
518 }
519 // Duplicate (metric, phase): the compare consults declared gates via
520 // a first-match `.find()`, so a second gate on the same (metric,
521 // phase) is silently dropped. Reject rather than pick-one-silently.
522 if !seen.insert((metric, a.phase())) {
523 let scope = match a.phase() {
524 Some(k) => format!("phase {k}"),
525 None => "the aggregate value".to_string(),
526 };
527 anyhow::bail!(
528 "KtstrTestEntry '{}'.perf_delta_assertions declares more than \
529 one gate on metric '{}' for {} — the compare applies only the \
530 first and silently drops the rest. Merge them into one gate.",
531 self.name,
532 metric,
533 scope,
534 );
535 }
536 }
537 Ok(())
538 }
539}