ktstr/cli/kernel_build/
build.rs

1//! Top-level kernel build orchestration.
2//!
3//! Holds [`kernel_build_pipeline`] (the post-acquisition orchestrator
4//! that runs `clean` → configure → build → validate → cache-store),
5//! the two-phase reservation acquisition
6//! ([`acquire_build_reservation`]) for LLC flock + cgroup sandbox +
7//! `make -jN` hint, and the source-tree flock helper
8//! ([`acquire_source_tree_lock`]) that serializes parallel builds
9//! against the same on-disk source tree.
10
11use std::path::Path;
12
13use anyhow::{Context, Result};
14
15use super::super::kernel_cmd::{
16    DIRTY_TREE_CACHE_SKIP_HINT, EMBEDDED_KCONFIG, NON_GIT_TREE_CACHE_SKIP_HINT,
17    embedded_kconfig_hash,
18};
19use super::super::util::{success, warn};
20use super::kconfig::{
21    all_fragment_lines_present, configure_kernel, validate_kernel_config,
22    warn_dropped_extra_kconfig_lines, warn_extra_kconfig_overrides_baked_in,
23};
24use super::make::{build_make_args, make_kernel_with_output, run_make, run_make_with_output};
25
26/// Result of the post-acquisition kernel build pipeline.
27///
28/// Returned by [`kernel_build_pipeline`] so callers can inspect
29/// the cache entry and built image path.
30#[non_exhaustive]
31pub struct KernelBuildResult {
32    /// Cache entry, if the build was cached. `None` for dirty trees
33    /// or when cache store fails.
34    pub entry: Option<crate::cache::CacheEntry>,
35    /// Path to the built kernel image.
36    pub image_path: std::path::PathBuf,
37    /// Whether the source tree was dirty as observed by the build
38    /// pipeline. `true` if either the acquire-time inspection
39    /// reported dirty OR the post-build re-check observed a
40    /// mid-build mutation (worktree edit, branch flip, mid-build
41    /// commit). The downstream label decoration in cargo-ktstr's
42    /// `resolve_one` uses this to append `_dirty` so a
43    /// non-reproducible run is distinguishable from a clean rebuild
44    /// of the same path.
45    pub post_build_is_dirty: bool,
46}
47
48/// Two-phase build reservation handles (LLC flock plan + cgroup v2
49/// sandbox + make -jN hint). Consumed by
50/// [`kernel_build_pipeline`]; the factored-out
51/// [`acquire_build_reservation`] builds it from `cpu_cap` without
52/// depending on kernel source, enabling integration tests that
53/// exercise the reservation logic against synthetic topologies.
54///
55/// Also acquired directly by `cargo-ktstr`'s harness-compile path
56/// (`bin/cargo_ktstr/run_cargo.rs`) so a colocated runner's
57/// `cargo build`/`nextest --no-run` warm-up coordinates against the
58/// SAME machine-global LLC flock namespace + cgroup cpuset as a
59/// kernel build — hence `pub` (the bin holds this as an opaque RAII
60/// guard, dropping it before the test-running phase). Fields stay
61/// `pub(crate)`: external holders only need the RAII lifetime and the
62/// [`BuildReservation::make_jobs`] accessor, not field access.
63///
64/// Drop order is load-bearing: `_sandbox` is declared first and
65/// drops first per Rust's declaration-order field-drop rule;
66/// this ensures the cgroup sandbox is removed before the LLC
67/// flock is released. Otherwise a peer could observe the LLC
68/// released before the cgroup is gone and mint a conflicting
69/// plan.
70#[derive(Debug)]
71pub struct BuildReservation {
72    /// cgroup v2 sandbox. `None` when `plan` is `None` (no reservation
73    /// to enforce). Drops FIRST per struct field order — cgroup
74    /// rmdir runs while LLC flocks are still held. `_` prefix
75    /// keeps the binding alive through Drop but marks it as
76    /// not-read — the RAII invariant IS the read.
77    pub(crate) _sandbox: Option<crate::vmm::cgroup_sandbox::BuildSandbox>,
78    /// LLC plan (flock fds + cpus + mems). `None` under
79    /// `KTSTR_BYPASS_LLC_LOCKS=1` or sysfs-unreadable host without
80    /// `--cpu-cap`. Drops SECOND per struct field order —
81    /// flocks release AFTER the sandbox rmdir lands.
82    pub(crate) plan: Option<crate::vmm::host_topology::LlcPlan>,
83    /// `make -jN` parallelism hint. `Some(N)` under an active
84    /// `plan`; `None` when no reservation exists (caller falls
85    /// back to `nproc`).
86    pub(crate) make_jobs: Option<usize>,
87}
88
89impl BuildReservation {
90    /// Reserved-CPU parallelism hint (`plan.cpus.len()`), or `None`
91    /// when no reservation was acquired (bypass / degraded sysfs).
92    /// The kernel-build path feeds it to `make -jN`; the cargo
93    /// harness-compile path feeds it to `CARGO_BUILD_JOBS` so parallel
94    /// rustc is capped to the reserved cpuset instead of exploding to
95    /// `nproc` inside a confined cgroup. `None` leaves the child build
96    /// on its own default.
97    pub fn make_jobs(&self) -> Option<usize> {
98        self.make_jobs
99    }
100}
101
102/// Acquire the two-phase reservation (LLC flocks + cgroup sandbox)
103/// for a kernel OR harness build. Factored out of
104/// [`kernel_build_pipeline`] so integration tests can exercise the
105/// cpu_cap → acquire → sandbox → make_jobs decision tree without
106/// requiring a real kernel source tree, and so `cargo-ktstr`'s
107/// harness-compile warm-up (`bin/cargo_ktstr/run_cargo.rs`) can take
108/// the identical reservation before a colocated runner's `cargo`
109/// build invades a perf-mode LLC reservation. `pub` for that
110/// cross-binary reuse; the escape hatches (`KTSTR_BYPASS_LLC_LOCKS`,
111/// `KTSTR_CARGO_TEST_MODE` inside `acquire_llc_plan`), degraded-sysfs
112/// warn/hard-error gating, and cgroup-degrade semantics are shared
113/// verbatim between both callers.
114///
115/// Returns a `BuildReservation` whose fields are the three values
116/// `kernel_build_pipeline` used to bind inline. `_sandbox` is
117/// declared first and drops first per Rust's declaration-order
118/// field-drop rule; this ensures the cgroup sandbox is removed
119/// before the LLC flock is released.
120///
121/// `cli_label` prefixes operator-facing error text.
122///
123/// `cpu_cap` is the resolved CPU-count cap from
124/// [`CpuCap::resolve`](crate::vmm::host_topology::CpuCap::resolve);
125/// `None` means "reserve 30% of the calling process's allowed-CPU
126/// set", applied inside the planner at acquire time.
127pub fn acquire_build_reservation(
128    cli_label: &str,
129    cpu_cap: Option<crate::vmm::host_topology::CpuCap>,
130) -> Result<BuildReservation> {
131    acquire_build_reservation_impl(cli_label, cpu_cap, false, None)
132}
133
134/// Acquire a build reservation, joining the progress-aware host queue when
135/// perf-mode work temporarily owns too much LLC capacity.
136///
137/// Harness prebuilds use this variant because many independent CI invocations
138/// compile concurrently and have no source-tree lock to serialize them. The
139/// resolved CPU budget is a ceiling: this path immediately takes the largest
140/// non-empty placement that does not overlap an exact VM reservation, and
141/// derives Cargo/make parallelism from that actual placement. It joins the
142/// queue only while every compatible CPU is occupied, then repeats that
143/// largest-fit plan on each wake. Direct kernel builds
144/// ([`acquire_build_reservation`]) instead hold a fixed exact width, but they
145/// too wait work-conservingly on that width through the same host queue rather
146/// than giving up the lane under transient contention.
147pub fn acquire_build_reservation_waiting(
148    cli_label: &str,
149    cpu_cap: Option<crate::vmm::host_topology::CpuCap>,
150) -> Result<BuildReservation> {
151    acquire_build_reservation_impl(cli_label, cpu_cap, true, None)
152}
153
154/// Cancellation-aware harness-build reservation.
155///
156/// A signal-published `cancelled` flag interrupts either the ticket's futex
157/// wait or the coordinator's inotify wait, releasing the current elastic
158/// registry claim, partial LLC holds, and any completed plan before returning
159/// [`std::io::ErrorKind::Interrupted`]. Sizing otherwise matches
160/// [`acquire_build_reservation_waiting`].
161pub fn acquire_build_reservation_waiting_interruptible(
162    cli_label: &str,
163    cpu_cap: Option<crate::vmm::host_topology::CpuCap>,
164    cancelled: &std::sync::atomic::AtomicBool,
165) -> Result<BuildReservation> {
166    acquire_build_reservation_impl(cli_label, cpu_cap, true, Some(cancelled))
167}
168
169/// Cancellation-aware harness-build reservation with synchronous wait ticks.
170///
171/// `progress` runs on the acquiring thread while an ordinary admission ticket is
172/// parked or its elected coordinator is blocked in inotify. The protocol only
173/// slices those blocking syscalls; reservation ordering, retry deadlines, and
174/// liveness deadlines are unchanged. Sizing otherwise matches
175/// [`acquire_build_reservation_waiting`].
176pub fn acquire_build_reservation_waiting_interruptible_with_progress(
177    cli_label: &str,
178    cpu_cap: Option<crate::vmm::host_topology::CpuCap>,
179    cancelled: &std::sync::atomic::AtomicBool,
180    progress: impl FnMut() + 'static,
181) -> Result<BuildReservation> {
182    crate::vmm::host_topology::with_reservation_wait_progress(progress, || {
183        acquire_build_reservation_impl(cli_label, cpu_cap, true, Some(cancelled))
184    })
185}
186
187fn acquire_build_reservation_impl(
188    cli_label: &str,
189    cpu_cap: Option<crate::vmm::host_topology::CpuCap>,
190    elastic: bool,
191    cancelled: Option<&std::sync::atomic::AtomicBool>,
192) -> Result<BuildReservation> {
193    check_reservation_cancelled(cancelled)?;
194    let bypass = crate::bypass_llc_locks_active();
195    // INVARIANT: `_sandbox` is declared first and drops first per
196    // Rust's declaration-order field-drop rule; this ensures the
197    // cgroup sandbox is removed before the LLC flock is released.
198    // Reordering either would either
199    // (a) unlock LLCs while the sandbox still enforces the
200    // cpuset — a concurrent peer could claim the LLC and stomp
201    // gcc children that haven't exited — or (b) leave the cgroup
202    // hierarchy non-empty when its parent tries to rmdir.
203    let plan: Option<crate::vmm::host_topology::LlcPlan> = if bypass {
204        if cpu_cap.is_some() {
205            anyhow::bail!(
206                "{cli_label}: --cpu-cap conflicts with KTSTR_BYPASS_LLC_LOCKS=1; \
207                 unset one of them. --cpu-cap is a resource contract; bypass \
208                 disables the contract entirely."
209            );
210        }
211        None
212    } else if let Ok(host_topo) = crate::vmm::host_topology::HostTopology::from_sysfs() {
213        let test_topo = crate::topology::TestTopology::from_system()?;
214        // Concurrent harness/scheduler prebuilds decouple cache footprint from
215        // CPU choice. They consolidate among LLCs carrying the effective
216        // free-first CPU set so whole cache domains remain available for
217        // hard-exclusive performance runs, then select least-held CPUs inside
218        // that footprint instead of converging on one busy CPU prefix. Their
219        // elastic width consumes currently unshared capacity first and retains
220        // cooperative SH overlap as the fully-occupied fallback.
221        //
222        // A direct kernel build keeps its fixed Consolidate contract. It is
223        // non-elastic and deliberately packs one bounded compile at its exact
224        // width rather than participating in the independently fanned-out
225        // harness-build storm. It waits work-conservingly for that width: the
226        // reservation is taken here BEFORE the source-tree flock below, so the
227        // wait holds no other lock, and it routes through the same event-driven
228        // registry admission that run claims use (`register_ticket_or_acquire`
229        // via the wait phase). Under contention it joins the host queue and
230        // re-plans on each coordinator wake instead of giving up the lane after
231        // a few TOCTOU retries — dead holders are pruned by the registry's
232        // liveness, and it never abandons a plan while live holders progress.
233        let acquired_plan = if elastic {
234            crate::vmm::host_topology::acquire_elastic_build_llc_plan(
235                &host_topo, &test_topo, cpu_cap, cancelled,
236            )?
237        } else {
238            crate::vmm::host_topology::acquire_build_llc_plan(
239                &host_topo, &test_topo, cpu_cap, true, cancelled,
240            )?
241        };
242        check_reservation_cancelled(cancelled)?;
243        crate::vmm::host_topology::warn_if_cross_node_spill(&acquired_plan, &host_topo);
244        Some(acquired_plan)
245    } else {
246        if cpu_cap.is_some() {
247            anyhow::bail!(
248                "{cli_label}: --cpu-cap set but host LLC topology unreadable \
249                 from sysfs — cannot enforce the resource budget. Run on a \
250                 host with /sys/devices/system/cpu populated, or drop \
251                 --cpu-cap to build without enforcement."
252            );
253        }
254        tracing::warn!(
255            "{cli_label}: could not read host LLC topology from sysfs; \
256             skipping kernel-build LLC reservation. Concurrent perf-mode \
257             runs on this host will NOT be serialized against this build"
258        );
259        None
260    };
261
262    // Phase 2: cgroup v2 sandbox that enforces cpu+mem binding on
263    // make/gcc children. `hard_error_on_degrade` is driven by
264    // whether `--cpu-cap` was set explicitly: degradation is fatal
265    // under the flag (the flag promises enforcement), and warn-only
266    // when the 30%-of-allowed default was expanded (the default
267    // contract is best-effort — a parent cgroup narrowing the
268    // reservation should not fail the build).
269    let sandbox: Option<crate::vmm::cgroup_sandbox::BuildSandbox> = match plan.as_ref() {
270        Some(p) => Some(crate::vmm::cgroup_sandbox::BuildSandbox::try_create(
271            &p.cpus,
272            &p.mems,
273            cpu_cap.is_some(),
274        )?),
275        None => None,
276    };
277    check_reservation_cancelled(cancelled)?;
278
279    // `make -jN` parallelism hint. `N` = `plan.cpus.len()` via
280    // `make_jobs_for_plan` — the reserved CPU count, whether that
281    // came from an explicit `--cpu-cap N` or the 30%-of-allowed
282    // default. See `make_kernel_with_output` for the resolution.
283    let make_jobs = plan
284        .as_ref()
285        .map(crate::vmm::host_topology::make_jobs_for_plan);
286
287    Ok(BuildReservation {
288        plan,
289        _sandbox: sandbox,
290        make_jobs,
291    })
292}
293
294fn check_reservation_cancelled(cancelled: Option<&std::sync::atomic::AtomicBool>) -> Result<()> {
295    if cancelled.is_some_and(|flag| flag.load(std::sync::atomic::Ordering::Acquire)) {
296        Err(std::io::Error::new(
297            std::io::ErrorKind::Interrupted,
298            "ktstr build-reservation acquisition interrupted",
299        )
300        .into())
301    } else {
302        Ok(())
303    }
304}
305
306/// Acquire an exclusive flock on a per-source-canonical-path lockfile
307/// so two concurrent `cargo ktstr test --kernel <path>` runs against
308/// the SAME source tree don't race in `make` (defconfig vs
309/// olddefconfig vs compile_commands.json) and stomp each other's
310/// `.config` and build artifacts.
311///
312/// The lockfile lives at
313/// `{KTSTR_CACHE_DIR}/.locks/source-{path_hash}.lock` where
314/// `{path_hash}` is the full 8-char CRC32 hex of the canonical
315/// source-path bytes (same shape and helper the
316/// `local-unknown-{path_hash}` cache key uses, see
317/// [`crate::fetch::canonical_path_hash`] /
318/// [`crate::fetch::compose_local_cache_key`]) — one per-tree
319/// identifier ties the source-tree flock to the cache key it gates.
320///
321/// Lockfile placement piggybacks on the cache root's `.locks/`
322/// subdirectory ([`crate::flock::LOCK_DIR_NAME`]) so source-tree
323/// flocks share the same filesystem-residency story as cache-entry
324/// flocks: never under `/tmp`, where `tmpwatch` (or the equivalent
325/// `systemd-tmpfiles` cleanup) can sweep stale-mtime files out from
326/// under an active flock holder. flock(2) does NOT update the
327/// inode's mtime, so a /tmp-resident lockfile would be a candidate
328/// for sweep on every run, with the resulting `unlink(2)` racing
329/// any peer trying to `open(2)` the same path. The `.locks/`
330/// directory under the user-controlled cache root is exempt from
331/// those sweeps.
332///
333/// Try-then-wait: attempts a non-blocking acquire first. If
334/// contended, reports the wait and falls through to a blocking
335/// acquire that parks until the peer releases. The wait path does
336/// not scan host-global `/proc/locks`: on a storm host that scan can
337/// itself take seconds, delaying the actual flock wait without
338/// affecting correctness. When the blocking acquire returns, the
339/// peer's build is done and the cache likely contains the artifact —
340/// the caller checks the cache after we return and skips the build
341/// if the slot is populated.
342///
343/// Distinct from the cache-entry flock acquired inside
344/// [`crate::cache::CacheDir::store`]: that lock serializes the
345/// atomic install of an artifact bundle into a cache slot; this
346/// lock serializes the BUILD itself against the source-tree
347/// `make` invocations.
348pub(crate) fn acquire_source_tree_lock(
349    canonical: &Path,
350    cli_label: &str,
351) -> Result<std::os::fd::OwnedFd> {
352    acquire_source_tree_lock_with_wait_hook(canonical, cli_label, || {})
353}
354
355fn acquire_source_tree_lock_with_wait_hook(
356    canonical: &Path,
357    cli_label: &str,
358    on_wait: impl FnOnce(),
359) -> Result<std::os::fd::OwnedFd> {
360    use anyhow::Context;
361
362    // Share the per-path CRC32 with `local-unknown-{hash}` cache
363    // keys so a single per-tree identifier ties the source-tree
364    // flock to the cache slot it gates.
365    let path_hash = crate::fetch::canonical_path_hash(canonical);
366    let cache = crate::cache::CacheDir::new()
367        .with_context(|| "open cache root for source-tree lockfile placement")?;
368    cache
369        .ensure_lock_dir()
370        .with_context(|| "create cache `.locks/` subdir for source-tree lock")?;
371    let lock_path = cache.lock_path(&format!("source-{path_hash}"));
372
373    match crate::flock::try_flock(&lock_path, crate::flock::FlockMode::Exclusive)
374        .with_context(|| format!("acquire source-tree flock {}", lock_path.display()))?
375    {
376        Some(fd) => Ok(fd),
377        None => {
378            // Non-blocking acquire failed (EWOULDBLOCK) — a live
379            // peer holds the lock. Publish the transition before
380            // entering the blocking syscall; production uses a no-op
381            // hook while the test seam observes it without polling
382            // host-global `/proc/locks`.
383            on_wait();
384            crate::ktstr_status!(
385                "{cli_label}: source tree {} is locked by a concurrent ktstr \
386                 build — waiting for it to finish.",
387                canonical.display(),
388            );
389            crate::flock::block_flock(&lock_path, crate::flock::FlockMode::Exclusive).with_context(
390                || format!("blocking wait on source-tree flock {}", lock_path.display()),
391            )
392        }
393    }
394}
395
396/// Classification of source-tree state at the post-acquire
397/// re-probe site inside [`kernel_build_pipeline`].
398///
399/// The pipeline re-probes the source tree after the source-tree EX
400/// wait completes so a mid-wait mutation (operator edit, branch flip,
401/// commit on top) can invalidate the cache-skip short-circuit instead
402/// of returning a cache slot keyed on the pre-wait identity. The
403/// 5-variant split keeps cause-attribution honest in the operator
404/// diagnostic emitted by [`MidWaitState::diagnostic`]: a `git commit`
405/// during the wait is not "your edits"; an operator who started dirty
406/// did not dirty the tree because of the wait; a probe failure is
407/// not a confirmed mutation, just unknowable state.
408#[derive(Debug, PartialEq, Eq)]
409enum MidWaitState {
410    /// Source tree unchanged across the wait (or non-local source
411    /// where the wait has no source-tree implication). The pipeline
412    /// proceeds to the cache_lookup short-circuit.
413    Clean,
414    /// Operator started with a dirty tree BEFORE the source-tree
415    /// EX wait was taken. The wait was not the cause of the dirty
416    /// state, so the diagnostic is silent (returns `None`) to avoid
417    /// fabricating wait-related attribution.
418    PreAcquireDirty,
419    /// Operator edited a tracked file DURING the wait (acquire-time
420    /// probe was clean, post-wait probe is dirty). Forces a rebuild
421    /// and emits a "your local edits" diagnostic.
422    DirtyEdit,
423    /// Operator advanced HEAD (commit / branch flip) during the wait
424    /// (acquire-time short-hash differs from post-wait short-hash;
425    /// post-wait worktree is clean). Forces a rebuild and emits a
426    /// "HEAD advanced" diagnostic.
427    HashAdvanced,
428    /// Post-wait probe returned `Err` (corrupt git state, removed
429    /// source dir, or a gix internal error). Forces a conservative
430    /// rebuild — unknowable state cannot be assumed Clean.
431    ProbeFailed,
432}
433
434impl MidWaitState {
435    /// Operator-facing diagnostic body (without the `{cli_label}: `
436    /// prefix — caller composes via `crate::ktstr_status!("{cli_label}: {body}")`).
437    ///
438    /// Returns `None` for [`Self::Clean`] (the cache-skip gate emits
439    /// its own message) and [`Self::PreAcquireDirty`] (the wait was
440    /// not the cause of the dirty state, so a wait-related diagnostic
441    /// would fabricate attribution).
442    fn diagnostic(&self) -> Option<&'static str> {
443        match self {
444            Self::DirtyEdit => Some(
445                "source tree changed during peer's build wait \
446                 — rebuilding to capture your local edits",
447            ),
448            Self::HashAdvanced => Some(
449                "source HEAD advanced during peer's build wait \
450                 — rebuilding for the new commit",
451            ),
452            Self::ProbeFailed => Some(
453                "source-tree dirty re-check failed during peer's \
454                 build wait — rebuilding conservatively (re-run with \
455                 RUST_LOG=warn for the probe error)",
456            ),
457            Self::Clean | Self::PreAcquireDirty => None,
458        }
459    }
460}
461
462/// Operator-facing diagnostic body emitted when the post-mid-wait
463/// `cache_lookup` short-circuit fires (without the `{cli_label}: `
464/// prefix — caller composes via `crate::ktstr_status!("{cli_label}: {body}")`,
465/// matching the [`MidWaitState::diagnostic`] convention).
466///
467/// Separate from [`MidWaitState::diagnostic`] because the cache-hit
468/// is downstream of the variant classification — the message fires
469/// only when all three of `mid_wait_clean`, a populated cache slot,
470/// and an extant image file align. Tying the message to a single
471/// MidWaitState variant would misrepresent that conjunction.
472fn cache_hit_diagnostic(cache_key: &str) -> String {
473    format!(
474        "concurrent ktstr build populated cache slot {cache_key} during \
475         peer's build wait — skipping redundant rebuild"
476    )
477}
478
479/// Post-acquisition kernel build pipeline.
480///
481/// Handles: clean, configure, build, validate config, generate
482/// compile_commands.json for local trees, find image, strip vmlinux,
483/// compute metadata, cache store, and remote cache store (when
484/// enabled). Callers handle source acquisition.
485///
486/// `cli_label` prefixes diagnostic status output (e.g. `"ktstr"` or
487/// `"cargo ktstr"`).
488///
489/// `force` mirrors `kernel build --force`: when set, the post-acquire
490/// cache re-check below does NOT short-circuit on a peer-populated slot.
491/// A forced build must actually rebuild — the peer-dedup skip only
492/// spares a redundant NON-forced rebuild. (The caller already does its
493/// own pre-acquire `!force` cache lookup; this gates the second,
494/// post-source-lock lookup the caller cannot reach.)
495///
496/// `is_local_source` should be true when the source is a local
497/// kernel source tree, regardless of how the caller arrived there
498/// (`kernel build --kernel <path>`, `cargo ktstr test --kernel <path>`,
499/// or any other Path-spec entry that funnels through
500/// [`super::super::resolve_kernel_dir`] /
501/// [`super::super::resolve_kernel_dir_to_entry`]). It controls the
502/// mrproper warning and `source_tree_path` in metadata.
503///
504/// `extra_kconfig` is an optional user-supplied kconfig fragment
505/// merged on top of [`EMBEDDED_KCONFIG`] before `configure_kernel`
506/// (which runs olddefconfig only when new lines are needed).
507/// `Some(content)` appends the fragment AFTER the baked-in fragment
508/// so kbuild's last-occurrence-wins semantics
509/// (`scripts/kconfig/confdata.c::conf_read_simple`) make user values
510/// override baked-in ones on conflict, and forces a re-configure pass
511/// even when `.config` already carries `CONFIG_SCHED_CLASS_EXT=y`
512/// (the user fragment may add or invert symbols the baked-in pass
513/// alone wouldn't have produced).
514///
515/// Two metadata fields capture the build inputs separately:
516/// - `ktstr_kconfig_hash` always holds the bare baked-in hash
517///   (`crate::kconfig_hash()` of `EMBEDDED_KCONFIG`) so
518///   `KconfigStatus::Matches/Stale/Untracked` keeps comparing
519///   against the live baked-in fragment.
520/// - `extra_kconfig_hash` holds `Some(crate::extra_kconfig_hash(content))`
521///   when extras were supplied, `None` otherwise. Drives the
522///   `(extra kconfig)` tag in `kernel list`.
523///
524/// Callers that don't expose `--extra-kconfig` (test/coverage/
525/// shell/verifier) pass `None`.
526#[allow(clippy::too_many_arguments)]
527pub fn kernel_build_pipeline(
528    acquired: &crate::fetch::AcquiredSource,
529    cache: &crate::cache::CacheDir,
530    cli_label: &str,
531    clean: bool,
532    force: bool,
533    is_local_source: bool,
534    cpu_cap: Option<crate::vmm::host_topology::CpuCap>,
535    extra_kconfig: Option<&str>,
536    progress: Option<&crate::cli::FetchProgress>,
537) -> Result<KernelBuildResult> {
538    let source_dir = &acquired.source_dir;
539    let (arch, image_name) = crate::fetch::arch_info();
540
541    // Bind a guaranteed-live progress group for the build phase: the
542    // caller's group (the parallel resolve's shared group, or a
543    // single-shot caller's local one), or a fresh local group when the
544    // caller passed none. The build phase renders through this group and
545    // NEVER through a standalone `Spinner`, so concurrent builds in the
546    // parallel resolve cannot race the process-global `SPINNER_ACTIVE`
547    // guard. Off-TTY the group is hidden and inert.
548    let owned_group;
549    let progress = match progress {
550        Some(p) => p,
551        None => {
552            owned_group = crate::cli::FetchProgress::new();
553            &owned_group
554        }
555    };
556
557    // Two-phase reservation. A concurrent perf-mode test run must
558    // not have its measured CPUs stomped by a `make -j$(nproc)`
559    // explosion of gcc children, and vice-versa a concurrent
560    // kernel build must not have its compile window extended by
561    // a test pinning RT-FIFO on shared cores. Phase 1 of the
562    // reservation is the LLC-level flock from
563    // [`acquire_llc_plan`]: whole-LLC flocks whose count is
564    // chosen to cover the CPU budget (either an explicit
565    // `--cpu-cap N` or the 30%-of-allowed default). Phase 2 is
566    // the cgroup v2 sandbox from
567    // [`BuildSandbox::try_create`] that binds make/gcc's
568    // cpu+mem sets to the plan's CPUs + NUMA nodes so the
569    // parallelism hint is enforced, not just advisory.
570    //
571    // Binding order is load-bearing: `_sandbox` is declared first
572    // and drops first per Rust's declaration-order field-drop rule,
573    // which migrates the build pid out of the cgroup and rmdirs the
574    // child while the LLC flocks are still held. Otherwise a peer
575    // could observe the LLC released before the cgroup is gone,
576    // mint a new plan against the same LLCs, and see an orphan
577    // cgroup lingering for up to the 24h sweep window.
578    //
579    // Escape hatches:
580    //   - `KTSTR_BYPASS_LLC_LOCKS=1`: skip the LLC plan+flock
581    //     acquisition entirely; the build proceeds immediately
582    //     without coordinating with any concurrent perf-mode run.
583    //     Use when the operator explicitly accepts measurement
584    //     noise (one shell doing unrelated work, an isolated
585    //     developer workstation, or a CI queue that already
586    //     serializes jobs at a higher layer). Mutually exclusive
587    //     with `--cpu-cap` at CLI parse time — see the CLI
588    //     binaries' pre-dispatch conflict check.
589    //   - Sysfs-unreadable host (non-Linux, degraded container):
590    //     `HostTopology::from_sysfs()` returns `Err`. Without
591    //     `--cpu-cap`, we emit a `tracing::warn!` and proceed
592    //     without locks. With `--cpu-cap`, the flag cannot be
593    //     honoured and we fail hard — cpu_cap is a contract, not
594    //     a hint: a silent degrade would let a build exceed the
595    //     declared resource budget without surfacing.
596    // `_plan` + `_sandbox` are kept alive via RAII — their Drops
597    // release the LLC flocks and cgroup on scope exit. Struct
598    // field order in BuildReservation ensures `_sandbox` drops
599    // BEFORE `plan`, per Rust's declaration-order field-drop rule.
600    let BuildReservation {
601        plan: _plan,
602        _sandbox,
603        make_jobs,
604    } = acquire_build_reservation(cli_label, cpu_cap)?;
605
606    // Source-tree flock for local sources. Two parallel
607    // `cargo ktstr test --kernel ./linux` runs would otherwise race
608    // in `make` against the same source tree (e.g. one's
609    // `make defconfig` racing with another's `make compile_commands.json`)
610    // and produce inconsistent .config / build artifacts. The flock is
611    // taken on the SOURCE TREE itself (per canonical path), distinct from
612    // the cache-entry flock acquired inside `cache.store` (per cache key).
613    // The two are complementary: the source-tree flock serializes the
614    // build phase; the cache-entry flock serializes the atomic install.
615    //
616    // Held via `OwnedFd` for the lifetime of `_source_lock` — drops at
617    // end of pipeline. Skipped under `KTSTR_BYPASS_LLC_LOCKS` to share
618    // the operator's escape hatch with the LLC-flock bypass; that
619    // env var already declares "I accept noise from concurrent runs."
620    //
621    // `acquire_source_tree_lock` does a non-blocking `try_flock`
622    // first; on EWOULDBLOCK it reports the source path and then parks
623    // in a blocking `flock(LOCK_EX)` until the holder releases. It
624    // deliberately avoids a host-global `/proc/locks` attribution
625    // scan before parking. The wait is intentional: when the peer's
626    // build finishes, the cache slot is likely populated and the
627    // post-acquire cache check below short-circuits the redundant
628    // rebuild. The pre-wait `eprintln!` ensures the operator sees
629    // what they're waiting on rather than a silent stall.
630    let _source_lock = if is_local_source && !crate::bypass_llc_locks_active() {
631        Some(acquire_source_tree_lock(source_dir, cli_label)?)
632    } else {
633        None
634    };
635
636    // Post-acquire cache re-check. N peers racing on a cold cache all
637    // queue on the source-tree EX above. When the first peer's build
638    // completes and releases, the cache slot is populated — every
639    // subsequent peer should observe the hit and skip a redundant
640    // rebuild rather than serially repeat the same work. The
641    // pre-acquire `cache_lookup` in `resolve_kernel_dir_to_entry`
642    // catches the warm-cache case (no lock taken at all); this check
643    // catches the cold-then-warmed-during-wait case.
644    let mid_wait_state = compute_mid_wait_state(acquired, source_dir, is_local_source, cli_label);
645    let mid_wait_clean = mid_wait_state == MidWaitState::Clean;
646
647    if let Some(body) = mid_wait_state.diagnostic() {
648        crate::ktstr_status!("{cli_label}: {body}");
649    }
650
651    if !force
652        && mid_wait_clean
653        && let Some(entry) =
654            crate::cli::resolve::cache_lookup(cache, &acquired.cache_key, cli_label)
655        && entry.image_path().exists()
656    {
657        crate::ktstr_status!("{cli_label}: {}", cache_hit_diagnostic(&acquired.cache_key));
658        let image_path = entry.image_path();
659        return Ok(KernelBuildResult {
660            entry: Some(entry),
661            image_path,
662            post_build_is_dirty: false,
663        });
664    }
665
666    if clean {
667        if !is_local_source {
668            crate::ktstr_status!(
669                "{cli_label}: --clean is only meaningful with a --kernel <path> source (downloaded/cloned sources start clean)"
670            );
671        } else {
672            crate::ktstr_status!("{cli_label}: make mrproper");
673            run_make(source_dir, &["mrproper"])?;
674        }
675    }
676
677    reconfigure_and_build(source_dir, extra_kconfig, cli_label, make_jobs, progress)?;
678
679    // Validate critical config options were not silently disabled.
680    // When `--extra-kconfig` is set, attach an actionable hint
681    // pointing at the user fragment as a likely cause. The most
682    // plausible failure mode is a user override that disables a
683    // baked-in invariant (e.g. a fragment containing
684    // `# CONFIG_BPF is not set` defeats the BPF dep chain), so
685    // name `--extra-kconfig` in the wrap context.
686    validate_kernel_config(source_dir).with_context(|| {
687        if extra_kconfig.is_some() {
688            "post-build kernel config validation failed; check that your \
689             --extra-kconfig fragment does not disable a CONFIG_X required by \
690             ktstr (e.g. CONFIG_BPF, CONFIG_DEBUG_INFO_BTF, CONFIG_FTRACE, \
691             CONFIG_SCHED_CLASS_EXT)"
692                .to_string()
693        } else {
694            "post-build kernel config validation failed".to_string()
695        }
696    })?;
697
698    if !acquired.is_temp {
699        generate_compile_commands(source_dir, progress)?;
700    }
701
702    let (image_path, vmlinux_opt) = find_built_image(source_dir, cli_label)?;
703    let vmlinux_ref = vmlinux_opt.as_deref();
704
705    // Cache (skip for dirty local trees).
706    if acquired.is_dirty {
707        crate::ktstr_status!("{cli_label}: kernel built at {}", image_path.display());
708        // Branch the hint wording: commit/stash is only an actionable
709        // remediation for an actual git repo. A non-git source tree
710        // is force-marked dirty (see `acquire_local_source` in
711        // `fetch.rs`) because dirty detection is impossible, and
712        // telling the operator to "commit or stash" leads nowhere.
713        let hint = dirty_cache_skip_hint(acquired.is_git);
714        crate::ktstr_status!("{cli_label}: {hint}");
715        return Ok(KernelBuildResult {
716            entry: None,
717            image_path,
718            post_build_is_dirty: true,
719        });
720    }
721
722    if let Some(skip_result) = post_build_dirty_skip(
723        acquired,
724        source_dir,
725        is_local_source,
726        &image_path,
727        cli_label,
728    ) {
729        return Ok(skip_result);
730    }
731
732    build_metadata_and_store(
733        acquired,
734        cache,
735        cli_label,
736        is_local_source,
737        arch,
738        image_name,
739        extra_kconfig,
740        source_dir,
741        image_path,
742        vmlinux_ref,
743    )
744}
745
746/// Classify the source tree at the post-acquire re-probe site.
747///
748/// Mid-wait edit guard: the operator may edit a tracked file in
749/// the source tree DURING our EX wait (long peer build = long
750/// window). `acquired.is_dirty` snapshots clean-at-acquire; a fresh
751/// probe via `inspect_local_source_state` catches edits that landed
752/// during the wait. If dirty/hash-changed, the operator's intent
753/// is "build what's on disk" — skip the cache re-check and fall
754/// through to the build branch, where the post-build dirty re-check
755/// at the cache-store site will recognise the mutation and skip
756/// caching. Probe errors are warnings (not fatal) — same Err
757/// disposition as the post-build re-check.
758/// PreAcquireDirty distinguishes "operator started with a dirty
759/// tree" (the wait wasn't the cause) from "operator dirtied the
760/// tree during the wait" (DirtyEdit). The split keeps the enum
761/// variants honest about cause-attribution per the
762/// [`MidWaitState::diagnostic`] dispatch in [`kernel_build_pipeline`].
763///
764/// TOCTOU acceptance: the source tree can mutate between this
765/// probe and the `cache_lookup` call in the caller — a microsecond
766/// window (typically) where an operator edit, background
767/// autoformatter write, `git commit`, or IDE pre-commit hook would
768/// slip through both this guard AND the post-build dirty re-check at
769/// the cache-store site (the cache-hit return in the caller
770/// short-circuits before `make`, so the post-build re-check never
771/// runs for the racing-into-hit path). Publication-side staleness is
772/// gated by the separate post-build dirty re-check at the cache-store
773/// site; this paragraph is about the consumer-side stale-serving
774/// window only. The cache slot is keyed on `acquired.cache_key`
775/// (frozen at acquire time inside `local_source`), so the served
776/// artifact's identity is the acquire-time HEAD — a mid-window
777/// mutation produces a cache hit that serves a slightly-stale
778/// source state without destroying the operator's later state.
779/// Bounded race; the operator's next invocation re-acquires and
780/// observes the new state.
781///
782/// "Next invocation correct" is NOT a remediation for: single-shot
783/// CI pipelines without retry, `git bisect run` invocations (each
784/// commit is independent), or pipelined CI flows where one job
785/// builds the kernel and a downstream job consumes the cached
786/// image without re-probing the source tree. Operators in those
787/// workflows should treat cache hits as acquire-time-correct, not
788/// invocation-time correct. Holding an EX flock across [probe,
789/// cache_lookup] or re-probing after the lookup were considered
790/// and rejected as adding common-path latency for a microsecond-
791/// wide window.
792fn compute_mid_wait_state(
793    acquired: &crate::fetch::AcquiredSource,
794    source_dir: &Path,
795    is_local_source: bool,
796    cli_label: &str,
797) -> MidWaitState {
798    if is_local_source && !acquired.is_dirty {
799        match crate::fetch::inspect_local_source_state(source_dir) {
800            Ok(post) => {
801                let hash_changed = post.short_hash
802                    != acquired
803                        .kernel_source
804                        .as_local_git_hash()
805                        .map(str::to_string);
806                if post.is_dirty {
807                    MidWaitState::DirtyEdit
808                } else if hash_changed {
809                    MidWaitState::HashAdvanced
810                } else {
811                    MidWaitState::Clean
812                }
813            }
814            Err(e) => {
815                tracing::warn!(
816                    cli_label = cli_label,
817                    err = %format!("{e:#}"),
818                    "mid-wait dirty re-check failed; proceeding to build",
819                );
820                MidWaitState::ProbeFailed
821            }
822        }
823    } else if acquired.is_dirty {
824        MidWaitState::PreAcquireDirty
825    } else {
826        MidWaitState::Clean
827    }
828}
829
830/// Merge the kconfig fragments, reconfigure when stale, then build.
831///
832/// Builds the merged fragment ONCE so the configure call observes
833/// the byte layout `{EMBEDDED_KCONFIG}\n{extra}` (with a `\n`
834/// interleave) defined in [`crate::merge_kconfig_fragments`]. The
835/// helper returns a `Cow<'_, str>` so the no-extras path borrows
836/// `EMBEDDED_KCONFIG` without allocating; only the user-fragment
837/// case heaps the merged string. Unit tests pin the exact
838/// ordering kbuild's last-wins rule operates on.
839///
840/// Reconfigures when any merged-fragment line is missing from the
841/// current `.config`. The prior `has_sched_ext` probe was a proxy for
842/// "configured" — but a stale `.config` from an earlier build can carry
843/// sched_ext while MISSING a changed baked-in value (e.g. an edited
844/// CONFIG_NR_CPUS in ktstr.kconfig) or every user `--extra-kconfig` line,
845/// silently ignoring the edit. `all_fragment_lines_present` checks the
846/// actual merged fragment (exact-line) instead, so an edited baked-in
847/// symbol or a user extra both trigger the merged configure.
848fn reconfigure_and_build(
849    source_dir: &Path,
850    extra_kconfig: Option<&str>,
851    cli_label: &str,
852    make_jobs: Option<usize>,
853    progress: &crate::cli::FetchProgress,
854) -> Result<()> {
855    let merged_fragment = crate::merge_kconfig_fragments(EMBEDDED_KCONFIG, extra_kconfig);
856
857    // Surface a `tracing::warn!` for each user fragment line that
858    // overrides a baked-in symbol from `EMBEDDED_KCONFIG`. The build
859    // proceeds with the user value winning (last-wins is the design
860    // intent) — the warning lets the operator see they are shadowing
861    // a baked-in setting before configure_kernel (which runs
862    // olddefconfig only when new lines are needed), which is when
863    // an over-aggressive override can still be addressed by editing
864    // the fragment. A separate post-build `validate_kernel_config`
865    // pass catches critical-baked-in disablement (e.g. CONFIG_BPF).
866    if let Some(extra) = extra_kconfig {
867        warn_extra_kconfig_overrides_baked_in(extra, cli_label);
868    }
869
870    let config_now = std::fs::read_to_string(source_dir.join(".config")).unwrap_or_default();
871    let needs_configure =
872        extra_kconfig.is_some() || !all_fragment_lines_present(&merged_fragment, &config_now);
873    if needs_configure {
874        let bar = progress.step_bar("Configuring kernel...");
875        let configure_result = configure_kernel(source_dir, &merged_fragment, Some(progress));
876        bar.finish();
877        // Wrap configure errors with `--extra-kconfig` context when
878        // extras are present so the user can pinpoint which input is
879        // responsible for an olddefconfig failure (e.g. a malformed
880        // `CONFIG_X=` line in their fragment).
881        configure_result.with_context(|| {
882            if extra_kconfig.is_some() {
883                "kernel configure failed (with --extra-kconfig fragment merged on top of \
884                 baked-in ktstr.kconfig); check the fragment for syntax errors or \
885                 conflicting symbol declarations"
886                    .to_string()
887            } else {
888                "kernel configure failed".to_string()
889            }
890        })?;
891
892        // Post-olddefconfig validation — warn (not error) when a
893        // user-requested option from `--extra-kconfig` did not
894        // survive into the final `.config` (typically because
895        // olddefconfig dropped it for an unmet dependency). Emits
896        // one `tracing::warn!` per dropped line naming the
897        // requested setting and the actual final value.
898        // The hard-fail "user override killed a baked-in invariant"
899        // case (e.g. user disabled `CONFIG_BPF`) is caught at
900        // `validate_kernel_config` post-build with extra context.
901        if let Some(extra) = extra_kconfig {
902            warn_dropped_extra_kconfig_lines(source_dir, extra, cli_label);
903        }
904    }
905
906    let bar = progress.step_bar("Building kernel...");
907    let build_result = make_kernel_with_output(source_dir, Some(progress), make_jobs);
908    bar.finish();
909    build_result?;
910    Ok(())
911}
912
913/// Generate `compile_commands.json` for local trees (LSP support).
914///
915/// The args MUST match `make_kernel_with_output`'s
916/// (`-jN`, `KCFLAGS=-Wno-error`) — otherwise make's "command line
917/// flag changed" detection invalidates the build's object cache
918/// and recompiles every translation unit single-threaded under
919/// the compile_commands.json rule.  Build the same `-jN +
920/// KCFLAGS=-Wno-error` prefix via `build_make_args`, then append
921/// the target.
922fn generate_compile_commands(
923    source_dir: &Path,
924    progress: &crate::cli::FetchProgress,
925) -> Result<()> {
926    let nproc = std::thread::available_parallelism()
927        .map(|n| n.get())
928        .unwrap_or(1);
929    let mut cc_args = build_make_args(nproc);
930    cc_args.push("compile_commands.json".into());
931    let cc_arg_refs: Vec<&str> = cc_args.iter().map(|s| s.as_str()).collect();
932    let bar = progress.step_bar("Generating compile_commands.json...");
933    let cc_result = run_make_with_output(source_dir, &cc_arg_refs, Some(progress));
934    bar.finish();
935    cc_result?;
936    Ok(())
937}
938
939/// Find the built kernel image and vmlinux. Returns `(image_path,
940/// vmlinux)` where `vmlinux` is `Some` only when `<source_dir>/vmlinux`
941/// exists; emits the operator-facing caching/missing diagnostic.
942fn find_built_image(
943    source_dir: &Path,
944    cli_label: &str,
945) -> Result<(std::path::PathBuf, Option<std::path::PathBuf>)> {
946    let image_path = crate::kernel_path::find_image_in_dir(source_dir)
947        .ok_or_else(|| anyhow::anyhow!("no kernel image found in {}", source_dir.display()))?;
948    let vmlinux_path = source_dir.join("vmlinux");
949    let vmlinux_opt = if vmlinux_path.exists() {
950        let orig_mib = std::fs::metadata(&vmlinux_path)
951            .map(|m| m.len() as f64 / (1024.0 * 1024.0))
952            .unwrap_or(0.0);
953        crate::ktstr_status!("{cli_label}: caching vmlinux ({orig_mib:.0} MiB, will be stripped)");
954        Some(vmlinux_path)
955    } else {
956        crate::ktstr_status!("{cli_label}: warning: vmlinux not found, BTF will not be cached");
957        None
958    };
959    Ok((image_path, vmlinux_opt))
960}
961
962/// Post-build dirty re-check. Returns `Some(result)` when the cache
963/// store must be skipped because the source tree changed during the
964/// build; `None` (proceed to store) otherwise.
965///
966/// `local_source` captures `is_dirty` ONCE at acquire time. The
967/// operator may then edit a tracked file (`.config` mutation, source
968/// patch) DURING the build window. The acquire-time `is_dirty=false`
969/// would say "safe to cache" but the on-disk content actually built
970/// differs from the HEAD commit recorded in the cache key — a future
971/// cache hit on that key would serve a build that no longer matches
972/// its identity. Re-running the same gix probes catches the race. On
973/// any change (dirty flip OR HEAD-hash shift from a concurrent
974/// commit), skip the cache store and emit a one-liner explaining why
975/// the cache slot was passed over.
976///
977/// Errors from the re-check are surfaced as a warning rather than a
978/// hard fail — the build itself succeeded; refusing to store on a
979/// re-check probe failure would penalize an otherwise-clean run for a
980/// transient gix glitch. The cache store proceeds with the original
981/// key, on the same pessimistic basis as a tree the re-check could not
982/// classify.
983fn post_build_dirty_skip(
984    acquired: &crate::fetch::AcquiredSource,
985    source_dir: &Path,
986    is_local_source: bool,
987    image_path: &Path,
988    cli_label: &str,
989) -> Option<KernelBuildResult> {
990    if is_local_source {
991        match crate::fetch::inspect_local_source_state(source_dir) {
992            Ok(post) => {
993                let (skip, hash_changed) =
994                    post_build_cache_store_skip(&post, acquired.kernel_source.as_local_git_hash());
995                if skip {
996                    crate::ktstr_status!(
997                        "{cli_label}: source tree changed during build \
998                         (acquire-time dirty={}, post-build dirty={}; \
999                         hash_changed={hash_changed}); skipping cache store \
1000                         to avoid recording a stale identity. Re-run after \
1001                         the working tree settles to populate the cache.",
1002                        acquired.is_dirty,
1003                        post.is_dirty,
1004                    );
1005                    return Some(KernelBuildResult {
1006                        entry: None,
1007                        image_path: image_path.to_path_buf(),
1008                        // Mid-build mutation flips the run's
1009                        // reproducibility — the cache key recorded at
1010                        // acquire time no longer identifies the actual
1011                        // build input. Mirror that into the outcome so
1012                        // the kernel-label downstream gets the
1013                        // `_dirty` suffix.
1014                        post_build_is_dirty: true,
1015                    });
1016                }
1017            }
1018            Err(e) => {
1019                tracing::warn!(
1020                    cli_label = cli_label,
1021                    err = %format!("{e:#}"),
1022                    "post-build dirty re-check failed; proceeding to cache store",
1023                );
1024            }
1025        }
1026    }
1027    None
1028}
1029
1030/// Build the kernel metadata + artifact set, store to cache, and
1031/// return the clean (non-dirty) [`KernelBuildResult`].
1032///
1033/// `too_many_arguments` allow: the cache-store tail threads the
1034/// acquire-time inputs (`acquired`, `arch`, `image_name`,
1035/// `extra_kconfig`, `is_local_source`) plus the build outputs
1036/// (`source_dir`, `image_path`, `vmlinux_ref`) the metadata records;
1037/// bundling them into a struct would add indirection without
1038/// changing the data flow.
1039#[allow(clippy::too_many_arguments)]
1040fn build_metadata_and_store(
1041    acquired: &crate::fetch::AcquiredSource,
1042    cache: &crate::cache::CacheDir,
1043    cli_label: &str,
1044    is_local_source: bool,
1045    arch: &str,
1046    image_name: &str,
1047    extra_kconfig: Option<&str>,
1048    source_dir: &Path,
1049    image_path: std::path::PathBuf,
1050    vmlinux_ref: Option<&Path>,
1051) -> Result<KernelBuildResult> {
1052    let config_hash = config_hash_for(source_dir)?;
1053
1054    // Two-segment metadata: the bare baked-in hash stays in
1055    // `ktstr_kconfig_hash` so `kernel list`'s matches/stale/
1056    // untracked verdict (see `CacheEntry::kconfig_status`) keeps
1057    // comparing against the live `EMBEDDED_KCONFIG`, and the user
1058    // extras hash lives in its own slot. Matches the cache-key
1059    // suffix shape `kc{baked}-xkc{extra}` produced by
1060    // [`crate::cache_key_suffix_with_extra`].
1061    let kconfig_hash = embedded_kconfig_hash();
1062    let extra_kconfig_hash_value = extra_kconfig.map(crate::extra_kconfig_hash);
1063
1064    // Source-tree vmlinux stat (size + mtime seconds) so a later
1065    // `prefer_source_tree_for_dwarf` lookup can detect a user
1066    // rebuild between cache store and DWARF read. Only meaningful
1067    // for local sources whose vmlinux survived the build —
1068    // `vmlinux_ref` is `None` if vmlinux wasn't found, in which
1069    // case there's nothing to stat. mtime read is best-effort:
1070    // failure leaves the validation pair `None` and prefers the
1071    // pre-validation behavior for this entry.
1072    let source_vmlinux_stat = source_vmlinux_stat_for(vmlinux_ref);
1073
1074    let mut metadata = crate::cache::KernelMetadata::new(
1075        acquired.kernel_source.clone(),
1076        arch,
1077        image_name,
1078        crate::test_support::now_iso8601(),
1079    )
1080    .with_ktstr_kconfig_hash(kconfig_hash);
1081    if let Some(v) = acquired.version.clone() {
1082        metadata = metadata.with_version(v);
1083    }
1084    if let Some(h) = config_hash {
1085        metadata = metadata.with_config_hash(h);
1086    }
1087    if let Some(h) = extra_kconfig_hash_value {
1088        metadata = metadata.with_extra_kconfig_hash(h);
1089    }
1090    if is_local_source && let Some((size, mtime_secs)) = source_vmlinux_stat {
1091        metadata = metadata.with_source_vmlinux_stat(size, mtime_secs);
1092    }
1093
1094    let mut artifacts = crate::cache::CacheArtifacts::new(&image_path);
1095    if let Some(v) = vmlinux_ref {
1096        artifacts = artifacts.with_vmlinux(v);
1097    }
1098    let entry = match cache.store(&acquired.cache_key, &artifacts, &metadata) {
1099        Ok(entry) => {
1100            success(&format!("\u{2713} Kernel cached: {}", acquired.cache_key));
1101            crate::ktstr_status!("{cli_label}: image: {}", entry.image_path().display());
1102            if crate::remote_cache::is_enabled() {
1103                crate::remote_cache::remote_store(&entry, cli_label);
1104            }
1105            Some(entry)
1106        }
1107        Err(e) => {
1108            warn(&format!("{cli_label}: cache store failed: {e:#}"));
1109            None
1110        }
1111    };
1112
1113    Ok(KernelBuildResult {
1114        entry,
1115        image_path,
1116        post_build_is_dirty: false,
1117    })
1118}
1119
1120/// CRC32 of `<source_dir>/.config` rendered as a fixed 8-hex-digit
1121/// lowercase string, or `None` when no `.config` exists. The
1122/// zero-padded `{:08x}` width is the cache-key suffix contract — a
1123/// `{:x}` (no pad) would drop a leading-zero nibble and silently
1124/// re-key every cached build, defeating cache hits. Pulled out of
1125/// [`kernel_build_pipeline`] so the derivation is unit-testable
1126/// without a real `make`.
1127pub(crate) fn config_hash_for(source_dir: &Path) -> std::io::Result<Option<String>> {
1128    let config_path = source_dir.join(".config");
1129    if config_path.exists() {
1130        let data = std::fs::read(&config_path)?;
1131        Ok(Some(format!("{:08x}", crc32fast::hash(&data))))
1132    } else {
1133        Ok(None)
1134    }
1135}
1136
1137/// `(size, mtime_secs)` of the source-tree vmlinux, or `None` when the
1138/// path is absent or unstattable. `mtime_secs` is signed seconds since
1139/// the epoch — a pre-epoch mtime (clock skew) maps to a negative
1140/// count rather than dropping the stat. The `None` short-circuit on a
1141/// missing vmlinux keeps a later `prefer_source_tree_for_dwarf`
1142/// staleness check from comparing against a phantom `(0, _)`. Pulled
1143/// out of [`kernel_build_pipeline`] for unit-testability.
1144pub(crate) fn source_vmlinux_stat_for(vmlinux_ref: Option<&Path>) -> Option<(u64, i64)> {
1145    let v = vmlinux_ref?;
1146    let stat = std::fs::metadata(v).ok()?;
1147    let mtime_secs = stat.modified().ok().and_then(|t| {
1148        t.duration_since(std::time::UNIX_EPOCH)
1149            .map(|d| d.as_secs() as i64)
1150            .ok()
1151            .or_else(|| {
1152                std::time::UNIX_EPOCH
1153                    .duration_since(t)
1154                    .ok()
1155                    .map(|d| -(d.as_secs() as i64))
1156            })
1157    })?;
1158    Some((stat.len(), mtime_secs))
1159}
1160
1161/// The cache-skip hint wording for a dirty local source tree. A git
1162/// repo can be remediated by commit/stash; a non-git tree is
1163/// force-marked dirty (dirty detection is impossible) so commit/stash
1164/// advice is unactionable — it gets the put-under-git wording
1165/// instead. Pulled out of [`kernel_build_pipeline`] for
1166/// unit-testability.
1167pub(crate) fn dirty_cache_skip_hint(is_git: bool) -> &'static str {
1168    if is_git {
1169        DIRTY_TREE_CACHE_SKIP_HINT
1170    } else {
1171        NON_GIT_TREE_CACHE_SKIP_HINT
1172    }
1173}
1174
1175/// The post-build cache-store-skip decision. After `make` returns,
1176/// `inspect_local_source_state` is re-run; the store is skipped when
1177/// the worktree went dirty during the build OR HEAD advanced (a
1178/// concurrent commit), because the acquire-time cache key no longer
1179/// identifies the built input. Returns `(skip, hash_changed)` so the
1180/// caller both decides and reports `hash_changed` in the skip message.
1181/// `acquired_local_git_hash` is the acquire-time hash from the kernel
1182/// source's `as_local_git_hash`. Pulled out of
1183/// [`kernel_build_pipeline`] for unit-testability.
1184pub(crate) fn post_build_cache_store_skip(
1185    post: &crate::fetch::LocalSourceState,
1186    acquired_local_git_hash: Option<&str>,
1187) -> (bool, bool) {
1188    let hash_changed = post.short_hash != acquired_local_git_hash.map(str::to_string);
1189    (post.is_dirty || hash_changed, hash_changed)
1190}
1191
1192#[cfg(test)]
1193mod tests {
1194    use super::super::super::kernel_cmd::KernelCommand;
1195    use super::*;
1196
1197    /// Returns `false` when `git` is not on `PATH`. Tests that drive
1198    /// a real git repo in a tempdir call this first and `return` early
1199    /// when git is unavailable so CI without git silently skips
1200    /// instead of failing on a hard-error.
1201    fn git_available() -> bool {
1202        std::process::Command::new("git")
1203            .arg("--version")
1204            .output()
1205            .is_ok()
1206    }
1207
1208    /// Runs `git` in `canonical` with the sandboxed env that the
1209    /// mid-wait tests share — neutralizes `~/.gitconfig` and
1210    /// `/etc/gitconfig` (so a CI host's git identity can't pollute
1211    /// the test repo) and pins author/committer identity so `commit`
1212    /// succeeds without depending on host config. Asserts the command
1213    /// exited successfully; failure surfaces stderr in the panic
1214    /// message.
1215    fn run_git(canonical: &Path, args: &[&str]) {
1216        let out = std::process::Command::new("git")
1217            .args(args)
1218            .current_dir(canonical)
1219            .env("GIT_CONFIG_GLOBAL", "/dev/null")
1220            .env("GIT_CONFIG_SYSTEM", "/dev/null")
1221            .env("GIT_AUTHOR_NAME", "ktstr-test")
1222            .env("GIT_AUTHOR_EMAIL", "ktstr-test@localhost")
1223            .env("GIT_COMMITTER_NAME", "ktstr-test")
1224            .env("GIT_COMMITTER_EMAIL", "ktstr-test@localhost")
1225            .output()
1226            .expect("git");
1227        assert!(
1228            out.status.success(),
1229            "git {args:?} failed: {}",
1230            String::from_utf8_lossy(&out.stderr)
1231        );
1232    }
1233
1234    /// Pins the post-acquire cache re-check at `kernel_build_pipeline`
1235    /// (the early-return path that fires when a peer publishes the
1236    /// cache slot during our source-tree EX wait).
1237    ///
1238    /// The early-return gate is 3-pronged: `!acquired.is_dirty` AND
1239    /// `cache_lookup(...).is_some()` AND `entry.image_path().exists()`.
1240    /// A regression that drops any prong (e.g. someone "simplifies"
1241    /// out the exists check) would let stale-manifest entries slip
1242    /// through and the runtime would crash later on a phantom image.
1243    ///
1244    /// Single-thread, deterministic — the "after EX wait" semantic
1245    /// reduces to "after the lookup, observe the planted state."
1246    /// Real thread orchestration is covered by
1247    /// `acquire_source_tree_lock_blocks_on_contention_then_succeeds`
1248    /// elsewhere in this module.
1249    #[test]
1250    fn cache_lookup_observes_peer_published_entry_after_ex_wait() {
1251        let _env_lock = crate::test_support::test_helpers::lock_env();
1252        let cache_tmp = tempfile::TempDir::new().expect("cache tempdir");
1253        let _cache_env = crate::test_support::test_helpers::EnvVarGuard::set(
1254            crate::KTSTR_CACHE_DIR_ENV,
1255            cache_tmp.path(),
1256        );
1257        let cache = crate::cache::CacheDir::with_root(cache_tmp.path().to_path_buf());
1258        let cache_key = "test-cache-key-7f8a9b";
1259
1260        // Plant a cache entry via `CacheDir::store` (the production
1261        // helper). Going through `store` rather than hand-writing
1262        // metadata.json keeps the test honest against schema drift.
1263        let (arch, image_name) = crate::fetch::arch_info();
1264        let staging = tempfile::TempDir::new().expect("staging tempdir");
1265        let fake_image = staging.path().join(image_name);
1266        std::fs::write(&fake_image, b"fake kernel image bytes").expect("write fake image");
1267        let metadata = crate::cache::KernelMetadata::new(
1268            crate::cache::KernelSource::Local {
1269                source_tree_path: None,
1270                git_hash: None,
1271            },
1272            arch,
1273            image_name,
1274            "2026-04-12T10:00:00Z",
1275        );
1276        let artifacts = crate::cache::CacheArtifacts::new(&fake_image);
1277        cache
1278            .store(cache_key, &artifacts, &metadata)
1279            .expect("plant cache entry");
1280
1281        // Exercise the 3-condition gate. `cache_lookup` is the same
1282        // helper `kernel_build_pipeline` calls at the post-acquire
1283        // re-check; `image_path().exists()` is the second gate; the
1284        // `is_dirty` gate is upstream (this test assumes a clean
1285        // source by construction since `acquired.is_dirty` is the
1286        // caller's responsibility).
1287        let entry = crate::cli::resolve::cache_lookup(&cache, cache_key, "test")
1288            .expect("cache_lookup must surface the planted entry");
1289        assert!(
1290            entry.image_path().exists(),
1291            "image_path existence check must hold for the planted entry",
1292        );
1293        assert_eq!(entry.metadata.built_at, "2026-04-12T10:00:00Z");
1294    }
1295
1296    /// Pins the HashAdvanced branch of [`MidWaitState`] classification
1297    /// at `kernel_build_pipeline` — operator advanced HEAD
1298    /// (`git commit`/`checkout`) during the peer's build wait, leaving
1299    /// the worktree clean but the short_hash bumped.
1300    ///
1301    /// Failure mode pinned: a future "simplification" that drops the
1302    /// `hash_changed` check and trusts only `post.is_dirty` would
1303    /// silently accept a cache slot keyed on the pre-commit hash even
1304    /// though the operator committed (clean post-state) on top during
1305    /// the wait. The served cache slot would correspond to an older
1306    /// HEAD than the operator's current source tree.
1307    #[test]
1308    fn mid_wait_hash_change_invalidates_cache_hit_skip() {
1309        if !git_available() {
1310            eprintln!(
1311                "mid_wait_hash_change_invalidates_cache_hit_skip: \
1312                 git unavailable, skipping"
1313            );
1314            return;
1315        }
1316
1317        let tmp = tempfile::TempDir::new().unwrap();
1318        let canonical = tmp.path().to_path_buf();
1319        run_git(&canonical, &["init", "-q", "-b", "main"]);
1320        std::fs::write(canonical.join("seed.txt"), "initial").unwrap();
1321        run_git(&canonical, &["add", "seed.txt"]);
1322        run_git(&canonical, &["commit", "-q", "-m", "initial"]);
1323
1324        let pre = crate::fetch::inspect_local_source_state(&canonical).expect("acquire-time probe");
1325        let acquired_hash = pre
1326            .short_hash
1327            .clone()
1328            .expect("clean repo must carry a short_hash");
1329
1330        // Mid-wait commit — different from the acquire-time hash.
1331        std::fs::write(canonical.join("file.txt"), "amended mid-wait").unwrap();
1332        run_git(&canonical, &["add", "file.txt"]);
1333        run_git(&canonical, &["commit", "-q", "-m", "mid-wait commit"]);
1334
1335        let post = crate::fetch::inspect_local_source_state(&canonical).expect("post-wait probe");
1336
1337        assert!(
1338            !post.is_dirty,
1339            "committed changes leave the worktree clean; the hash \
1340             change is what must invalidate the cache hit (not is_dirty)",
1341        );
1342        assert!(
1343            post.short_hash.is_some(),
1344            "clean post-wait state must carry a short_hash",
1345        );
1346        assert_ne!(
1347            post.short_hash.as_ref(),
1348            Some(&acquired_hash),
1349            "the new commit must yield a different short_hash than the \
1350             acquire-time hash",
1351        );
1352
1353        // Mirror the production ternary in `kernel_build_pipeline`'s
1354        // mid_wait_state classification.
1355        let hash_changed = post.short_hash != Some(acquired_hash);
1356        let state = if post.is_dirty {
1357            MidWaitState::DirtyEdit
1358        } else if hash_changed {
1359            MidWaitState::HashAdvanced
1360        } else {
1361            MidWaitState::Clean
1362        };
1363        assert_eq!(
1364            state,
1365            MidWaitState::HashAdvanced,
1366            "clean worktree + advanced HEAD must classify as HashAdvanced",
1367        );
1368        assert!(
1369            state != MidWaitState::Clean,
1370            "hash_changed=true must falsify mid_wait_clean, forcing a \
1371             rebuild for the new cache key",
1372        );
1373    }
1374
1375    /// Pins the Clean branch of [`MidWaitState`] classification at
1376    /// `kernel_build_pipeline` — the positive path where a peer's
1377    /// build wait completes with the source tree unchanged and the
1378    /// `cache_lookup` short-circuit fires.
1379    ///
1380    /// Failure mode pinned: a future refactor that flips the
1381    /// `if post.is_dirty` / `else if hash_changed` order, or one that
1382    /// inverts a `!is_dirty` check, would route a no-mutation
1383    /// post-wait probe into DirtyEdit or HashAdvanced and force a
1384    /// redundant rebuild every time. This test ensures the no-op
1385    /// path keeps returning [`MidWaitState::Clean`] so the cache
1386    /// short-circuit at the consumer site remains reachable.
1387    #[test]
1388    fn mid_wait_clean_path_allows_cache_hit_skip() {
1389        if !git_available() {
1390            eprintln!(
1391                "mid_wait_clean_path_allows_cache_hit_skip: \
1392                 git unavailable, skipping"
1393            );
1394            return;
1395        }
1396
1397        let tmp = tempfile::TempDir::new().unwrap();
1398        let canonical = tmp.path().to_path_buf();
1399        run_git(&canonical, &["init", "-q", "-b", "main"]);
1400        std::fs::write(canonical.join("seed.txt"), "initial").unwrap();
1401        run_git(&canonical, &["add", "seed.txt"]);
1402        run_git(&canonical, &["commit", "-q", "-m", "initial"]);
1403
1404        let pre = crate::fetch::inspect_local_source_state(&canonical).expect("acquire-time probe");
1405        let acquired_hash = pre
1406            .short_hash
1407            .clone()
1408            .expect("clean repo must carry a short_hash");
1409
1410        // No mid-wait mutation. Post-probe must observe the same hash
1411        // and a clean worktree.
1412        let post = crate::fetch::inspect_local_source_state(&canonical).expect("post-wait probe");
1413
1414        assert!(
1415            !post.is_dirty,
1416            "no mid-wait mutation must leave the post-wait probe clean",
1417        );
1418        assert_eq!(
1419            post.short_hash.as_ref(),
1420            Some(&acquired_hash),
1421            "no mid-wait commit must leave the short_hash unchanged",
1422        );
1423
1424        let hash_changed = post.short_hash != Some(acquired_hash);
1425        let state = if post.is_dirty {
1426            MidWaitState::DirtyEdit
1427        } else if hash_changed {
1428            MidWaitState::HashAdvanced
1429        } else {
1430            MidWaitState::Clean
1431        };
1432        assert_eq!(
1433            state,
1434            MidWaitState::Clean,
1435            "no-mutation post-wait state must classify as Clean so the \
1436             cache_lookup short-circuit fires",
1437        );
1438        assert_eq!(
1439            state.diagnostic(),
1440            None,
1441            "Clean must be silent — the cache-skip gate emits its own \
1442             diagnostic when the lookup hits",
1443        );
1444    }
1445
1446    /// Pins the DirtyEdit branch of [`MidWaitState`] classification at
1447    /// `kernel_build_pipeline` — operator edited a tracked file
1448    /// during the peer's build wait, post-wait probe surfaces
1449    /// `is_dirty=true` with no HEAD advance.
1450    ///
1451    /// Failure mode pinned: a future change that elides the
1452    /// `post.is_dirty` arm (e.g. trusting only `hash_changed`) would
1453    /// silently return a cache slot keyed on the pre-edit HEAD even
1454    /// though the operator's worktree no longer matches it — the
1455    /// rebuilt artifact would reflect the operator's local edits and
1456    /// the served cache slot would not.
1457    #[test]
1458    fn mid_wait_dirty_edit_invalidates_cache_hit_skip() {
1459        if !git_available() {
1460            eprintln!(
1461                "mid_wait_dirty_edit_invalidates_cache_hit_skip: \
1462                 git unavailable, skipping"
1463            );
1464            return;
1465        }
1466
1467        let tmp = tempfile::TempDir::new().unwrap();
1468        let canonical = tmp.path().to_path_buf();
1469        run_git(&canonical, &["init", "-q", "-b", "main"]);
1470        std::fs::write(canonical.join("seed.txt"), "initial").unwrap();
1471        run_git(&canonical, &["add", "seed.txt"]);
1472        run_git(&canonical, &["commit", "-q", "-m", "initial"]);
1473
1474        let pre = crate::fetch::inspect_local_source_state(&canonical).expect("acquire-time probe");
1475        let acquired_hash = pre
1476            .short_hash
1477            .clone()
1478            .expect("clean repo must carry a short_hash");
1479
1480        // Mid-wait edit to a tracked file (no commit). The post-wait
1481        // probe must classify this as DirtyEdit — same hash, dirty
1482        // worktree.
1483        std::fs::write(canonical.join("seed.txt"), "operator edit during wait").unwrap();
1484
1485        let post = crate::fetch::inspect_local_source_state(&canonical).expect("post-wait probe");
1486
1487        assert!(
1488            post.is_dirty,
1489            "uncommitted edit to a tracked file must mark the post-wait \
1490             probe dirty",
1491        );
1492
1493        let hash_changed = post.short_hash != Some(acquired_hash);
1494        let state = if post.is_dirty {
1495            MidWaitState::DirtyEdit
1496        } else if hash_changed {
1497            MidWaitState::HashAdvanced
1498        } else {
1499            MidWaitState::Clean
1500        };
1501        assert_eq!(
1502            state,
1503            MidWaitState::DirtyEdit,
1504            "dirty worktree without HEAD advance must classify as DirtyEdit",
1505        );
1506        assert!(
1507            state != MidWaitState::Clean,
1508            "DirtyEdit must falsify mid_wait_clean — the cache slot \
1509             corresponds to pre-edit state",
1510        );
1511    }
1512
1513    /// Pins the ProbeFailed branch of [`MidWaitState`] classification at
1514    /// `kernel_build_pipeline` — the probe used to re-check the source
1515    /// tree returned `Err` and the pipeline conservatively rebuilds.
1516    ///
1517    /// Provoke strategy: init + commit, then truncate `.git/HEAD` to
1518    /// empty so `gix::discover` still succeeds (the `.git` dir
1519    /// exists) but `repo.head_id()` fails on the malformed ref —
1520    /// that error path is `inspect_local_source_state`'s only route
1521    /// to `Result::Err`. The non-git arm of `gix::discover` returns
1522    /// `Ok((None, true, false))`, NOT an `Err`, so simply removing
1523    /// `.git` does not reach ProbeFailed.
1524    ///
1525    /// Failure mode pinned: a future refactor that treats probe
1526    /// errors as Clean would silently return a cache slot keyed on
1527    /// unknowable post-wait state. The conservative-rebuild
1528    /// disposition is correct precisely because the alternative
1529    /// hides genuine corruption from the operator.
1530    #[test]
1531    fn mid_wait_probe_failure_invalidates_cache_hit_skip() {
1532        if !git_available() {
1533            eprintln!(
1534                "mid_wait_probe_failure_invalidates_cache_hit_skip: \
1535                 git unavailable, skipping"
1536            );
1537            return;
1538        }
1539
1540        let tmp = tempfile::TempDir::new().unwrap();
1541        let canonical = tmp.path().to_path_buf();
1542        run_git(&canonical, &["init", "-q", "-b", "main"]);
1543        std::fs::write(canonical.join("seed.txt"), "initial").unwrap();
1544        run_git(&canonical, &["add", "seed.txt"]);
1545        run_git(&canonical, &["commit", "-q", "-m", "initial"]);
1546
1547        let pre = crate::fetch::inspect_local_source_state(&canonical).expect("acquire-time probe");
1548        assert!(
1549            pre.short_hash.is_some(),
1550            "pre-corruption probe must succeed (the corruption happens \
1551             mid-wait, not at acquire time)",
1552        );
1553
1554        // Corrupt HEAD mid-wait. `gix::discover` still sees `.git/`
1555        // and succeeds; the subsequent `head_id()` call fails on the
1556        // empty ref and `inspect_local_source_state` propagates the
1557        // error.
1558        std::fs::write(canonical.join(".git/HEAD"), b"").expect("truncate .git/HEAD");
1559
1560        let post = crate::fetch::inspect_local_source_state(&canonical);
1561        assert!(
1562            post.is_err(),
1563            "truncated .git/HEAD must surface as a probe error, not a \
1564             silent Clean classification — found: {post:?}",
1565        );
1566
1567        // Mirror the production dispatch: probe Err → ProbeFailed,
1568        // which falsifies mid_wait_clean and forces a rebuild.
1569        let state = match post {
1570            Ok(_) => MidWaitState::Clean,
1571            Err(_) => MidWaitState::ProbeFailed,
1572        };
1573        assert_eq!(
1574            state,
1575            MidWaitState::ProbeFailed,
1576            "probe Err must classify as ProbeFailed",
1577        );
1578        assert!(
1579            state != MidWaitState::Clean,
1580            "ProbeFailed must falsify mid_wait_clean — unknowable state \
1581             cannot be assumed Clean",
1582        );
1583    }
1584
1585    /// Pins the non-local-source branch of [`MidWaitState`]
1586    /// classification at `kernel_build_pipeline` — when the source
1587    /// came from a non-local kernel spec (e.g. `Git+ref`,
1588    /// `Tarball`, downloaded archive), the outer
1589    /// `if is_local_source && !acquired.is_dirty` guard short-circuits
1590    /// the probe entirely and the fall-through reaches
1591    /// [`MidWaitState::Clean`] via the `else { Clean }` arm.
1592    ///
1593    /// Failure mode pinned: a future refactor that inverts the outer
1594    /// guard (e.g. mistakenly calls `inspect_local_source_state` on a
1595    /// Git+ref source, which doesn't have a meaningful local probe
1596    /// target) would route a non-local source into the probe branch
1597    /// and likely surface ProbeFailed against a non-git tree — a
1598    /// noisy regression. This test pins the no-probe short-circuit.
1599    #[test]
1600    fn mid_wait_non_local_source_classifies_as_clean() {
1601        // Mirror the outer production switch with is_local_source=false.
1602        // No probe call — the outer `if is_local_source && !acquired.is_dirty`
1603        // guard short-circuits when !is_local_source, falling through
1604        // to the `else if acquired.is_dirty` / else arms.
1605        let is_local_source = false;
1606        let acquired_is_dirty = false;
1607        let state = if is_local_source && !acquired_is_dirty {
1608            unreachable!(
1609                "is_local_source=false must skip the probe branch — the \
1610                 outer guard requires both is_local_source AND \
1611                 !acquired.is_dirty to reach the probe arm"
1612            )
1613        } else if acquired_is_dirty {
1614            MidWaitState::PreAcquireDirty
1615        } else {
1616            MidWaitState::Clean
1617        };
1618        assert_eq!(
1619            state,
1620            MidWaitState::Clean,
1621            "non-local clean source must classify as Clean — the cache \
1622             short-circuit applies to any source whose state we cannot \
1623             probe (or did not need to probe)",
1624        );
1625        assert_eq!(
1626            state.diagnostic(),
1627            None,
1628            "Clean non-local source must be silent",
1629        );
1630    }
1631
1632    /// Pins the PreAcquireDirty variant identity and its silent
1633    /// diagnostic — `MidWaitState::PreAcquireDirty.diagnostic()`
1634    /// returns `None` because the wait was not the cause of the
1635    /// dirty state.
1636    ///
1637    /// SCOPE: does NOT exercise the caller-side dispatch order in
1638    /// `kernel_build_pipeline` — the test reconstructs the
1639    /// `if is_local_source && !acquired.is_dirty / else if
1640    /// acquired.is_dirty / else` chain inline because PreAcquireDirty
1641    /// is constructed without any probe call. A future refactor that
1642    /// flipped the guard order in `kernel_build_pipeline` would not
1643    /// fail this test; the other 4 mid_wait tests ground against
1644    /// `inspect_local_source_state` and would catch a probe-arm
1645    /// regression. This test pins the variant + diagnostic pair only.
1646    #[test]
1647    fn mid_wait_pre_acquire_dirty_suppresses_wait_diagnostic() {
1648        // Mirror the production dispatch with acquired.is_dirty=true.
1649        // No probe call — the `else if acquired.is_dirty` arm fires
1650        // before the probe-bearing branch. If the guard structure in
1651        // `kernel_build_pipeline` changes (e.g. PreAcquireDirty moves
1652        // inside the probe match), update this mirror.
1653        let is_local_source = true;
1654        let acquired_is_dirty = true;
1655        let state = if is_local_source && !acquired_is_dirty {
1656            unreachable!(
1657                "the guard requires !acquired.is_dirty before the probe \
1658                 branch; acquired_is_dirty=true must skip this arm"
1659            )
1660        } else if acquired_is_dirty {
1661            MidWaitState::PreAcquireDirty
1662        } else {
1663            MidWaitState::Clean
1664        };
1665        assert_eq!(
1666            state,
1667            MidWaitState::PreAcquireDirty,
1668            "acquired.is_dirty=true must classify as PreAcquireDirty",
1669        );
1670        assert_eq!(
1671            state.diagnostic(),
1672            None,
1673            "PreAcquireDirty must be silent — the wait was not the \
1674             cause of the dirty state, so a wait-related diagnostic \
1675             would fabricate attribution",
1676        );
1677    }
1678
1679    /// Pins the exact diagnostic bodies emitted by each
1680    /// [`MidWaitState`] variant so a future copywriting change to
1681    /// the operator-facing messages is a deliberate, reviewed
1682    /// edit rather than silent drift.
1683    ///
1684    /// Clean and PreAcquireDirty return `None` (silent). DirtyEdit,
1685    /// HashAdvanced, and ProbeFailed return their full body strings
1686    /// without the `{cli_label}: ` prefix — the caller composes the
1687    /// prefix at the eprintln site.
1688    #[test]
1689    fn mid_wait_state_diagnostics_pinned() {
1690        assert_eq!(MidWaitState::Clean.diagnostic(), None);
1691        assert_eq!(MidWaitState::PreAcquireDirty.diagnostic(), None);
1692        assert_eq!(
1693            MidWaitState::DirtyEdit.diagnostic(),
1694            Some(
1695                "source tree changed during peer's build wait \
1696                 — rebuilding to capture your local edits"
1697            ),
1698        );
1699        assert_eq!(
1700            MidWaitState::HashAdvanced.diagnostic(),
1701            Some(
1702                "source HEAD advanced during peer's build wait \
1703                 — rebuilding for the new commit"
1704            ),
1705        );
1706        assert_eq!(
1707            MidWaitState::ProbeFailed.diagnostic(),
1708            Some(
1709                "source-tree dirty re-check failed during peer's \
1710                 build wait — rebuilding conservatively (re-run with \
1711                 RUST_LOG=warn for the probe error)"
1712            ),
1713        );
1714    }
1715
1716    /// Pins the exact diagnostic body emitted by the post-mid-wait
1717    /// `cache_lookup` short-circuit so a future copywriting change
1718    /// to the operator-facing message is a deliberate, reviewed
1719    /// edit rather than silent drift — parallel to
1720    /// [`mid_wait_state_diagnostics_pinned`] for the
1721    /// [`MidWaitState`] family.
1722    ///
1723    /// Two assertions: a byte-for-byte match on the formatted body
1724    /// with a representative cache_key, plus an inequality between
1725    /// two distinct keys to prove the `{cache_key}` placeholder is
1726    /// load-bearing (catches a regression that replaces the
1727    /// substitution with a static label and would silently produce
1728    /// a constant string regardless of input).
1729    #[test]
1730    fn cache_hit_diagnostic_pinned() {
1731        let cache_key = "test-cache-key-7f8a9b";
1732        assert_eq!(
1733            cache_hit_diagnostic(cache_key),
1734            "concurrent ktstr build populated cache slot test-cache-key-7f8a9b \
1735             during peer's build wait — skipping redundant rebuild",
1736        );
1737        assert_ne!(
1738            cache_hit_diagnostic(cache_key),
1739            cache_hit_diagnostic("different-key-x86-64"),
1740            "cache_key substitution must be load-bearing, not a no-op",
1741        );
1742    }
1743
1744    /// Pins the `force` gate on the post-acquire cache short-circuit in
1745    /// [`kernel_build_pipeline`]. The peer-dedup skip fires only when ALL
1746    /// of `!force && mid_wait_clean && cache-hit && image-exists` hold;
1747    /// `--force` must rebuild even when a peer already populated the slot
1748    /// (the observed bug: a forced build silently no-op'd via this path).
1749    /// Mirrors the production `if !force && mid_wait_clean && ...` guard —
1750    /// keep in lockstep with the short-circuit condition.
1751    #[test]
1752    fn force_bypasses_post_acquire_cache_short_circuit() {
1753        // All three non-force preconditions hold (a warm, peer-populated
1754        // slot on a clean tree): the skip fires only when NOT forced.
1755        let mid_wait_clean = true;
1756        let cache_hit = true;
1757        let image_exists = true;
1758        let short_circuits = |force: bool| !force && mid_wait_clean && cache_hit && image_exists;
1759        assert!(
1760            short_circuits(false),
1761            "non-forced build with a warm peer-populated slot must skip \
1762             the redundant rebuild",
1763        );
1764        assert!(
1765            !short_circuits(true),
1766            "--force must rebuild through a warm peer-populated slot, not \
1767             short-circuit — a forced build that no-ops defeats --force",
1768        );
1769    }
1770
1771    /// `kernel build --cpu-cap N` parses through clap into
1772    /// `KernelCommand::Build { cpu_cap: Some(N), .. }`. Pins the
1773    /// flag's wire path: a future rename of the field, a stray
1774    /// `default_value`, or a `value_parser` change that altered
1775    /// rejection semantics would surface as a parse failure or a
1776    /// shape mismatch on the assertion.
1777    #[test]
1778    fn kernel_build_parses_cpu_cap_without_extra_flags() {
1779        use clap::Parser as _;
1780        #[derive(clap::Parser, Debug)]
1781        struct TestCli {
1782            #[command(subcommand)]
1783            cmd: KernelCommand,
1784        }
1785        let parsed =
1786            TestCli::try_parse_from(["prog", "build", "--kernel", "6.14.2", "--cpu-cap", "4"])
1787                .expect("kernel build --cpu-cap N must parse");
1788        match parsed.cmd {
1789            KernelCommand::Build {
1790                cpu_cap, kernel, ..
1791            } => {
1792                assert_eq!(cpu_cap, Some(4));
1793                assert_eq!(kernel.as_deref(), Some("6.14.2"));
1794            }
1795            other => panic!("expected KernelCommand::Build, got {other:?}"),
1796        }
1797    }
1798
1799    /// `kernel build` without `--cpu-cap` parses with `cpu_cap: None`
1800    /// — the "unset" sentinel the downstream planner expands into the
1801    /// 30%-of-allowed default. Pins the no-flag path so a future
1802    /// rename of the clap field or a stray `default_value = "0"`
1803    /// surfaces as a test failure, not a silent runtime behavior change.
1804    #[test]
1805    fn kernel_build_without_cpu_cap_defaults_to_none() {
1806        use clap::Parser as _;
1807        #[derive(clap::Parser, Debug)]
1808        struct TestCli {
1809            #[command(subcommand)]
1810            cmd: KernelCommand,
1811        }
1812        let parsed = TestCli::try_parse_from(["prog", "build", "--kernel", "6.14.2"])
1813            .expect("kernel build without --cpu-cap must parse");
1814        match parsed.cmd {
1815            KernelCommand::Build { cpu_cap, .. } => {
1816                assert_eq!(cpu_cap, None, "no --cpu-cap must produce None, not Some(0)",);
1817            }
1818            other => panic!("expected KernelCommand::Build, got {other:?}"),
1819        }
1820    }
1821
1822    /// `kernel build --cpu-cap 0` parses successfully at clap level
1823    /// — the "must be ≥ 1" check lives in [`CpuCap::new`], not in
1824    /// the clap value parser. Pins the two-layer validation: clap
1825    /// accepts any usize; runtime resolution via `CpuCap::resolve` is
1826    /// responsible for the "0 is rejected" diagnostic.
1827    #[test]
1828    fn kernel_build_cpu_cap_zero_passes_clap() {
1829        use clap::Parser as _;
1830        #[derive(clap::Parser, Debug)]
1831        struct TestCli {
1832            #[command(subcommand)]
1833            cmd: KernelCommand,
1834        }
1835        let parsed =
1836            TestCli::try_parse_from(["prog", "build", "--kernel", "6.14.2", "--cpu-cap", "0"])
1837                .expect("clap-level parse must accept 0; runtime validation rejects");
1838        match parsed.cmd {
1839            KernelCommand::Build { cpu_cap, .. } => {
1840                assert_eq!(
1841                    cpu_cap,
1842                    Some(0),
1843                    "clap parses 0 verbatim; validation is downstream",
1844                );
1845            }
1846            other => panic!("expected KernelCommand::Build, got {other:?}"),
1847        }
1848    }
1849
1850    // ---------------------------------------------------------------
1851    // kernel_build_pipeline reservation phase — factored-out
1852    // `acquire_build_reservation` covers the cpu_cap → acquire →
1853    // sandbox → make_jobs flow without needing a real kernel source.
1854    // ---------------------------------------------------------------
1855
1856    /// Serialize `KTSTR_BYPASS_LLC_LOCKS` env-var mutation across test
1857    /// threads. Delegates to the ONE crate-wide env mutex so it
1858    /// serializes against EVERY env-touching test (process-wide
1859    /// `std::env`), including the builder tests that read
1860    /// `KTSTR_BYPASS_LLC_LOCKS` — a module-local mutex left them racing.
1861    /// `lock_env()` recovers from poison.
1862    fn bypass_env_lock() -> std::sync::MutexGuard<'static, ()> {
1863        crate::test_support::test_helpers::lock_env()
1864    }
1865
1866    /// RAII guard for scoped `KTSTR_BYPASS_LLC_LOCKS` mutation.
1867    /// Caller holds `bypass_env_lock()` before constructing.
1868    struct BypassGuard;
1869    impl BypassGuard {
1870        fn set(value: &str) -> Self {
1871            // SAFETY: env_lock held by caller; serializes with
1872            // every other env-mutating test.
1873            unsafe {
1874                std::env::set_var(crate::KTSTR_BYPASS_LLC_LOCKS_ENV, value);
1875            }
1876            BypassGuard
1877        }
1878        fn remove() -> Self {
1879            // SAFETY: caller holds env_lock.
1880            unsafe {
1881                std::env::remove_var(crate::KTSTR_BYPASS_LLC_LOCKS_ENV);
1882            }
1883            BypassGuard
1884        }
1885    }
1886    impl Drop for BypassGuard {
1887        fn drop(&mut self) {
1888            // SAFETY: guard lifetime bounded by env_lock held by
1889            // caller; Drop runs before the mutex guard releases.
1890            unsafe {
1891                std::env::remove_var(crate::KTSTR_BYPASS_LLC_LOCKS_ENV);
1892            }
1893        }
1894    }
1895
1896    /// `acquire_build_reservation` with `KTSTR_BYPASS_LLC_LOCKS=1`
1897    /// plus `cpu_cap=None` returns a no-reservation `BuildReservation`:
1898    /// plan, sandbox, and make_jobs all None. Pins the "bypass
1899    /// disables both layers" contract.
1900    #[test]
1901    fn acquire_build_reservation_bypass_returns_no_reservation() {
1902        let _lock = bypass_env_lock();
1903        let _env = BypassGuard::set("1");
1904        let r = acquire_build_reservation("test", None).expect("bypass + no cap must succeed");
1905        assert!(r.plan.is_none(), "bypass must produce no LLC plan");
1906        assert!(
1907            r._sandbox.is_none(),
1908            "bypass must produce no cgroup sandbox",
1909        );
1910        assert!(
1911            r.make_jobs.is_none(),
1912            "bypass must fall back to nproc (None signals to caller)",
1913        );
1914    }
1915
1916    /// Regression pin: empty-string-as-unset contract for
1917    /// `KTSTR_BYPASS_LLC_LOCKS`. A bare `KTSTR_BYPASS_LLC_LOCKS=`
1918    /// (CI shells, Docker `--env` pass-through without value) must
1919    /// NOT activate the bypass — the reader at L102 uses
1920    /// `.is_some_and(|v| !v.is_empty())` and that contract is
1921    /// shared by all 7 sibling readers. If a future contributor
1922    /// flips to `.is_some_and(|_| true)` or bare `.is_ok()`, this
1923    /// test catches the regression before it silently disables LLC
1924    /// flock contention enforcement in CI.
1925    #[test]
1926    fn acquire_build_reservation_bypass_empty_string_rejected() {
1927        let _lock = bypass_env_lock();
1928        let _env = BypassGuard::set("");
1929        match acquire_build_reservation("test", None) {
1930            Ok(r) => {
1931                // Empty-as-unset means we take the standard branch,
1932                // not the bypass branch. Standard branch produces a
1933                // BuildReservation with plan / sandbox / make_jobs
1934                // tied together (set-or-unset together per the
1935                // `plan_and_make_jobs_consistent` invariant). If the
1936                // bypass had been (incorrectly) triggered, all 3
1937                // would be None.
1938                assert_eq!(
1939                    r.plan.is_some(),
1940                    r.make_jobs.is_some(),
1941                    "empty-string must NOT activate bypass — plan + make_jobs \
1942                     should follow the standard-branch invariant",
1943                );
1944            }
1945            Err(e) => {
1946                // Sysfs-unreadable host: standard branch failed for
1947                // unrelated reasons. The empty-string-as-unset
1948                // contract is still proven because the bypass branch
1949                // would have returned `Ok` with all-None fields (per
1950                // the `bypass_returns_no_reservation` test); reaching
1951                // Err proves the standard branch was taken.
1952                eprintln!("standard-branch error confirms bypass was NOT taken (good): {e:#}");
1953            }
1954        }
1955    }
1956
1957    /// `acquire_build_reservation` with `KTSTR_BYPASS_LLC_LOCKS=1`
1958    /// plus `cpu_cap=Some(_)` must error with the "resource contract"
1959    /// substring. Pins the conflict check at the pipeline's
1960    /// reservation entry point.
1961    #[test]
1962    fn acquire_build_reservation_bypass_with_cap_errors() {
1963        let _lock = bypass_env_lock();
1964        let _env = BypassGuard::set("1");
1965        let cap = crate::vmm::host_topology::CpuCap::new(2).expect("cap=2 valid");
1966        let err =
1967            acquire_build_reservation("test", Some(cap)).expect_err("bypass + cap must error");
1968        let msg = format!("{err:#}");
1969        assert!(
1970            msg.contains("resource contract"),
1971            "err must name the resource contract: {msg}",
1972        );
1973    }
1974
1975    /// `acquire_build_reservation` without bypass on a sysfs-capable
1976    /// host: returns a `BuildReservation` whose fields populate
1977    /// consistently — plan.is_some() iff make_jobs.is_some() iff
1978    /// sandbox.is_some(). Pins the "plan and make_jobs must never
1979    /// diverge" invariant.
1980    #[test]
1981    fn acquire_build_reservation_plan_and_make_jobs_consistent() {
1982        let _lock = bypass_env_lock();
1983        let _env = BypassGuard::remove();
1984        match acquire_build_reservation("test", None) {
1985            Ok(r) => {
1986                assert_eq!(
1987                    r.plan.is_some(),
1988                    r.make_jobs.is_some(),
1989                    "plan and make_jobs must agree on reservation presence",
1990                );
1991                if let (Some(p), Some(jobs)) = (r.plan.as_ref(), r.make_jobs) {
1992                    assert_eq!(
1993                        jobs,
1994                        crate::vmm::host_topology::make_jobs_for_plan(p),
1995                        "make_jobs must equal make_jobs_for_plan(&plan)",
1996                    );
1997                }
1998                assert_eq!(
1999                    r.plan.is_some(),
2000                    r._sandbox.is_some(),
2001                    "sandbox and plan must agree on reservation presence",
2002                );
2003            }
2004            Err(e) => {
2005                // Sysfs-unreadable host or contested LLCs. Accept
2006                // either outcome; the test's intent is to pin the
2007                // invariant in the success case, not force success.
2008                eprintln!("acquire_build_reservation unavailable on this host: {e:#}");
2009            }
2010        }
2011    }
2012
2013    /// `acquire_build_reservation` plain bypass (no `--cpu-cap`)
2014    /// must NOT touch the sysfs probe. The test sets the bypass and
2015    /// confirms no error escapes, even on a host whose
2016    /// `HostTopology::from_sysfs()` would otherwise fail (the
2017    /// bypass branch is taken FIRST in the function, before the
2018    /// sysfs probe is attempted). Pins the "bypass short-circuits
2019    /// the topology probe" branch shape — a regression that
2020    /// re-ordered the bypass check below the sysfs probe would
2021    /// surface as a sysfs-error escape.
2022    #[test]
2023    fn acquire_build_reservation_bypass_does_not_touch_sysfs() {
2024        let _lock = bypass_env_lock();
2025        let _env = BypassGuard::set("1");
2026        let r = acquire_build_reservation("test", None)
2027            .expect("bypass must succeed regardless of sysfs availability");
2028        // The bypass branch produces (None, None, None) by
2029        // construction — no further state to assert beyond the
2030        // sibling tests that already pin the field shape.
2031        assert!(r.plan.is_none());
2032        assert!(r._sandbox.is_none());
2033        assert!(r.make_jobs.is_none());
2034    }
2035
2036    // ---------------------------------------------------------------
2037    // acquire_source_tree_lock — per-source-tree flock that
2038    // serializes parallel builds against the same on-disk source.
2039    // ---------------------------------------------------------------
2040    //
2041    // Tests use `isolated_cache_dir()` to point `KTSTR_CACHE_DIR` at
2042    // a tempdir for the test's lifetime, so the production
2043    // `CacheDir::new()` resolves into the tempdir without touching
2044    // the operator's real cache directory. The lockfile path is
2045    // deterministic (cache_root/.locks/source-{path_hash}.lock) so
2046    // we can re-derive it from the canonical input path and assert
2047    // its presence.
2048
2049    /// `acquire_source_tree_lock` on a fresh canonical path under
2050    /// an isolated cache root succeeds (no peer holding the lock)
2051    /// and creates the lockfile under `cache_root/.locks/`. Pins
2052    /// the lockfile placement: a regression that moved the lockfile
2053    /// to `/tmp/` (where `tmpwatch` could sweep it under an active
2054    /// holder) would surface here as the assertion failing on
2055    /// "lockfile not found at expected path."
2056    #[test]
2057    fn acquire_source_tree_lock_succeeds_on_fresh_path() {
2058        use crate::test_support::test_helpers::{isolated_cache_dir, lock_env};
2059        let _env_lock = lock_env();
2060        let cache = isolated_cache_dir();
2061        let canonical = std::path::PathBuf::from("/tmp/fake-source-tree-for-test");
2062        let fd = acquire_source_tree_lock(&canonical, "test")
2063            .expect("fresh-path acquire must succeed under isolated cache");
2064        // Lockfile must land under the isolated cache root's
2065        // `.locks/` subdirectory. The naming is `source-{hash}.lock`
2066        // where `{hash}` is `canonical_path_hash(canonical)`.
2067        let path_hash = crate::fetch::canonical_path_hash(&canonical);
2068        let expected = cache
2069            .path()
2070            .join(crate::flock::LOCK_DIR_NAME)
2071            .join(format!("source-{path_hash}.lock"));
2072        assert!(
2073            expected.exists(),
2074            "lockfile must exist at {} after acquire",
2075            expected.display(),
2076        );
2077        // Drop the FD explicitly to release the flock before the
2078        // tempdir cleanup races with it.
2079        drop(fd);
2080    }
2081
2082    /// `acquire_source_tree_lock` returns the SAME lockfile path
2083    /// for two different canonical inputs IFF they share the same
2084    /// `canonical_path_hash`. Two distinct inputs (`/srv/linux-a`
2085    /// and `/srv/linux-b`) must produce DIFFERENT lockfiles so
2086    /// concurrent builds against unrelated source trees don't
2087    /// serialize against each other. Pins the per-tree
2088    /// disambiguation contract.
2089    #[test]
2090    fn acquire_source_tree_lock_distinct_paths_yield_distinct_lockfiles() {
2091        use crate::test_support::test_helpers::{isolated_cache_dir, lock_env};
2092        let _env_lock = lock_env();
2093        let cache = isolated_cache_dir();
2094        let path_a = std::path::PathBuf::from("/tmp/fake-source-a");
2095        let path_b = std::path::PathBuf::from("/tmp/fake-source-b");
2096        let fd_a = acquire_source_tree_lock(&path_a, "test")
2097            .expect("path A acquire must succeed under isolated cache");
2098        // Acquiring path B while path A's lock is still held must
2099        // ALSO succeed — they hash to different lockfiles, so
2100        // there's no contention.
2101        let fd_b = acquire_source_tree_lock(&path_b, "test").expect(
2102            "path B acquire must succeed concurrently with A — \
2103                 distinct canonical paths must hash to distinct \
2104                 lockfiles so unrelated builds don't serialize",
2105        );
2106        let hash_a = crate::fetch::canonical_path_hash(&path_a);
2107        let hash_b = crate::fetch::canonical_path_hash(&path_b);
2108        assert_ne!(
2109            hash_a, hash_b,
2110            "distinct canonical paths must produce distinct CRC32 hashes",
2111        );
2112        let lock_a = cache
2113            .path()
2114            .join(crate::flock::LOCK_DIR_NAME)
2115            .join(format!("source-{hash_a}.lock"));
2116        let lock_b = cache
2117            .path()
2118            .join(crate::flock::LOCK_DIR_NAME)
2119            .join(format!("source-{hash_b}.lock"));
2120        assert!(lock_a.exists());
2121        assert!(lock_b.exists());
2122        assert_ne!(lock_a, lock_b);
2123        drop(fd_a);
2124        drop(fd_b);
2125    }
2126
2127    /// `acquire_source_tree_lock` on a path whose lockfile is
2128    /// already held by a peer parks in a blocking flock(2) until the
2129    /// holder releases, then succeeds. Pins the try-then-wait
2130    /// contract: a regression that re-introduced the bail-on-EWOULDBLOCK
2131    /// behavior, or any other path that returns without ever calling
2132    /// `flock(LOCK_EX)` blocking, would either bypass the wait hook,
2133    /// complete while the original holder remains live, or fail to
2134    /// acquire after that holder releases.
2135    ///
2136    /// We simulate "concurrent peer" by holding the first FD on the
2137    /// main thread, spawn a worker that issues a second acquire, wait
2138    /// for the contended-path hook immediately before `block_flock`,
2139    /// assert that the worker has not completed, release the holder,
2140    /// and collect the successful acquisition. The hook makes this
2141    /// fully event-driven and avoids a host-global `/proc/locks` scan,
2142    /// whose cost scales with every runner on a storm host.
2143    #[test]
2144    fn acquire_source_tree_lock_blocks_on_contention_then_succeeds() {
2145        use crate::test_support::test_helpers::{isolated_cache_dir, lock_env};
2146        // `_env_lock` and `_cache` MUST outlive the spawned worker
2147        // thread. The worker reads `KTSTR_CACHE_DIR` inside
2148        // `acquire_source_tree_lock`'s `CacheDir::new()`; if
2149        // `IsolatedCacheDir`'s drop ran while the worker was still
2150        // resolving the cache root, the worker would observe a
2151        // restored / empty env var and either land outside the
2152        // tempdir or fail with a stale-cache-root error. The bindings
2153        // below are declared here and dropped at end-of-scope, AFTER
2154        // the explicit `worker_result` collection point below.
2155        let _env_lock = lock_env();
2156        let _cache = isolated_cache_dir();
2157        let canonical = std::path::PathBuf::from("/tmp/fake-source-contention");
2158        let holder = acquire_source_tree_lock(&canonical, "test")
2159            .expect("first acquire must succeed under isolated cache");
2160
2161        let worker_canonical = canonical.clone();
2162        let (waiting_tx, waiting_rx) = std::sync::mpsc::sync_channel(1);
2163        let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1);
2164        let _worker = std::thread::spawn(move || {
2165            let result = acquire_source_tree_lock_with_wait_hook(&worker_canonical, "test", || {
2166                let _ = waiting_tx.send(());
2167            });
2168            let _ = result_tx.send(result);
2169        });
2170
2171        waiting_rx
2172            .recv_timeout(std::time::Duration::from_secs(30))
2173            .expect("contended acquire must publish the blocking transition");
2174        assert!(
2175            matches!(
2176                result_rx.try_recv(),
2177                Err(std::sync::mpsc::TryRecvError::Empty)
2178            ),
2179            "the contended acquisition must not complete while the holder is live",
2180        );
2181
2182        drop(holder);
2183        let worker_result = result_rx
2184            .recv_timeout(std::time::Duration::from_secs(30))
2185            .expect("worker must deliver after the holder releases");
2186        let acquired = worker_result.expect("worker acquire must succeed once the holder releases");
2187
2188        // Drop the worker's FD explicitly so the lockfile flock
2189        // releases before the isolated cache dir is torn down.
2190        // `_env_lock` and `_cache` are bound at function-scope above
2191        // and drop at end-of-scope, AFTER this point.
2192        drop(acquired);
2193    }
2194
2195    /// `BuildReservation` field declaration order is load-bearing:
2196    /// `_sandbox` MUST be declared BEFORE `plan` so Rust's
2197    /// in-declaration-order field-drop runs the sandbox cgroup
2198    /// rmdir BEFORE the LLC flock release.
2199    ///
2200    /// A regression that swapped the field order would mean
2201    /// LLC flocks release first, which lets a peer claim the LLC
2202    /// while gcc children are still bound to a cgroup whose rmdir
2203    /// hasn't run yet.
2204    ///
2205    /// We can't assert drop ORDER directly without exotic
2206    /// machinery, but we can assert the field order is what we
2207    /// expect via the `Debug` derive: `_sandbox` appears in the
2208    /// formatted struct BEFORE `plan` IFF the field declaration
2209    /// order matches the Drop-order requirement. The field-name
2210    /// regex is enough to pin the order without depending on the
2211    /// inner field shapes (which evolve as the planner / sandbox
2212    /// types add or rename their own fields).
2213    #[test]
2214    fn build_reservation_field_order_pins_drop_invariant() {
2215        let r = BuildReservation {
2216            _sandbox: None,
2217            plan: None,
2218            make_jobs: None,
2219        };
2220        let dbg = format!("{r:?}");
2221        let sandbox_pos = dbg
2222            .find("_sandbox")
2223            .expect("Debug output must mention _sandbox field");
2224        let plan_pos = dbg
2225            .find("plan")
2226            .expect("Debug output must mention plan field");
2227        assert!(
2228            sandbox_pos < plan_pos,
2229            "_sandbox MUST be declared before plan so cgroup rmdir \
2230             runs BEFORE LLC flock release on Drop. Debug: {dbg}",
2231        );
2232    }
2233
2234    // ---------------------------------------------------------------
2235    // Post-build metadata-derivation arms inside
2236    // `kernel_build_pipeline`. These pure-logic blocks (config_hash
2237    // CRC32, source_vmlinux_stat, the dirty-tree cache-skip hint, the
2238    // post-build dirty re-check store decision) are unreachable through
2239    // the public `kernel_build_pipeline` entry without a real `make`
2240    // invocation, so each was extracted into a `pub(crate)` helper
2241    // (`config_hash_for`, `source_vmlinux_stat_for`,
2242    // `dirty_cache_skip_hint`, `post_build_cache_store_skip`) that both
2243    // the pipeline and these tests call — so the tests exercise the
2244    // real production code, not a copy.
2245    // ---------------------------------------------------------------
2246
2247    /// Drives the production [`config_hash_for`] helper (extracted from
2248    /// `kernel_build_pipeline`). Pins: (1) absent `.config` yields
2249    /// `None`; (2) present `.config` hashes to `crc32fast::hash` of the
2250    /// exact bytes; (3) the `{:08x}` zero-pad is load-bearing.
2251    ///
2252    /// Property (2) is checked against an INDEPENDENT `crc32fast::hash`
2253    /// so a hasher swap diverges. Property (3) is checked with an input
2254    /// whose CRC32 has a leading-zero nibble: a `{:x}` regression would
2255    /// drop the leading zero, shortening the hash below 8 chars and
2256    /// silently re-keying every cached build.
2257    #[test]
2258    fn config_hash_derivation_matches_crc32_and_width() {
2259        let tmp = tempfile::TempDir::new().expect("config tempdir");
2260        let source_dir = tmp.path();
2261
2262        // Absent `.config` arm: the production `else { None }` branch.
2263        assert_eq!(
2264            config_hash_for(source_dir).expect("absent-config read"),
2265            None,
2266            "no .config must yield None config_hash",
2267        );
2268
2269        // Present `.config` arm: hash matches an independent crc32fast.
2270        let body = b"CONFIG_SCHED_CLASS_EXT=y\nCONFIG_BPF=y\n";
2271        std::fs::write(source_dir.join(".config"), body).expect("write .config");
2272        let present = config_hash_for(source_dir).expect("present-config read");
2273        let expected = format!("{:08x}", crc32fast::hash(body));
2274        assert_eq!(
2275            present.as_deref(),
2276            Some(expected.as_str()),
2277            "present .config must hash to crc32fast::hash of its bytes",
2278        );
2279
2280        // Width guard: find an input whose CRC32 has a leading-zero
2281        // nibble (< 0x1000_0000) so the `{:08x}` zero-pad is the only
2282        // thing keeping the rendered hash at 8 chars. A `{:x}`
2283        // regression would render it as 7 chars and fail the length
2284        // assert below. The probe sequence is deterministic; ~1 in 16
2285        // inputs qualifies, so 10k iterations always finds one.
2286        let probe = (0u32..10_000)
2287            .map(|n| format!("probe-{n}"))
2288            .find(|p| crc32fast::hash(p.as_bytes()) < 0x1000_0000)
2289            .expect("a leading-zero CRC32 within 10k probes");
2290        std::fs::write(source_dir.join(".config"), probe.as_bytes()).expect("rewrite .config");
2291        let zh = config_hash_for(source_dir)
2292            .expect("probe read")
2293            .expect("present .config must hash");
2294        assert_eq!(
2295            zh.len(),
2296            8,
2297            "config_hash must always be 8 hex chars ({{:08x}}): {zh}"
2298        );
2299        assert!(
2300            zh.starts_with('0'),
2301            "leading-zero CRC32 must keep its zero pad: {zh}"
2302        );
2303        assert!(
2304            zh.bytes().all(|b| b.is_ascii_hexdigit()),
2305            "config_hash must be lowercase hex: {zh}",
2306        );
2307    }
2308
2309    /// Drives the production [`source_vmlinux_stat_for`] helper
2310    /// (extracted from `kernel_build_pipeline`). Pins three arms: a
2311    /// `None` ref, a ref to a missing file, and a present file.
2312    ///
2313    /// Failure mode pinned: a regression that dropped the `None`
2314    /// short-circuit (yielding `Some((0, _))` for an absent/missing
2315    /// vmlinux) or returned the wrong size would defeat the
2316    /// `prefer_source_tree_for_dwarf` staleness check that compares
2317    /// this stat against a later read.
2318    #[test]
2319    fn source_vmlinux_stat_present_and_absent_arms() {
2320        let tmp = tempfile::TempDir::new().expect("vmlinux tempdir");
2321        let vmlinux_path = tmp.path().join("vmlinux");
2322
2323        // None ref → short-circuits to None.
2324        assert_eq!(
2325            source_vmlinux_stat_for(None),
2326            None,
2327            "a None vmlinux_ref must yield None",
2328        );
2329        // Ref to a non-existent file → metadata fails → None (NOT a
2330        // phantom (0, _)).
2331        assert_eq!(
2332            source_vmlinux_stat_for(Some(vmlinux_path.as_path())),
2333            None,
2334            "a vmlinux_ref to a missing file must yield None, not (0, _)",
2335        );
2336
2337        // Present file → (real length, positive post-epoch mtime).
2338        let body = b"\x7fELF fake vmlinux payload bytes for stat";
2339        std::fs::write(&vmlinux_path, body).expect("write vmlinux");
2340        let (size, mtime_secs) = source_vmlinux_stat_for(Some(vmlinux_path.as_path()))
2341            .expect("present vmlinux must stat to Some");
2342        assert_eq!(
2343            size,
2344            body.len() as u64,
2345            "stat size must equal the real source-tree vmlinux length",
2346        );
2347        assert!(
2348            mtime_secs > 0,
2349            "a freshly-written vmlinux must carry a positive post-epoch \
2350             mtime in seconds, got {mtime_secs}",
2351        );
2352    }
2353
2354    /// Drives the production [`dirty_cache_skip_hint`] helper (extracted
2355    /// from `kernel_build_pipeline`). A git repo with uncommitted
2356    /// changes gets the "commit or stash" hint; a non-git tree
2357    /// (force-marked dirty because dirty detection is impossible) gets
2358    /// the "put the source under git" hint — telling a non-git operator
2359    /// to "commit or stash" leads nowhere.
2360    ///
2361    /// Failure mode pinned: a regression that dropped the `is_git`
2362    /// branch and always returned `DIRTY_TREE_CACHE_SKIP_HINT` would
2363    /// give non-git operators unactionable advice. The inequality of
2364    /// the two constants proves the branch the helper selects on is
2365    /// load-bearing.
2366    #[test]
2367    fn dirty_tree_cache_skip_hint_branches_on_is_git() {
2368        assert_eq!(
2369            dirty_cache_skip_hint(true),
2370            DIRTY_TREE_CACHE_SKIP_HINT,
2371            "is_git=true must select the commit/stash hint",
2372        );
2373        assert_eq!(
2374            dirty_cache_skip_hint(false),
2375            NON_GIT_TREE_CACHE_SKIP_HINT,
2376            "is_git=false must select the put-under-git hint",
2377        );
2378        assert_ne!(
2379            DIRTY_TREE_CACHE_SKIP_HINT, NON_GIT_TREE_CACHE_SKIP_HINT,
2380            "the two hints must differ so the is_git branch is \
2381             load-bearing — a non-git operator must not be told to \
2382             commit/stash",
2383        );
2384    }
2385
2386    /// Drives the production [`post_build_cache_store_skip`] predicate
2387    /// (extracted from `kernel_build_pipeline`, DISTINCT from the
2388    /// mid-wait dirty classifier above) against a real git tree for the
2389    /// clean (store proceeds) and mid-build-commit (hash advanced →
2390    /// skip) cases. `inspect_local_source_state` is the real probe; the
2391    /// skip decision is the real predicate.
2392    ///
2393    /// Failure mode pinned: a regression that dropped the
2394    /// `hash_changed` disjunct and trusted only `post.is_dirty` would
2395    /// store a build under a cache key keyed on the pre-commit HEAD
2396    /// even though the operator committed (clean worktree) on top
2397    /// during the build — a future cache hit would serve a build that
2398    /// no longer matches its recorded identity.
2399    #[test]
2400    fn post_build_dirty_recheck_skips_store_on_hash_advance() {
2401        if !git_available() {
2402            eprintln!(
2403                "post_build_dirty_recheck_skips_store_on_hash_advance: \
2404                 git unavailable, skipping"
2405            );
2406            return;
2407        }
2408
2409        let tmp = tempfile::TempDir::new().unwrap();
2410        let canonical = tmp.path().to_path_buf();
2411        run_git(&canonical, &["init", "-q", "-b", "main"]);
2412        std::fs::write(canonical.join("seed.txt"), "initial").unwrap();
2413        run_git(&canonical, &["add", "seed.txt"]);
2414        run_git(&canonical, &["commit", "-q", "-m", "initial"]);
2415
2416        // Acquire-time identity (frozen at build start in `local_source`).
2417        let acquire =
2418            crate::fetch::inspect_local_source_state(&canonical).expect("acquire-time probe");
2419        let acquired_hash = acquire.short_hash.clone();
2420
2421        // Clean post-build: same hash, clean worktree → store proceeds.
2422        let post_clean =
2423            crate::fetch::inspect_local_source_state(&canonical).expect("post-build clean probe");
2424        let (skip_clean, hash_changed_clean) =
2425            post_build_cache_store_skip(&post_clean, acquired_hash.as_deref());
2426        assert!(
2427            !hash_changed_clean,
2428            "an unchanged HEAD must not flag hash_changed",
2429        );
2430        assert!(
2431            !skip_clean,
2432            "an unchanged tree post-build must NOT skip the cache store",
2433        );
2434
2435        // Mid-build commit: HEAD advanced, worktree clean → skip store.
2436        std::fs::write(canonical.join("midbuild.txt"), "landed during make").unwrap();
2437        run_git(&canonical, &["add", "midbuild.txt"]);
2438        run_git(&canonical, &["commit", "-q", "-m", "mid-build commit"]);
2439        let post_advanced = crate::fetch::inspect_local_source_state(&canonical)
2440            .expect("post-build advanced probe");
2441        assert!(
2442            !post_advanced.is_dirty,
2443            "a committed mid-build change leaves the worktree clean; the \
2444             hash advance (not is_dirty) must drive the store skip",
2445        );
2446        let (skip_advanced, hash_changed_advanced) =
2447            post_build_cache_store_skip(&post_advanced, acquired_hash.as_deref());
2448        assert!(
2449            hash_changed_advanced,
2450            "the mid-build commit must yield a short_hash distinct from \
2451             the acquire-time hash",
2452        );
2453        assert!(
2454            skip_advanced,
2455            "a HEAD advance during the build must skip the cache store so \
2456             a stale identity is never recorded",
2457        );
2458    }
2459}