ktstr/
fetch.rs

1//! Kernel source acquisition: tarball download, GitHub codeload
2//! snapshot, git clone, local tree.
3//!
4//! The acquisition entry points each return an [`AcquiredSource`]
5//! carrying the source directory, cache key, and metadata the caller
6//! needs to proceed to configuration and build: [`download_tarball`]
7//! (kernel.org stable/RC), `download_github_archive` (a GitHub codeload
8//! commit snapshot), `git_clone_kinded` (a kind-directed shallow clone
9//! that dispatches to `git_clone_tag` / [`git_clone`]), and
10//! [`local_source`] (an on-disk tree).
11
12use std::collections::{HashSet, VecDeque};
13use std::io::Read;
14use std::net::{SocketAddr, ToSocketAddrs};
15use std::num::NonZeroU32;
16use std::path::{Path, PathBuf};
17use std::sync::OnceLock;
18use std::sync::atomic::Ordering;
19use std::time::{Duration, Instant};
20use std::{hash::BuildHasher, hash::Hasher};
21
22use ::gix;
23use anyhow::{Context, Result, anyhow};
24use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
25use md5::{Digest as _, Md5};
26use reqwest::Url;
27use reqwest::blocking::{Client, ClientBuilder};
28use sha2::{Digest, Sha256};
29
30#[path = "../build_support/gix_policy.rs"]
31mod gix_policy;
32
33/// Process-wide [`reqwest::blocking::Client`] lazily initialized on
34/// first access via [`shared_client`]. Keeping a single `Client`
35/// instance across the fetch-family reuses its TCP connection pool
36/// and TLS session cache across repeated calls to the same host
37/// within a CLI run. Cross-host fetches in the same run still
38/// re-handshake because reqwest's connection pool keys on host.
39static SHARED_CLIENT: OnceLock<Client> = OnceLock::new();
40
41/// Process interruption observed by every in-process source-acquisition
42/// operation. cargo-ktstr's signal handler updates this through the
43/// compatibility entry point [`set_git_operation_interrupted`]; binaries
44/// retaining the default signal disposition terminate directly and do not
45/// need the bridge.
46static SOURCE_OPERATION_INTERRUPTED: std::sync::atomic::AtomicBool =
47    std::sync::atomic::AtomicBool::new(false);
48
49/// Bridge a process-level signal epoch into in-process source acquisition.
50///
51/// This performs one atomic store and is safe to call from
52/// cargo-ktstr's SIGINT/SIGTERM handler. `false` starts a new guard epoch;
53/// `true` stops active and subsequent HTTP request/backoff/read/extract work
54/// as well as gix ref-map/receive/checkout work.
55#[doc(hidden)]
56pub fn set_git_operation_interrupted(interrupted: bool) {
57    SOURCE_OPERATION_INTERRUPTED.store(interrupted, Ordering::SeqCst);
58}
59
60/// Read the process-level source-acquisition interruption bridge.
61#[doc(hidden)]
62pub fn git_operation_interrupted() -> bool {
63    SOURCE_OPERATION_INTERRUPTED.load(Ordering::SeqCst)
64}
65
66fn ensure_source_operation_not_interrupted(phase: &str) -> Result<()> {
67    if SOURCE_OPERATION_INTERRUPTED.load(Ordering::Acquire) {
68        anyhow::bail!("source acquisition interrupted {phase}");
69    }
70    Ok(())
71}
72
73fn interrupted_io_error(phase: &str) -> std::io::Error {
74    std::io::Error::new(
75        std::io::ErrorKind::Interrupted,
76        format!("source acquisition interrupted {phase}"),
77    )
78}
79
80/// Connect-phase timeout for [`shared_client`]: bounds the TCP + TLS
81/// handshake before reqwest gives up on a dead peer or blackholed route.
82const SHARED_CLIENT_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
83
84/// Default blocking-operation timeout for [`shared_client`].
85///
86/// Reqwest's blocking client applies this budget to connection, request-write,
87/// and response-read waits. Keeping it explicit here means a response that
88/// delivered headers but then stopped producing body bytes cannot wedge a
89/// caller that forgot a per-request override. Small metadata/index/probe
90/// requests also pass this value explicitly; large archive and package
91/// downloads override it with [`DOWNLOAD_REQUEST_READ_TIMEOUT`].
92const SMALL_RESPONSE_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
93
94/// Return the process-wide shared [`reqwest::blocking::Client`]. First
95/// call constructs it via `Client::builder()` with
96/// `SHARED_CLIENT_CONNECT_TIMEOUT` and
97/// `SMALL_RESPONSE_REQUEST_TIMEOUT` applied; every subsequent call returns
98/// a reference to the same instance. This helper is for top-level CLI entries
99/// that want the default client.
100///
101/// Tests that need to verify a network round-trip (rather than a
102/// cache hit) must NOT pass `shared_client()` to a cache-routed
103/// helper (`cached_releases`, `cached_releases_with`,
104/// [`fetch_latest_stable_version`], [`fetch_version_for_prefix`]) —
105/// `RELEASES_CACHE` may already be populated by a peer test, in
106/// which case the helper returns cached data and the network is
107/// never touched. Construct a local `Client` and pass it to the
108/// cache-routed helper to skip the cache; the pointer-equality gate
109/// in `cached_releases_with` routes a non-singleton client to a
110/// direct `fetch_releases` call against `RELEASES_URL` (the
111/// production URL — the bypass skips the cache, NOT the URL). For
112/// full URL injection, call `fetch_releases` directly. Cache-routing
113/// tests inject the fetch operation into `cached_releases_with_fetch`
114/// so they can prove the bypass without scheduling a local HTTP
115/// server.
116///
117/// # Panics
118///
119/// Panics on the first call if `Client::builder().build()` fails to
120/// construct a client. Documented failure modes include TLS backend
121/// initialization (e.g. rustls/native-tls subsystem unreachable) and
122/// system-resolver config load failure; both are treated as setup
123/// bugs rather than runtime errors. The
124/// `expect` here, rather than propagating the error, mirrors the
125/// inherited behavior of `reqwest::blocking::Client::new()` (which
126/// is itself an infallible wrapper around `builder().build().expect`).
127pub fn shared_client() -> &'static Client {
128    SHARED_CLIENT.get_or_init(|| {
129        default_http_client_builder()
130            .build()
131            .expect("build shared reqwest client")
132    })
133}
134
135/// Construct the common reqwest client builder used by the process-wide
136/// client and by retry-only DNS-edge clients.
137///
138/// Keeping the builder policy in one place makes a routed retry retain the
139/// shared client's connect timeout, TLS defaults, and automatic system-proxy
140/// handling. A routed client adds only a per-origin DNS override; it does not
141/// rewrite the URL, Host header, TLS SNI, or proxy configuration.
142fn default_http_client_builder() -> ClientBuilder {
143    Client::builder()
144        .connect_timeout(SHARED_CLIENT_CONNECT_TIMEOUT)
145        .timeout(SMALL_RESPONSE_REQUEST_TIMEOUT)
146}
147
148/// Process-wide cache of the parsed `releases.json` payload.
149/// Populated by [`cached_releases_with`] on its first successful
150/// singleton-path fetch; every subsequent singleton call returns a
151/// clone of the cached vector without re-issuing the HTTP request.
152/// Lifetime matches the process — `releases.json` does not change
153/// underneath a single CLI invocation, so a per-process cache
154/// cannot serve stale data in any way the user would notice.
155///
156/// Failures are NOT cached: a transient kernel.org outage that
157/// errors the first call must allow a later caller to retry, since
158/// the underlying network condition may have cleared. Storing
159/// `Vec<Release>` rather than `Result<Vec<Release>>` enforces this
160/// at the type level — there's no way to populate the cache with
161/// a failure.
162///
163/// Companion to [`SHARED_CLIENT`]: both amortize per-invocation
164/// network cost across the resolve pipeline. Without this cache,
165/// `cargo ktstr test --kernel 6.10..6.12 --kernel 6.14..6.16`
166/// fetches `releases.json` twice — once per Range spec — under
167/// the rayon par_iter that drives `resolve_kernel_set`. With
168/// the cache the first Range to reach `expand_kernel_range`
169/// populates the slot; the second observes the populated slot
170/// and skips the network entirely.
171static RELEASES_CACHE: OnceLock<Vec<Release>> = OnceLock::new();
172
173/// Cache for the gregkh stable-mirror release tags — the `X.Y.Z`
174/// version strings parsed from its `refs/tags/vX.Y.Z` advertisement.
175/// Companion to [`RELEASES_CACHE`]: `--include-eol` may expand several
176/// `A..B` ranges under one `resolve_kernel_set`, and each would
177/// otherwise re-ls-remote the mirror. Populated on the first successful
178/// enumeration; a failed ls-remote leaves it empty so the next caller
179/// retries (`Vec`, not `Result`, mirroring `RELEASES_CACHE`).
180static STABLE_TAGS_CACHE: OnceLock<Vec<String>> = OnceLock::new();
181
182/// Fetch `releases.json` via the process-wide [`shared_client`],
183/// routing through [`RELEASES_CACHE`].
184///
185/// Thin wrapper for callers that don't already thread a `&Client`
186/// — top-level CLI entries like [`crate::cli::expand_kernel_range`]
187/// (under the rayon-driven `cargo ktstr` resolve pipeline) and
188/// `crate::cli::fetch_active_prefixes` (the EOL-annotation pass).
189/// Caching, race semantics, and fault-injection routing are all
190/// documented on [`cached_releases_with`].
191pub(crate) fn cached_releases() -> Result<Vec<Release>> {
192    cached_releases_with(shared_client())
193}
194
195/// Pointer-equality against the [`OnceLock`]-backed
196/// [`shared_client`] singleton is the correct predicate because
197/// `shared_client()` returns a stable `&'static Client` address.
198/// The [`cached_releases_with`] gate uses this predicate to
199/// decide whether to consult [`RELEASES_CACHE`]: the singleton
200/// hits the cache, every other (test-constructed) `Client`
201/// bypasses it and exercises the underlying [`fetch_releases`]
202/// path.
203///
204/// Caveat: `shared_client().clone()` produces a distinct
205/// `Client` at a different address even though it shares the
206/// singleton's connection pool via the inner `Arc`, so the
207/// clone bypasses the cache. Always pass `shared_client()`
208/// directly — never a clone — when cache routing is desired.
209///
210/// Side-effect-free when [`SHARED_CLIENT`] is uninitialized:
211/// no client can equal a not-yet-allocated singleton, so we
212/// return `false` without triggering `get_or_init` — tests
213/// that pass a local `Client` before any production code path
214/// has touched the singleton skip the construction entirely.
215fn is_shared_client(client: &Client) -> bool {
216    match SHARED_CLIENT.get() {
217        Some(singleton) => std::ptr::eq(client, singleton),
218        None => false,
219    }
220}
221
222/// Unified cache-aware entry point for `releases.json`. Routes
223/// the process-wide [`shared_client`] singleton through
224/// [`RELEASES_CACHE`]; any other (test-constructed) `Client`
225/// bypasses [`RELEASES_CACHE`] and calls [`fetch_releases`] with
226/// [`RELEASES_URL`] directly — the cache is skipped but the
227/// production URL is used.
228///
229/// Used by every in-file caller that already threads a `&Client`
230/// — [`fetch_latest_stable_version`], [`fetch_version_for_prefix`],
231/// [`latest_in_series`] — so production callers reuse
232/// [`RELEASES_CACHE`] and tests still get cache-bypass via the
233/// pointer-equality gate. [`cached_releases`] is the no-`Client`
234/// wrapper for top-level CLI entries.
235///
236/// This wrapper is the production entry point and
237/// pins the URL to [`RELEASES_URL`]; production code MUST go
238/// through this wrapper. A singleton call with a non-RELEASES_URL
239/// would otherwise populate [`RELEASES_CACHE`] with
240/// non-production data and corrupt every later production
241/// call — the singleton-path branch in
242/// [`cached_releases_with_url`] guards against this in both
243/// dev (`debug_assert!`) and release builds (fall back to
244/// bypass), but routing every production call through this
245/// wrapper makes the misuse impossible by construction.
246/// Caching, race semantics, and the bypass-vs-cache routing
247/// are fully documented on [`cached_releases_with_url`].
248fn cached_releases_with(client: &Client) -> Result<Vec<Release>> {
249    cached_releases_with_url(client, RELEASES_URL)
250}
251
252/// URL-explicit form of [`cached_releases_with`]. Production reaches this
253/// through the wrapper above, which pins `url` to [`RELEASES_URL`]. The
254/// routing core below accepts an injected fetch operation for deterministic
255/// cache/bypass tests.
256///
257/// Cache contract is identical to [`cached_releases_with`]:
258/// non-singleton clients bypass [`RELEASES_CACHE`] and call
259/// [`fetch_releases`] with `url`; the singleton routes through
260/// the cache only when `url == RELEASES_URL` (consulting via
261/// `OnceLock::get`, populating via `OnceLock::set` on miss). A
262/// singleton call with a non-RELEASES_URL trips the
263/// `debug_assert!` in dev builds and falls back to the bypass
264/// behavior in release builds — fetches directly via `url`,
265/// returns the result, never touches [`RELEASES_CACHE`]. The
266/// cache only ever stores data fetched from the singleton +
267/// RELEASES_URL combination, so a test that injects a mock URL
268/// on either branch cannot pollute the production cache.
269///
270/// Failures are propagated without populating [`RELEASES_CACHE`],
271/// so a transient kernel.org outage on the first call lets the
272/// next caller retry. Storing `Vec<Release>` (not
273/// `Result<Vec<Release>>`) enforces this at the type level.
274///
275/// Concurrent population on the singleton path is safe via the
276/// `OnceLock::set` race: the loser's `set` returns `Err(clone)`
277/// (the cloned vector that was passed in is moved back), the
278/// returned `Err` is discarded via `let _ = …`, and the loser
279/// returns its own original `fresh` vector. Both winner and
280/// loser return content-equivalent data since both fetched the
281/// same `releases.json`. Worst case under concurrent first
282/// calls: both callers issue the network round-trip, only one
283/// populates [`RELEASES_CACHE`]; every later call — from any
284/// thread — observes the populated slot via the `get` fast-path
285/// and skips the network.
286fn cached_releases_with_url(client: &Client, url: &str) -> Result<Vec<Release>> {
287    cached_releases_with_fetch(client, url, fetch_releases)
288}
289
290/// Cache-routing core with an injectable fetch operation.
291///
292/// Production passes [`fetch_releases`]. Keeping the network operation behind
293/// this narrow seam lets cache-routing tests prove singleton/bypass behavior
294/// without starting a localhost HTTP runtime whose response thread can be
295/// starved by a full CI storm.
296fn cached_releases_with_fetch<F>(client: &Client, url: &str, fetch: F) -> Result<Vec<Release>>
297where
298    F: FnOnce(&Client, &str) -> Result<Vec<Release>>,
299{
300    // Non-singleton clients bypass the cache (test fault injection).
301    if !is_shared_client(client) {
302        return fetch(client, url);
303    }
304    // Cache-poison guard: the singleton path populates
305    // RELEASES_CACHE on miss. A test author that mistakenly
306    // passes a non-production URL with shared_client() would
307    // fill the cache with non-production data and corrupt every
308    // later production call (which reaches the cache via
309    // get-fast-path). Catch the misuse at debug-build time —
310    // production callers always thread RELEASES_URL through the
311    // `cached_releases_with` wrapper, so the assertion is a
312    // no-op for them; only a future test author wiring this
313    // function up with shared_client() and a mock URL would trip
314    // it.
315    debug_assert!(
316        url == RELEASES_URL,
317        "cached_releases_with_url: shared_client() must use RELEASES_URL \
318         to avoid RELEASES_CACHE pollution — got url={url:?}, expected \
319         RELEASES_URL ({RELEASES_URL:?}). Tests that need URL injection \
320         must pass a non-singleton Client (which takes the bypass branch \
321         above and never touches the cache).",
322    );
323    // Release-build guard: `debug_assert!` is stripped in
324    // optimized builds, so a non-RELEASES_URL on the singleton
325    // path would otherwise reach the populate-on-miss path below
326    // and persistently poison RELEASES_CACHE for every later
327    // production caller. Mirror the bypass-branch behavior
328    // (fetch directly, do not touch the cache) so the misuse
329    // degrades to a slow per-call fetch instead of a permanently
330    // wrong cache. The debug_assert above still fires loudly in
331    // dev builds; this branch only catches the misuse that
332    // slipped through to release.
333    if url != RELEASES_URL {
334        return fetch(client, url);
335    }
336    if let Some(cached) = RELEASES_CACHE.get() {
337        return Ok(cached.clone());
338    }
339    let fresh = fetch(client, url)?;
340    // Race-loss: `set` returns `Err(clone)` carrying back the
341    // clone we passed in; we discard it and return the original
342    // `fresh` below. See the rustdoc above for full semantics.
343    let _ = RELEASES_CACHE.set(fresh.clone());
344    Ok(fresh)
345}
346
347/// Downloaded/cloned kernel source ready for building.
348#[non_exhaustive]
349pub struct AcquiredSource {
350    /// Path to the kernel source directory.
351    pub source_dir: PathBuf,
352    /// Cache key for this source (e.g. "6.14.2-tarball-x86_64-kc{kconfig_hash}").
353    pub cache_key: String,
354    /// Version string if known (e.g. "6.14.2", "6.15-rc3").
355    pub version: Option<String>,
356    /// How the source was acquired, with per-variant payload
357    /// (git hash/ref for `Git`, source tree path and git hash for
358    /// `Local`).
359    pub kernel_source: crate::cache::KernelSource,
360    /// Whether the source is a temporary directory that should be
361    /// cleaned up after building.
362    pub is_temp: bool,
363    /// For local sources: whether the working tree is dirty.
364    /// Dirty trees must not be cached.
365    pub is_dirty: bool,
366    /// For local sources: whether the source is an actual git
367    /// repository. `true` when `gix::discover` succeeded and the
368    /// crate could compute index + worktree dirty state; `false`
369    /// for non-git source trees (tarball-extracted, rsync'd,
370    /// hand-assembled) where dirty detection is impossible and
371    /// the source is always cache-skipped pessimistically. Lets
372    /// the cache-skip hint branch on whether `commit` / `stash`
373    /// are actionable remediations (they aren't for non-git
374    /// sources).
375    ///
376    /// For non-local sources (tarball, git clone) the field is
377    /// set to `true` by convention — these paths are always
378    /// `is_dirty = false`, so the cache-skip branch that reads
379    /// `is_git` is never reached and the value is inert. Pinning
380    /// to `true` (rather than leaving the field meaningless)
381    /// keeps the invariant "is_git is meaningful only when
382    /// is_dirty is true, but always set" so a future code path
383    /// that reaches `is_git` outside the cache-skip context does
384    /// not trip on an `is_git = false` under a known-good source.
385    pub is_git: bool,
386}
387
388/// Target architecture string and boot image name.
389pub fn arch_info() -> (&'static str, &'static str) {
390    #[cfg(target_arch = "x86_64")]
391    {
392        ("x86_64", "bzImage")
393    }
394    #[cfg(target_arch = "aarch64")]
395    {
396        ("aarch64", "Image")
397    }
398}
399
400/// Parse a version string into its major version for URL construction.
401///
402/// "6.14.2" -> 6, "6.15-rc3" -> 6.
403fn major_version(version: &str) -> Result<u32> {
404    let major_str = version
405        .split('.')
406        .next()
407        .ok_or_else(|| anyhow!("invalid version: {version}"))?;
408    major_str
409        .parse::<u32>()
410        .with_context(|| format!("invalid major version in {version}"))
411}
412
413/// Determine if a version string represents an RC release.
414///
415/// RC releases use a different URL pattern and gzip compression
416/// (vs xz for stable).
417fn is_rc(version: &str) -> bool {
418    version.contains("-rc")
419}
420
421/// One (`moniker`, `version`) row from kernel.org's `releases.json`.
422///
423/// A named struct instead of a bare `(String, String)` tuple so every
424/// call site reads its field by name (`r.moniker`, `r.version`) rather
425/// than positional destructuring — the two strings are trivially
426/// swappable at a tuple-destructure call site, and a silent swap
427/// would mis-drive `is_skippable_release_moniker` while the
428/// now-misnamed "moniker" string flows into `version_prefix`
429/// downstream. Naming the fields removes that class of bug at the
430/// type-checker level and shows up in IDE hints on every iteration
431/// site.
432///
433/// Both fields are owned `String` (not `&str`) because the values are
434/// parsed out of a `reqwest::Response` body whose lifetime ends when
435/// `fetch_releases` returns; downstream callers iterate the vector
436/// long after that borrow would dangle.
437#[derive(Clone, Debug)]
438pub(crate) struct Release {
439    /// releases.json `moniker` field — stable / longterm / mainline /
440    /// linux-next / etc. Consumed by
441    /// [`is_skippable_release_moniker`] and by
442    /// [`fetch_latest_stable_version`]'s stable/longterm filter.
443    pub moniker: String,
444    /// releases.json `version` field — e.g. `"6.14.2"`, `"6.15-rc3"`,
445    /// `"6.16-rc2-next-20260420"`. Consumed by
446    /// [`version_tuple`], [`patch_level`], and
447    /// `cli::version_prefix`.
448    pub version: String,
449}
450
451/// Is this releases.json moniker one that the version-resolution
452/// pipeline should skip?
453///
454/// `linux-next` is a rolling integration branch whose version strings
455/// carry a date suffix rather than a stable tag, so it does not fit
456/// the major.minor.patch resolution model used by `latest_in_series`,
457/// `fetch_version_for_prefix`, and `cli::fetch_active_prefixes`. The
458/// release iteration in all three sites filters it out; this helper
459/// is the single point of truth for that decision so a future moniker
460/// that also warrants skipping can be added in one place.
461pub(crate) fn is_skippable_release_moniker(moniker: &str) -> bool {
462    moniker == "linux-next"
463}
464
465/// Find the latest version in the same major.minor series from releases.json.
466///
467/// Returns `Some("6.14.10")` for prefix `"6.14"` if that series exists in
468/// releases.json. Returns `None` if the series is not found (EOL or invalid).
469fn latest_in_series(client: &Client, version: &str) -> Option<String> {
470    let prefix = {
471        let parts: Vec<&str> = version.split('.').collect();
472        if parts.len() >= 2 {
473            format!("{}.{}", parts[0], parts[1])
474        } else {
475            return None;
476        }
477    };
478
479    // Routes through [`RELEASES_CACHE`] for the singleton; see
480    // [`cached_releases_with`] for the bypass gate.
481    let releases = cached_releases_with(client).ok()?;
482    let mut best: Option<(String, (u32, u32, u32))> = None;
483    for r in &releases {
484        if is_skippable_release_moniker(&r.moniker) {
485            continue;
486        }
487        if !r.version.starts_with(&prefix) {
488            continue;
489        }
490        if r.version.len() != prefix.len() && r.version.as_bytes()[prefix.len()] != b'.' {
491            continue;
492        }
493        if let Some(tuple) = version_tuple(&r.version)
494            && (best.is_none() || tuple > best.as_ref().unwrap().1)
495        {
496            best = Some((r.version.clone(), tuple));
497        }
498    }
499    best.map(|(v, _)| v)
500}
501
502/// Build a user-facing error message for a version that was not found.
503///
504/// Suggests the latest version in the same major.minor series when
505/// releases.json contains one.
506fn version_not_found_msg(client: &Client, version: &str) -> String {
507    let parts: Vec<&str> = version.split('.').collect();
508    let prefix = if parts.len() >= 2 {
509        format!("{}.{}", parts[0], parts[1])
510    } else {
511        version.to_string()
512    };
513    match latest_in_series(client, version) {
514        Some(latest) if latest != version => {
515            format!("version {version} not found. latest {prefix}.x: {latest}")
516        }
517        _ => format!("version {version} not found"),
518    }
519}
520
521/// Reject responses where the server returned HTML instead of a binary
522/// archive. Some CDN error pages return 200 with text/html.
523fn reject_html_response(response: &reqwest::blocking::Response, url: &str) -> Result<()> {
524    if let Some(ct) = response.headers().get(reqwest::header::CONTENT_TYPE)
525        && let Ok(ct_str) = ct.to_str()
526        && ct_str.contains("text/html")
527    {
528        anyhow::bail!(
529            "download {url}: server returned HTML instead of tarball (URL may be invalid)"
530        );
531    }
532    Ok(())
533}
534
535/// Print download size from Content-Length header if available.
536///
537/// `cli_label` prefixes the diagnostic line so the message matches the
538/// binary the user invoked (`"ktstr"` vs `"cargo ktstr"`).
539fn print_download_size(
540    response: &reqwest::blocking::Response,
541    url: &str,
542    cli_label: &str,
543    mp: Option<&crate::cli::FetchProgress>,
544) {
545    let line = if let Some(len) = response.content_length() {
546        let mib = len as f64 / (1024.0 * 1024.0);
547        format!("{cli_label}: downloading {url} ({mib:.1} MiB)")
548    } else {
549        format!("{cli_label}: downloading {url}")
550    };
551    // Route through the progress group so the line coordinates with
552    // concurrent bars on a TTY (and still reaches piped/CI stderr when
553    // the group is hidden); raw `eprintln!` when no group is present.
554    match mp {
555        Some(fp) => fp.println(&line),
556        None => crate::cli::print_status_line(&line),
557    }
558}
559
560/// Maximum tolerated stretch of "no body bytes received" before a
561/// streaming download is declared stalled. Catches a TCP connection
562/// that completed handshake (so connect_timeout doesn't fire) but
563/// then silently stops delivering body data — a common CDN failure
564/// mode where keepalive holds the socket open while the upstream
565/// origin is unreachable. The 60s value is generous enough that a
566/// real slow uplink delivering chunks every few seconds never
567/// triggers it, but tight enough that a wedged connection surfaces
568/// before the run's overall test timeout.
569const DOWNLOAD_NO_PROGRESS_TIMEOUT: Duration = Duration::from_secs(60);
570
571/// Streaming `Read` adapter for kernel tarball downloads.
572///
573/// Wraps the [`reqwest::blocking::Response`] body to do two things
574/// the bare response cannot:
575///
576/// 1. **Body-progress watchdog.** Tracks `last_progress` (the
577///    instant of the last successful read with `n > 0`) and errors
578///    when more than [`DOWNLOAD_NO_PROGRESS_TIMEOUT`] elapses
579///    between byte-producing reads. Without this, a CDN edge that
580///    keepalives the socket but stops delivering body bytes would
581///    only surface after reqwest's per-request read timeout
582///    ([`DOWNLOAD_REQUEST_READ_TIMEOUT`], 300s), which bounds a
583///    single stalled `read()`; the watchdog applies the tighter
584///    60s no-progress bound across successive reads. The check fires
585///    BEFORE the inner `read()` so a stalled inner reader cannot
586///    out-block the watchdog.
587///
588/// 2. **Streaming checksums.** Updates SHA-256 and MD5 hashers with
589///    every byte that flows past, so kernel.org/distro SHA-256 and
590///    Google Cloud Storage MD5 metadata can both use this one
591///    watchdog/progress stream without a second pass over the data.
592///
593/// Sits between [`reqwest::blocking::Response`] and the
594/// decompression layer (`XzDecoder` / `GzDecoder`); both
595/// decompressors expose `into_inner()` so the wrapper can be
596/// recovered after extraction completes (see
597/// [`Self::finalize`]).
598struct DownloadStream<R: Read> {
599    /// Underlying reqwest response body. Owned because `XzDecoder`
600    /// and `GzDecoder` take ownership of their inner reader, so
601    /// the wrapper must hold the response by value rather than by
602    /// reference.
603    inner: R,
604    /// Running SHA-256 hasher updated on every byte-producing read.
605    /// Consumed by [`DownloadStream::finalize`] (which takes `self`
606    /// by value); the call site recovers the wrapper from inside
607    /// the decoder + tar archive chain via `into_inner` before
608    /// finalizing.
609    sha256: Sha256,
610    /// Running MD5 hasher for GCS `md5Hash` verification. GCS exposes
611    /// this digest as standard base64 rather than hexadecimal.
612    md5: Md5,
613    /// Total body bytes read so far. Surfaced in the watchdog
614    /// error message so an operator triaging "no progress" can see
615    /// how many bytes did arrive before the stall — distinguishing
616    /// "connection dropped after a few bytes" from "connection
617    /// dropped after most of the payload".
618    bytes_total: u64,
619    /// `Instant` of the last successful read with `n > 0`. Set at
620    /// construction (not on first read) so a connection that wins
621    /// the handshake but never delivers any body bytes still
622    /// trips the watchdog after [`DOWNLOAD_NO_PROGRESS_TIMEOUT`]
623    /// rather than waiting for an indeterminate pre-data window.
624    last_progress: Instant,
625    /// Tolerated stretch of zero-progress time. Pinned at
626    /// construction from [`DOWNLOAD_NO_PROGRESS_TIMEOUT`]; held in
627    /// the struct rather than read from the constant on every
628    /// `read()` so a future per-call override (e.g. shorter
629    /// timeouts in tests) lands without touching the watchdog
630    /// logic.
631    no_progress_timeout: Duration,
632    /// Optional indicatif download bar, advanced by `inc(n)` on
633    /// every byte-producing read in lockstep with `bytes_total`.
634    /// `None` is the no-bar path (non-TTY, or no progress group
635    /// threaded in) and carries zero per-read overhead beyond the
636    /// `Option` check. Advancing here — the single byte-accounting
637    /// site — guarantees `bar.position() == finalize().1`, so the
638    /// bar can never drift from the bytes the hasher and watchdog
639    /// observed.
640    progress: Option<indicatif::ProgressBar>,
641}
642
643impl<R: Read> DownloadStream<R> {
644    /// Construct a streaming wrapper around `inner` with the production
645    /// no-progress budget, optionally attaching an indicatif progress
646    /// bar. `last_progress` is set to "now" so the watchdog clock starts
647    /// at construction; the downstream decoder may take an indeterminate
648    /// time before the first `read()`, but any actual progress resets
649    /// the clock. The optional bar is advanced by `inc(n)` on every
650    /// byte-producing read (see the `progress` field); `progress = None`
651    /// is the non-TTY / no-group path (no bar). The bar is a pure
652    /// observer — it never affects the watchdog gate or the streaming
653    /// sha256, so a stalled or truncated download still surfaces its
654    /// error unchanged.
655    fn with_progress(inner: R, progress: Option<indicatif::ProgressBar>) -> Self {
656        Self {
657            inner,
658            sha256: Sha256::new(),
659            md5: Md5::new(),
660            bytes_total: 0,
661            last_progress: Instant::now(),
662            no_progress_timeout: DOWNLOAD_NO_PROGRESS_TIMEOUT,
663            progress,
664        }
665    }
666
667    /// Consume the wrapper and return `(hex_digest, bytes_total)`.
668    /// Lowercase hex matches the format kernel.org publishes in
669    /// `sha256sums.asc`, so the caller can do a direct
670    /// `eq_ignore_ascii_case` comparison without re-encoding.
671    fn finalize(self) -> (String, u64) {
672        (hex::encode(self.sha256.finalize()), self.bytes_total)
673    }
674
675    /// Consume the wrapper and return the GCS-style standard-base64
676    /// MD5 digest plus the byte total.
677    fn finalize_md5_base64(self) -> (String, u64) {
678        (
679            BASE64_STANDARD.encode(self.md5.finalize()),
680            self.bytes_total,
681        )
682    }
683}
684
685impl<R: Read> Read for DownloadStream<R> {
686    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
687        if SOURCE_OPERATION_INTERRUPTED.load(Ordering::Acquire) {
688            return Err(interrupted_io_error("before reading response body"));
689        }
690        // Watchdog gate: trip BEFORE delegating to the inner reader
691        // so a stalled inner read does not get a fresh chance to
692        // run after the no-progress window has already expired. The
693        // wrapper cannot interrupt a `read()` that is currently
694        // blocked in a syscall — that protection comes from the
695        // per-request timeout configured via
696        // `RequestBuilder::timeout` — but it can refuse to issue
697        // the next call once the cumulative no-progress window
698        // crosses the bound.
699        let elapsed = self.last_progress.elapsed();
700        if elapsed > self.no_progress_timeout {
701            return Err(std::io::Error::new(
702                std::io::ErrorKind::TimedOut,
703                format!(
704                    "download stalled: no body bytes for {}s after {} bytes received",
705                    elapsed.as_secs(),
706                    self.bytes_total,
707                ),
708            ));
709        }
710        match self.inner.read(buf) {
711            Ok(0) => {
712                // EOF: do NOT update last_progress — a 0-byte read
713                // is not progress, and updating here would let a
714                // decoder that polls past EOF reset the watchdog
715                // indefinitely.
716                Ok(0)
717            }
718            Ok(n) => {
719                // A signal can land while the inner blocking read owns this
720                // thread. Reject the bytes before feeding a decoder, checksum,
721                // or destination file so the caller's transaction rolls back
722                // instead of publishing after cancellation.
723                if SOURCE_OPERATION_INTERRUPTED.load(Ordering::Acquire) {
724                    return Err(interrupted_io_error("while reading response body"));
725                }
726                self.sha256.update(&buf[..n]);
727                self.md5.update(&buf[..n]);
728                self.bytes_total += n as u64;
729                self.last_progress = Instant::now();
730                // Advance the bar in lockstep with `bytes_total` (same
731                // `n`, same reads) so `position()` and `finalize().1`
732                // never diverge. No-op when no bar is attached.
733                if let Some(pb) = &self.progress {
734                    pb.inc(n as u64);
735                }
736                Ok(n)
737            }
738            Err(e) => Err(e),
739        }
740    }
741}
742
743/// Per-request body-stream timeout passed to
744/// [`reqwest::blocking::RequestBuilder::timeout`] for tarball
745/// downloads. The blocking client treats this as a per-`read()`
746/// deadline (reset on every successful read), so it complements the
747/// [`DownloadStream`] watchdog: reqwest's deadline kills a single
748/// stalled syscall, and the watchdog observes the cumulative
749/// no-progress window across multiple reads. Set generously
750/// (5 minutes) because a slow but progressing connection can
751/// legitimately take that long for a single read on a large CDN
752/// chunk; the watchdog provides the tighter 60s no-progress bound.
753const DOWNLOAD_REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(300);
754
755/// Retry policy for [`get_with_transient_retry`]: attempt count and
756/// the unit the exponential backoff scales from (sleep =
757/// `backoff_unit << attempt`, i.e. 2s/4s/8s for a 1s unit — the same
758/// timing as `build_helpers::retry_with_backoff`, which covers the
759/// build-script blob downloads; this covers the runtime kernel-source
760/// fetches). A struct rather than two bare params so the production
761/// sites share one const and the fetch_tests seam can inject a
762/// zero-backoff policy without sleeping through real backoff.
763struct HttpRetry {
764    attempts: u32,
765    backoff_unit: Duration,
766}
767
768/// Production retry policy: 4 attempts, 2s/4s/8s between them.
769/// Matches `MAX_TARBALL_ATTEMPTS`/`MAX_CLONE_ATTEMPTS` in build.rs so
770/// every network fetch in the project tolerates the same outage
771/// window (~14s) before giving up.
772const TRANSIENT_HTTP_RETRY: HttpRetry = HttpRetry {
773    attempts: 4,
774    backoff_unit: Duration::from_secs(1),
775};
776
777/// Whether an HTTP status is worth retrying: 429 (rate limit) and the
778/// gateway trio 502/503/504. kernel.org fronts cdn.kernel.org with
779/// Varnish, which returns bursts of `503` with `retry-after: 0`
780/// during cache churn — the exact failure that killed every
781/// `just kernel-build` job in one CI run. 500 is deliberately NOT
782/// retried: it signals a server bug rather than a transient edge
783/// condition, and the existing contract (pinned by
784/// `download_stable_tarball_500_is_hard_error_without_marker`) treats
785/// it as an immediate hard error.
786fn is_transient_http_status(status: reqwest::StatusCode) -> bool {
787    matches!(status.as_u16(), 429 | 502 | 503 | 504)
788}
789
790/// Whether a retryable status identifies a failed HTTP gateway/CDN edge.
791///
792/// 429 remains retryable but deliberately does not select another address:
793/// changing peers to evade a server's rate limit is incorrect. The gateway
794/// trio instead says that a connectable peer could not serve the request, so
795/// another address published for the same hostname is the useful next route.
796fn is_dns_edge_failure_status(status: reqwest::StatusCode) -> bool {
797    matches!(status.as_u16(), 502..=504)
798}
799
800/// Remaining DNS addresses for one origin during a single retry loop.
801struct HttpRetryRoutes {
802    host: String,
803    port: u16,
804    remaining: VecDeque<SocketAddr>,
805}
806
807impl HttpRetryRoutes {
808    fn matches(&self, host: &str, port: u16) -> bool {
809        self.host == host && self.port == port
810    }
811
812    fn matches_url(&self, url: &Url) -> bool {
813        url.host_str()
814            .zip(url.port_or_known_default())
815            .is_some_and(|(host, port)| self.matches(host, port))
816    }
817
818    fn mark_failed(&mut self, addr: SocketAddr) {
819        self.remaining
820            .retain(|candidate| candidate.ip() != addr.ip());
821    }
822}
823
824/// Resolve every address currently published for an HTTP origin.
825fn resolve_http_origin(host: &str, port: u16) -> std::io::Result<Vec<SocketAddr>> {
826    (host, port).to_socket_addrs().map(Iterator::collect)
827}
828
829/// Build a fresh client with an ordered set of origin-address candidates while
830/// retaining the hostname in the request URL.
831///
832/// Passing the whole remaining tail lets reqwest retain its connector-level
833/// multi-address/Happy-Eyeballs fallback within one attempt. The URL hostname
834/// still supplies the Host header and TLS SNI. If a system proxy intercepts
835/// the request, reqwest connects to that proxy instead; these origin
836/// candidates then remain advisory and do not claim which upstream peer the
837/// proxy selects.
838fn build_routed_http_client(host: &str, addrs: &[SocketAddr]) -> reqwest::Result<Client> {
839    default_http_client_builder()
840        .resolve_to_addrs(host, addrs)
841        .build()
842}
843
844/// Replace the retry-address queue with a fresh resolution of `request_url`.
845///
846/// Returning `false` means the URL has no routable HTTP origin (host or known
847/// port), so the caller should preserve the ordinary same-client retry.
848fn reset_http_retry_routes<R>(
849    routes: &mut Option<HttpRetryRoutes>,
850    request_url: &Url,
851    resolve: &R,
852) -> std::io::Result<bool>
853where
854    R: Fn(&str, u16) -> std::io::Result<Vec<SocketAddr>>,
855{
856    let Some(host) = request_url.host_str() else {
857        *routes = None;
858        return Ok(false);
859    };
860    let Some(port) = request_url.port_or_known_default() else {
861        *routes = None;
862        return Ok(false);
863    };
864
865    let mut seen = HashSet::new();
866    let remaining: VecDeque<_> = resolve(host, port)?
867        .into_iter()
868        .filter(|addr| seen.insert(*addr))
869        .collect();
870    let has_candidates = !remaining.is_empty();
871    *routes = Some(HttpRetryRoutes {
872        host: host.to_owned(),
873        port,
874        remaining,
875    });
876    Ok(has_candidates)
877}
878
879/// Populate or update the alternate-address queue after a gateway response.
880fn record_failed_http_edge<R>(
881    routes: &mut Option<HttpRetryRoutes>,
882    response_url: &Url,
883    failed_remote: Option<SocketAddr>,
884    resolve: &R,
885) -> std::io::Result<bool>
886where
887    R: Fn(&str, u16) -> std::io::Result<Vec<SocketAddr>>,
888{
889    let Some(host) = response_url.host_str() else {
890        return Ok(false);
891    };
892    let Some(port) = response_url.port_or_known_default() else {
893        return Ok(false);
894    };
895
896    if routes
897        .as_ref()
898        .is_none_or(|existing| !existing.matches(host, port))
899    {
900        reset_http_retry_routes(routes, response_url, resolve)?;
901    }
902    if let (Some(existing), Some(failed)) = (routes.as_mut(), failed_remote) {
903        existing.mark_failed(failed);
904    }
905
906    Ok(routes
907        .as_ref()
908        .is_some_and(|existing| !existing.remaining.is_empty()))
909}
910
911/// Prepare the next routed attempt after a transport error.
912///
913/// [`next_routed_http_client`] removes the current primary before issuing the
914/// request, leaving only untried candidates in `routes.remaining`. When the
915/// error URL still names that origin, preserve the advanced tail rather than
916/// re-resolving and retrying the same primary again. A missing route set
917/// (initial-client failure) or a different error origin (redirect failure)
918/// gets a fresh resolution for the URL that actually failed.
919fn prepare_routes_after_transport_error<R>(
920    routes: &mut Option<HttpRetryRoutes>,
921    failed_url: &Url,
922    resolve: &R,
923) -> std::io::Result<bool>
924where
925    R: Fn(&str, u16) -> std::io::Result<Vec<SocketAddr>>,
926{
927    if routes
928        .as_ref()
929        .is_some_and(|existing| existing.matches_url(failed_url))
930    {
931        return Ok(routes
932            .as_ref()
933            .is_some_and(|existing| !existing.remaining.is_empty()));
934    }
935    reset_http_retry_routes(routes, failed_url, resolve)
936}
937
938/// Route-relevant facts from one failed HTTP attempt.
939///
940/// Keeping this independent of reqwest's concrete response/error objects gives
941/// the retry loop one transition point for both failure classes and lets tests
942/// drive the route state deterministically without scheduling local sockets.
943enum HttpRetryRouteFailure<'a> {
944    Response {
945        status: reqwest::StatusCode,
946        response_url: &'a Url,
947        remote_addr: Option<SocketAddr>,
948    },
949    Transport {
950        failed_url: &'a Url,
951    },
952}
953
954/// Update retry-route state from one failed attempt.
955///
956/// Non-gateway responses (notably 429) deliberately leave routing unchanged.
957/// Gateway responses discard the observed peer after resolving the response
958/// origin, while transport errors initialize or preserve the candidate tail
959/// according to [`prepare_routes_after_transport_error`].
960fn prepare_http_retry_route<R>(
961    routes: &mut Option<HttpRetryRoutes>,
962    failure: HttpRetryRouteFailure<'_>,
963    resolve: &R,
964) -> std::io::Result<bool>
965where
966    R: Fn(&str, u16) -> std::io::Result<Vec<SocketAddr>>,
967{
968    match failure {
969        HttpRetryRouteFailure::Response {
970            status,
971            response_url,
972            remote_addr,
973        } if is_dns_edge_failure_status(status) => {
974            record_failed_http_edge(routes, response_url, remote_addr, resolve)
975        }
976        HttpRetryRouteFailure::Response { .. } => Ok(false),
977        HttpRetryRouteFailure::Transport { failed_url } => {
978            prepare_routes_after_transport_error(routes, failed_url, resolve)
979        }
980    }
981}
982
983/// Build a retry client for the next untried primary address plus every
984/// remaining fallback candidate.
985///
986/// Client-construction failure is non-fatal: skipping that primary address
987/// preserves the original retry contract rather than replacing an HTTP outage
988/// with a TLS/backend setup error.
989fn next_routed_http_client<B>(
990    routes: &mut Option<HttpRetryRoutes>,
991    build_client: &B,
992) -> Option<(Client, Vec<SocketAddr>)>
993where
994    B: Fn(&str, &[SocketAddr]) -> reqwest::Result<Client>,
995{
996    let routes = routes.as_mut()?;
997    while let Some(primary) = routes.remaining.pop_front() {
998        let candidates: Vec<_> = std::iter::once(primary)
999            .chain(routes.remaining.iter().copied())
1000            .collect();
1001        match build_client(&routes.host, &candidates) {
1002            Ok(client) => return Some((client, candidates)),
1003            Err(err) => {
1004                tracing::warn!(
1005                    host = %routes.host,
1006                    ?candidates,
1007                    %err,
1008                    "failed to build an HTTP retry client for resolved candidates; \
1009                     trying a shorter candidate tail",
1010                );
1011            }
1012        }
1013    }
1014    None
1015}
1016
1017/// GET `url`, retrying transient failures (transport errors and
1018/// [`is_transient_http_status`] statuses) per `retry`.
1019///
1020/// The failure surface is IDENTICAL to a bare
1021/// `client.get(url).send()`: every non-transient response — success,
1022/// 404, or any other status — is returned to the caller untouched, so
1023/// each call site keeps its own status routing (the 404 →
1024/// [`TarballNotFound`] git-fallback marker, the RC/codeload bespoke
1025/// 404 messages) and its exact error wording. A transient status on
1026/// the FINAL attempt is likewise returned (the caller's status gate
1027/// bails with the same `HTTP 503` text an unretried call would have
1028/// produced); a transport error on the final attempt propagates with
1029/// the same `"{what} {url}"` context the call sites previously
1030/// attached. Retries are purely additive.
1031///
1032/// `what` is the call site's error-context verb ("download" /
1033/// "fetch") so the transport-error context matches the wording each
1034/// site used before retries existed.
1035///
1036/// Only the GET is retried — response-body streaming failures are
1037/// out of scope (the tarball sites stream through [`DownloadStream`]
1038/// with its own watchdog, and a mid-stream abort after a 200 leaves
1039/// staging state a plain re-GET could not resume anyway).
1040fn get_with_transient_retry(
1041    client: &Client,
1042    url: &str,
1043    timeout: Option<Duration>,
1044    what: &str,
1045    retry: &HttpRetry,
1046) -> Result<reqwest::blocking::Response> {
1047    get_with_transient_retry_and_headers(
1048        client,
1049        url,
1050        timeout,
1051        what,
1052        retry,
1053        &reqwest::header::HeaderMap::new(),
1054    )
1055}
1056
1057/// [`get_with_transient_retry`] with headers reapplied to every attempt.
1058///
1059/// This is used by the ranged URL-existence probe. Keeping request
1060/// construction inside the retry loop is important: a retry must carry the
1061/// same `Range` contract as the first attempt instead of silently becoming a
1062/// full-body GET.
1063fn get_with_transient_retry_and_headers(
1064    client: &Client,
1065    url: &str,
1066    timeout: Option<Duration>,
1067    what: &str,
1068    retry: &HttpRetry,
1069    headers: &reqwest::header::HeaderMap,
1070) -> Result<reqwest::blocking::Response> {
1071    get_with_transient_retry_and_routes(
1072        client,
1073        url,
1074        timeout,
1075        what,
1076        retry,
1077        headers,
1078        HttpRouteFailover {
1079            enabled: is_shared_client(client),
1080            resolve: resolve_http_origin,
1081            build_client: build_routed_http_client,
1082        },
1083    )
1084}
1085
1086struct HttpRouteFailover<R, B> {
1087    enabled: bool,
1088    resolve: R,
1089    build_client: B,
1090}
1091
1092/// Implementation seam for [`get_with_transient_retry`].
1093///
1094/// Production enables address failover only for [`shared_client`], whose
1095/// builder configuration is known and can be reproduced safely. A
1096/// caller-provided client may carry custom roots, proxies, or redirect policy,
1097/// so its retries remain on that exact client. Tests inject deterministic
1098/// address resolution and a no-proxy routed-client builder here.
1099fn get_with_transient_retry_and_routes<R, B>(
1100    client: &Client,
1101    url: &str,
1102    timeout: Option<Duration>,
1103    what: &str,
1104    retry: &HttpRetry,
1105    headers: &reqwest::header::HeaderMap,
1106    route_failover: HttpRouteFailover<R, B>,
1107) -> Result<reqwest::blocking::Response>
1108where
1109    R: Fn(&str, u16) -> std::io::Result<Vec<SocketAddr>>,
1110    B: Fn(&str, &[SocketAddr]) -> reqwest::Result<Client>,
1111{
1112    assert!(retry.attempts > 0, "HttpRetry.attempts must be >= 1");
1113    let mut routes = None;
1114    let mut routed_client: Option<(Client, Vec<SocketAddr>)> = None;
1115
1116    for attempt in 1..=retry.attempts {
1117        ensure_source_operation_not_interrupted("before HTTP request")?;
1118        let request_client = routed_client
1119            .as_ref()
1120            .map(|(client, _)| client)
1121            .unwrap_or(client);
1122        let mut req = request_client.get(url).headers(headers.clone());
1123        if let Some(t) = timeout {
1124            req = req.timeout(t);
1125        }
1126        let mut advance_route = false;
1127        match req.send() {
1128            Ok(response) => {
1129                ensure_source_operation_not_interrupted("after HTTP response headers")?;
1130                let status = response.status();
1131                if !is_transient_http_status(status) || attempt == retry.attempts {
1132                    return Ok(response);
1133                }
1134                let remote_addr = response.remote_addr();
1135                if route_failover.enabled {
1136                    match prepare_http_retry_route(
1137                        &mut routes,
1138                        HttpRetryRouteFailure::Response {
1139                            status,
1140                            response_url: response.url(),
1141                            remote_addr,
1142                        },
1143                        &route_failover.resolve,
1144                    ) {
1145                        Ok(has_routes) => advance_route = has_routes,
1146                        Err(err) => tracing::warn!(
1147                            url = %response.url(),
1148                            %err,
1149                            "failed to resolve alternate HTTP addresses; retrying normally",
1150                        ),
1151                    }
1152                }
1153                tracing::warn!(
1154                    %url, %status, ?remote_addr, attempt, max_attempts = retry.attempts,
1155                    "{what} hit a transient HTTP status; retrying",
1156                );
1157            }
1158            Err(e) => {
1159                if attempt == retry.attempts {
1160                    return Err(anyhow::Error::new(e).context(format!("{what} {url}")));
1161                }
1162                if route_failover.enabled {
1163                    let failed_url = e.url().cloned().or_else(|| Url::parse(url).ok());
1164                    if let Some(failed_url) = failed_url {
1165                        match prepare_http_retry_route(
1166                            &mut routes,
1167                            HttpRetryRouteFailure::Transport {
1168                                failed_url: &failed_url,
1169                            },
1170                            &route_failover.resolve,
1171                        ) {
1172                            Ok(has_routes) => advance_route = has_routes,
1173                            Err(resolve_err) => tracing::warn!(
1174                                url = %failed_url,
1175                                err = %resolve_err,
1176                                "failed to refresh HTTP candidates after a transport error; \
1177                                 retrying normally",
1178                            ),
1179                        }
1180                    }
1181                }
1182                tracing::warn!(
1183                    %url, error_url = ?e.url(), err = %e,
1184                    attempt, max_attempts = retry.attempts,
1185                    "{what} failed in transport; retrying",
1186                );
1187            }
1188        }
1189        // Reached only on a transient failure with attempts left:
1190        // both match arms return on the final attempt.
1191        source_operation_backoff(retry.backoff_unit * (1u32 << attempt))?;
1192        if advance_route {
1193            routed_client = next_routed_http_client(&mut routes, &route_failover.build_client);
1194            if let Some((_, candidates)) = routed_client.as_ref() {
1195                tracing::warn!(
1196                    %url,
1197                    ?candidates,
1198                    next_attempt = attempt + 1,
1199                    "retrying with alternate resolved origin candidates; \
1200                     an active proxy may select a different upstream route",
1201                );
1202            }
1203        }
1204    }
1205    unreachable!("the attempt == retry.attempts arms above return on the final iteration")
1206}
1207
1208/// Sleep between HTTP attempts without making cancellation wait for the whole
1209/// exponential-backoff interval. The 100 ms quantum matches the flock wait
1210/// poller and is far below the shortest production backoff (2 s).
1211fn source_operation_backoff(duration: Duration) -> Result<()> {
1212    const QUANTUM: Duration = Duration::from_millis(100);
1213    let deadline = Instant::now() + duration;
1214    loop {
1215        ensure_source_operation_not_interrupted("during HTTP retry backoff")?;
1216        let now = Instant::now();
1217        if now >= deadline {
1218            return Ok(());
1219        }
1220        std::thread::sleep((deadline - now).min(QUANTUM));
1221    }
1222}
1223
1224/// GET `url` through the process-wide [`shared_client`] with the
1225/// standard [`TRANSIENT_HTTP_RETRY`] policy and return the full
1226/// response body as bytes.
1227///
1228/// The distro repo-metadata resolver ([`crate::distro::repo`]) fetches
1229/// repomd.xml, primary.xml.{gz,zst,xz}, `Packages.gz`, and `mirror.list`
1230/// through this so every metadata fetch rides the same retry/backoff
1231/// seam as the kernel-source downloads. `what` is the error-context
1232/// verb ("fetch"). A non-success status is a hard error carrying the
1233/// status; the caller adds its own context. The explicit
1234/// [`SMALL_RESPONSE_REQUEST_TIMEOUT`] bounds both the wait for response
1235/// headers and every blocking body read, including the headers-then-stall
1236/// failure mode.
1237pub(crate) fn fetch_metadata_bytes(url: &str, what: &str) -> Result<Vec<u8>> {
1238    fetch_metadata_bytes_with(
1239        shared_client(),
1240        url,
1241        what,
1242        &TRANSIENT_HTTP_RETRY,
1243        SMALL_RESPONSE_REQUEST_TIMEOUT,
1244    )
1245}
1246
1247/// Injectable core of [`fetch_metadata_bytes`] for timeout/retry tests.
1248fn fetch_metadata_bytes_with(
1249    client: &Client,
1250    url: &str,
1251    what: &str,
1252    retry: &HttpRetry,
1253    timeout: Duration,
1254) -> Result<Vec<u8>> {
1255    let response = get_with_transient_retry(client, url, Some(timeout), what, retry)?;
1256    if !response.status().is_success() {
1257        anyhow::bail!("{what} {url}: HTTP {}", response.status());
1258    }
1259    read_response_bytes(response, url)
1260}
1261
1262/// GET `url` like [`fetch_metadata_bytes`] but decode the body as
1263/// UTF-8 text — for the plain-text metadata endpoints (`mirror.list`,
1264/// `meta-release-lts`).
1265pub(crate) fn fetch_metadata_text(url: &str, what: &str) -> Result<String> {
1266    let bytes = fetch_metadata_bytes(url, what)?;
1267    String::from_utf8(bytes).with_context(|| format!("decode body of {url} as UTF-8"))
1268}
1269
1270/// Ranged GET used by live distro-resolution tests to prove a resolved
1271/// package URL still exists upstream without downloading the package.
1272///
1273/// The range header is rebuilt on every transient retry, and the small-response
1274/// timeout bounds both header and body waits. Returning the final status keeps
1275/// 404 and other permanent failures visible to the caller's assertion.
1276#[cfg(test)]
1277pub(crate) fn probe_url_status(url: &str) -> Result<reqwest::StatusCode> {
1278    probe_url_status_with(
1279        shared_client(),
1280        url,
1281        &TRANSIENT_HTTP_RETRY,
1282        SMALL_RESPONSE_REQUEST_TIMEOUT,
1283    )
1284}
1285
1286#[cfg(test)]
1287fn probe_url_status_with(
1288    client: &Client,
1289    url: &str,
1290    retry: &HttpRetry,
1291    timeout: Duration,
1292) -> Result<reqwest::StatusCode> {
1293    let mut headers = reqwest::header::HeaderMap::new();
1294    headers.insert(
1295        reqwest::header::RANGE,
1296        reqwest::header::HeaderValue::from_static("bytes=0-0"),
1297    );
1298    let response =
1299        get_with_transient_retry_and_headers(client, url, Some(timeout), "probe", retry, &headers)?;
1300    Ok(response.status())
1301}
1302
1303/// Construct the cdn.kernel.org `sha256sums.asc` URL for a stable
1304/// major series:
1305/// `https://cdn.kernel.org/pub/linux/kernel/v{major}.x/sha256sums.asc`.
1306/// Single source of truth for the manifest URL shape, used by
1307/// [`resolve_expected_sha256`] (production) and shared with the
1308/// URL-injection test seam so the two never drift.
1309fn sha256sums_url(major: u32) -> String {
1310    format!("https://cdn.kernel.org/pub/linux/kernel/v{major}.x/sha256sums.asc")
1311}
1312
1313/// GET the cleartext SHA-256 manifest at `url` and return its body.
1314///
1315/// Returns the file body as a `String` on success. Any error
1316/// (transport failure, non-2xx status, non-UTF-8 body) is
1317/// propagated; the caller treats failure as "no expected hash
1318/// available" and downgrades verification to a warning.
1319///
1320/// Takes the full `url` rather than a `major` so the GET-and-status
1321/// mechanics are reachable with an injected URL (a localhost mock)
1322/// without a real cdn.kernel.org round-trip — mirrors the
1323/// [`fetch_releases`] / [`cached_releases_with_url`] seam. Production
1324/// reaches this only via [`resolve_expected_sha256_from_url`], whose
1325/// URL is pinned by [`sha256sums_url`].
1326fn fetch_sha256sums_from_url(client: &Client, url: &str) -> Result<String> {
1327    tracing::info!(%url, "fetching kernel tarball sha256sums (requires network)");
1328    let response = get_with_transient_retry(
1329        client,
1330        url,
1331        Some(SMALL_RESPONSE_REQUEST_TIMEOUT),
1332        "fetch",
1333        &TRANSIENT_HTTP_RETRY,
1334    )?;
1335    if !response.status().is_success() {
1336        anyhow::bail!("fetch {url}: HTTP {}", response.status());
1337    }
1338    let bytes = read_response_bytes(response, url)?;
1339    String::from_utf8(bytes).with_context(|| format!("decode body of {url} as UTF-8"))
1340}
1341
1342/// Read a short HTTP response through the same cancellation-aware
1343/// streaming adapter as large archives. Metadata endpoints do not need
1344/// a visible byte bar, but they must observe the shared signal epoch
1345/// before and after every blocking body read.
1346fn read_response_bytes(response: reqwest::blocking::Response, url: &str) -> Result<Vec<u8>> {
1347    let mut stream = DownloadStream::with_progress(response, None);
1348    let mut bytes = Vec::new();
1349    stream
1350        .read_to_end(&mut bytes)
1351        .with_context(|| format!("read body of {url}"))?;
1352    ensure_source_operation_not_interrupted("after reading HTTP response body")?;
1353    Ok(bytes)
1354}
1355
1356/// Extract the SHA-256 hex digest for `target_filename` from the
1357/// cleartext-signed `sha256sums.asc` body.
1358///
1359/// kernel.org publishes `sha256sums.asc` as a PGP-cleartext-signed
1360/// document: a `-----BEGIN PGP SIGNED MESSAGE-----` header, an
1361/// optional `Hash:` line, a blank line, the cleartext body
1362/// (`<64-hex-chars>  <filename>` per line), then a
1363/// `-----BEGIN PGP SIGNATURE-----` block. We only need the
1364/// cleartext body — signature verification is a separate concern
1365/// (the user-facing instruction is "If no expected hash available,
1366/// log warning", not "require signature").
1367///
1368/// Returns `Some(lowercase_hex)` on first match. Returns `None` if
1369/// the target filename does not appear in the manifest (e.g. the
1370/// upstream rotated or removed the entry).
1371fn parse_sha256_for_file(manifest: &str, target_filename: &str) -> Option<String> {
1372    // Strip the PGP signature trailer if present. Everything after
1373    // the signature marker is binary noise that never contains
1374    // checksum lines.
1375    let body = manifest
1376        .split_once("-----BEGIN PGP SIGNATURE-----")
1377        .map(|(before, _)| before)
1378        .unwrap_or(manifest);
1379    for line in body.lines() {
1380        let line = line.trim();
1381        // sha256sum format: `<64-hex-chars><whitespace><filename>`.
1382        // Split on whitespace; require exactly two tokens and a
1383        // 64-char hex first token.
1384        let mut parts = line.split_whitespace();
1385        let Some(hash) = parts.next() else { continue };
1386        let Some(name) = parts.next() else { continue };
1387        if name != target_filename {
1388            continue;
1389        }
1390        if hash.len() != 64 || !hash.chars().all(|c| c.is_ascii_hexdigit()) {
1391            continue;
1392        }
1393        return Some(hash.to_ascii_lowercase());
1394    }
1395    None
1396}
1397
1398/// Verify `actual_hex` against `expected_hex` (case-insensitive).
1399/// Returns `Ok(())` on match, `Err` with a diagnostic message on
1400/// mismatch. Pulled out of the call site so the comparison logic
1401/// has one home and the diagnostic carries both digests in lowercase
1402/// hex for direct copy-paste reuse.
1403fn verify_sha256(actual_hex: &str, expected_hex: &str, url: &str) -> Result<()> {
1404    if actual_hex.eq_ignore_ascii_case(expected_hex) {
1405        Ok(())
1406    } else {
1407        anyhow::bail!(
1408            "sha256 mismatch for {url}: expected {}, got {}. \
1409             If cdn.kernel.org updated this tarball in-place, \
1410             retry with --skip-sha256 to bypass verification.",
1411            expected_hex.to_ascii_lowercase(),
1412            actual_hex.to_ascii_lowercase(),
1413        );
1414    }
1415}
1416
1417/// Resolve the expected SHA-256 digest for a stable tarball from
1418/// cdn.kernel.org's `sha256sums.asc` manifest.
1419///
1420/// Three outcomes:
1421/// - `Some(hex)` — manifest fetched and the entry for `tarball_name`
1422///   was parsed cleanly.
1423/// - `None` with no warning (only when `skip_sha256 = true`) —
1424///   operator explicitly opted out of verification; emits a single
1425///   security-sensitive bypass warning instead.
1426/// - `None` with a per-cause warning (manifest fetch failed, or
1427///   manifest fetched but entry missing) — best-effort fallback so
1428///   a transient cdn.kernel.org outage / schema drift does not
1429///   gate the whole download.
1430///
1431/// The fallback path is deliberately permissive: we trade strict
1432/// authentication for build availability. A network-path attacker
1433/// who can deny `sha256sums.asc` while serving a poisoned
1434/// `linux-{version}.tar.xz` could exploit this; operators who
1435/// require strict verification should pin the source via a
1436/// `--kernel <path>` or `--kernel git+…` source rather than the
1437/// download path. The bypass warnings
1438/// surface on the operator's diagnostic stream so the lost
1439/// guarantee is visible to ops triage.
1440///
1441/// Extracted from [`download_stable_tarball`] so the gate is
1442/// directly unit-testable without mocking network calls — the
1443/// caller-supplied `client` reaches a `Client::get` only when
1444/// `skip_sha256 == false`, so a `skip_sha256 = true` test does not
1445/// need a configured `Client`.
1446fn resolve_expected_sha256(
1447    client: &Client,
1448    major: u32,
1449    tarball_name: &str,
1450    skip_sha256: bool,
1451) -> Option<String> {
1452    resolve_expected_sha256_from_url(client, &sha256sums_url(major), tarball_name, skip_sha256)
1453}
1454
1455/// URL-explicit HTTP wrapper for [`resolve_expected_sha256`]. Production
1456/// reaches this with [`sha256sums_url`]; the routing core below accepts an
1457/// injected fetch operation for deterministic no-network tests.
1458fn resolve_expected_sha256_from_url(
1459    client: &Client,
1460    sha256sums_url: &str,
1461    tarball_name: &str,
1462    skip_sha256: bool,
1463) -> Option<String> {
1464    resolve_expected_sha256_with(tarball_name, skip_sha256, || {
1465        fetch_sha256sums_from_url(client, sha256sums_url)
1466    })
1467}
1468
1469/// Checksum-resolution core with an injectable manifest fetch.
1470///
1471/// Production supplies the real retrying HTTP operation through
1472/// [`resolve_expected_sha256_from_url`]. The seam keeps parsing and
1473/// warn-and-continue routing directly testable without making a synthetic
1474/// localhost server compete with thousands of CI test processes.
1475fn resolve_expected_sha256_with<F>(
1476    tarball_name: &str,
1477    skip_sha256: bool,
1478    fetch_manifest: F,
1479) -> Option<String>
1480where
1481    F: FnOnce() -> Result<String>,
1482{
1483    if skip_sha256 {
1484        tracing::warn!(
1485            tarball = %tarball_name,
1486            "--skip-sha256: bypassing checksum verification — the \
1487             downloaded tarball will not be authenticated against \
1488             cdn.kernel.org's sha256sums.asc manifest. Use only when \
1489             upstream has updated a tarball in-place and the manifest \
1490             is mismatched.",
1491        );
1492        return None;
1493    }
1494    // Best-effort expected-hash lookup: any failure (network,
1495    // status, parse, missing entry) downgrades to a warning so the
1496    // download still proceeds. The warning surfaces the cause so an
1497    // operator triaging "kernel build went weird" can spot that
1498    // verification was skipped.
1499    match fetch_manifest() {
1500        Ok(manifest) => match parse_sha256_for_file(&manifest, tarball_name) {
1501            Some(hex) => Some(hex),
1502            None => {
1503                tracing::warn!(
1504                    tarball = %tarball_name,
1505                    "sha256sums.asc fetched but no entry for {tarball_name}; \
1506                     download will proceed without checksum verification. \
1507                     Pass --skip-sha256 to bypass the manifest fetch when \
1508                     the entry is known to be absent.",
1509                );
1510                None
1511            }
1512        },
1513        Err(err) => {
1514            tracing::warn!(
1515                error = %format!("{err:#}"),
1516                "failed to fetch sha256sums.asc; download will proceed \
1517                 without checksum verification. Pass --skip-sha256 to \
1518                 bypass the manifest fetch when the manifest is known \
1519                 to be unavailable.",
1520            );
1521            None
1522        }
1523    }
1524}
1525
1526/// GitHub mirror of the linux-stable tree — comprehensive (stable +
1527/// base-release `vX.Y.Z` tags back to v2.6) and the authoritative
1528/// source for tags whose `.tar.xz` is no longer on cdn.kernel.org.
1529///
1530/// cdn.kernel.org keeps only the LATEST tarball of each series
1531/// currently in `releases.json`; every superseded point release AND
1532/// every tag of an EOL series is pruned (a GET for the tarball 404s,
1533/// verified empirically — and HEAD is not a dependable existence probe
1534/// on the CDN). The gregkh mirror still carries every `vX.Y.Z` tag, and
1535/// codeload serves each tag's snapshot as a `tar.gz`, so a codeload
1536/// download recovers the source a pruned tarball would have provided —
1537/// no clone. Its `ls-refs` advertises every release tag, which
1538/// `--include-eol` enumerates to surface EOL series absent from
1539/// `releases.json` (see [`cached_stable_tags`]) and which
1540/// [`fetch_version_for_prefix`] resolves for an EOL/unreleased series.
1541/// github.com advertises allow-sha + a ref-prefix filter and a codeload
1542/// CDN; git.kernel.org offers neither. Used by [`download_tarball`]'s
1543/// [`TarballNotFound`] fallback and the prefix resolver.
1544const STABLE_MIRROR_URL: &str = "https://github.com/gregkh/linux";
1545
1546/// Marker error attached to a stable-tarball download failure when
1547/// cdn.kernel.org returns HTTP 404.
1548///
1549/// A 404 means the tarball is pruned — an EOL series (absent from
1550/// `releases.json`) or a superseded point release (the CDN retains
1551/// only each maintained series' latest). [`download_tarball`] detects
1552/// this via `downcast_ref` (the context-aware anyhow accessor — a
1553/// `chain().any(..is::<T>())` walk would MISS a context-wrapped
1554/// marker) and falls back to a codeload snapshot of the tag from the
1555/// gregkh mirror ([`STABLE_MIRROR_URL`]). Any other HTTP status is a
1556/// hard error with no fallback.
1557#[derive(Debug)]
1558struct TarballNotFound;
1559
1560impl std::fmt::Display for TarballNotFound {
1561    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1562        f.write_str("stable tarball pruned from cdn.kernel.org (EOL or superseded point release)")
1563    }
1564}
1565
1566impl std::error::Error for TarballNotFound {}
1567
1568/// Download a stable kernel tarball (.tar.xz) from cdn.kernel.org.
1569///
1570/// Returns a [`TarballNotFound`] error (downcast-detectable) when the
1571/// CDN 404s the tarball — see that type for the pruning semantics and
1572/// [`download_tarball`] for the git-tag fallback it triggers.
1573///
1574/// Streams the body through a [`DownloadStream`] watchdog so a
1575/// stalled connection (no body bytes for
1576/// [`DOWNLOAD_NO_PROGRESS_TIMEOUT`]) surfaces as an error rather
1577/// than blocking indefinitely. Computes SHA-256 over the streamed
1578/// bytes and verifies against the digest in
1579/// `sha256sums.asc` for the matching `linux-{version}.tar.xz`
1580/// entry; if the manifest fetch / parse fails (transient outage,
1581/// schema drift, missing entry), logs a warning and continues
1582/// without verification rather than failing the whole download.
1583///
1584/// `skip_sha256 = true` bypasses the manifest fetch entirely and
1585/// emits a single bypass warning. Intended for the case where
1586/// cdn.kernel.org has updated a tarball in-place (a new point
1587/// release reusing the same URL) and the manifest is stale or
1588/// mismatched. Unverified downloads are a security-sensitive
1589/// fallback — the bypass warning surfaces the lost guarantee on
1590/// the operator's diagnostic stream.
1591fn download_stable_tarball(
1592    client: &Client,
1593    version: &str,
1594    dest_dir: &Path,
1595    cli_label: &str,
1596    skip_sha256: bool,
1597    mp: Option<&crate::cli::FetchProgress>,
1598) -> Result<PathBuf> {
1599    let major = major_version(version)?;
1600    let url = format!("https://cdn.kernel.org/pub/linux/kernel/v{major}.x/linux-{version}.tar.xz");
1601    download_stable_tarball_from_url(client, &url, version, dest_dir, cli_label, skip_sha256, mp)
1602}
1603
1604/// URL-injectable core of [`download_stable_tarball`]: the GET, the
1605/// 404→[`TarballNotFound`] / other-status→hard-error status gate, and
1606/// the stream→verify→extract pipeline, against an arbitrary tarball
1607/// `url`. Production reaches this only via [`download_stable_tarball`],
1608/// which pins the cdn.kernel.org URL; the seam exists so the status
1609/// routing (404 marker vs hard error) is unit-testable against a
1610/// localhost mock without a real cdn round-trip — mirrors
1611/// [`resolve_expected_sha256_from_url`] / [`fetch_releases`].
1612fn download_stable_tarball_from_url(
1613    client: &Client,
1614    url: &str,
1615    version: &str,
1616    dest_dir: &Path,
1617    cli_label: &str,
1618    skip_sha256: bool,
1619    mp: Option<&crate::cli::FetchProgress>,
1620) -> Result<PathBuf> {
1621    let major = major_version(version)?;
1622    let tarball_name = format!("linux-{version}.tar.xz");
1623
1624    let expected_sha256 = resolve_expected_sha256(client, major, &tarball_name, skip_sha256);
1625
1626    tracing::info!(%url, "downloading stable kernel tarball (requires network)");
1627    let response = get_with_transient_retry(
1628        client,
1629        url,
1630        Some(DOWNLOAD_REQUEST_READ_TIMEOUT),
1631        "download",
1632        &TRANSIENT_HTTP_RETRY,
1633    )?;
1634    if !response.status().is_success() {
1635        if response.status() == reqwest::StatusCode::NOT_FOUND {
1636            // Pruned tarball (EOL series or superseded point release).
1637            // Return the downcast-detectable marker so `download_tarball`
1638            // falls back to a codeload snapshot of the tag from the
1639            // gregkh mirror (`STABLE_MIRROR_URL`) rather than failing
1640            // outright.
1641            return Err(anyhow::Error::new(TarballNotFound));
1642        }
1643        anyhow::bail!("download {url}: HTTP {}", response.status());
1644    }
1645    reject_html_response(&response, url)?;
1646    print_download_size(&response, url, cli_label, mp);
1647    // Capture the total before `response` is moved into the stream so a
1648    // determinate (percent + ETA) bar can be built; `None` when the
1649    // server sent no Content-Length, in which case the bar degrades to
1650    // a live byte counter.
1651    let total = response.content_length();
1652
1653    // Route status lines through the progress group (see
1654    // `print_download_size`); `eprintln!` when no group is threaded in.
1655    let status = |line: &str| match mp {
1656        Some(fp) => fp.println(line),
1657        None => crate::cli::print_status_line(line),
1658    };
1659    status(&format!("{cli_label}: extracting tarball (xz)"));
1660    // Stage extraction inside `dest_dir` (same filesystem) so the
1661    // final `RENAME_NOREPLACE` into place is atomic and a verification
1662    // failure leaves `dest_dir` untouched. A bad mirror that serves
1663    // a wrong-version archive — or sneaks stray top-level entries
1664    // alongside `linux-{version}/` — gets caught after extraction
1665    // but before anything lands in `dest_dir`. The TempDir's Drop
1666    // sweeps every entry the malicious archive deposited.
1667    let staging =
1668        tempfile::TempDir::new_in(dest_dir).with_context(|| "create extraction staging dir")?;
1669    let download_bar = mp.map(|fp| fp.download_bar(version, total));
1670    let stream = DownloadStream::with_progress(response, download_bar.as_ref().map(|b| b.bar()));
1671    let decoder = xz2::read::XzDecoder::new(stream);
1672    let mut archive = tar::Archive::new(decoder);
1673    ensure_source_operation_not_interrupted("before extracting stable kernel tarball")?;
1674    archive
1675        .unpack(staging.path())
1676        .with_context(|| "extract tarball")?;
1677    ensure_source_operation_not_interrupted("after extracting stable kernel tarball")?;
1678
1679    // Recover the watchdog wrapper from inside the decoder/archive
1680    // chain to read the streaming digest. `into_inner` on tar +
1681    // xz2 each peel one layer of the chain. Done after a successful
1682    // unpack so we don't compute over a partial stream.
1683    let stream = archive.into_inner().into_inner();
1684    let (actual_hex, bytes_total) = stream.finalize();
1685    // Download is complete (every byte streamed) — clear the bar
1686    // before emitting the verification status so the two don't overlap.
1687    if let Some(bar) = &download_bar {
1688        bar.finish();
1689    }
1690    if let Some(expected) = expected_sha256.as_deref() {
1691        verify_sha256(&actual_hex, expected, url)?;
1692        status(&format!(
1693            "{cli_label}: sha256 verified ({bytes_total} bytes, hash {actual_hex})"
1694        ));
1695    } else if !skip_sha256 {
1696        // Skip path already emitted its bespoke bypass warning
1697        // before the download; firing again here under "no
1698        // expected sha256 available" would mislead — that wording
1699        // implies a fallback, not an explicit operator opt-out.
1700        tracing::warn!(
1701            url = %url,
1702            bytes = bytes_total,
1703            sha256 = %actual_hex,
1704            "no expected sha256 available for {url}; computed digest \
1705             {actual_hex} over {bytes_total} bytes is unverified",
1706        );
1707    }
1708
1709    ensure_source_operation_not_interrupted("before promoting stable kernel source")?;
1710    let promoted = promote_staged_kernel_tree_transaction(&staging, dest_dir, version)?;
1711    if let Err(err) =
1712        ensure_source_operation_not_interrupted("after promoting stable kernel source")
1713    {
1714        return Err(promoted.rollback(err));
1715    }
1716    Ok(promoted.commit())
1717}
1718
1719/// Verify a kernel tarball's staged extraction contains exactly one
1720/// top-level entry named `linux-{version}/` and atomically rename it
1721/// into `dest_dir/linux-{version}`. Bails — leaving `dest_dir`
1722/// untouched — when the staging dir holds a stray entry, when the
1723/// expected inner directory is missing, or when the rename fails.
1724/// The caller's `TempDir` outlives this helper, so its Drop sweeps
1725/// any residual staging contents whether this returns Ok or Err.
1726#[cfg(test)]
1727fn promote_staged_kernel_tree(
1728    staging: &tempfile::TempDir,
1729    dest_dir: &Path,
1730    version: &str,
1731) -> Result<PathBuf> {
1732    Ok(promote_staged_kernel_tree_transaction(staging, dest_dir, version)?.commit())
1733}
1734
1735fn promote_staged_kernel_tree_transaction(
1736    staging: &tempfile::TempDir,
1737    dest_dir: &Path,
1738    version: &str,
1739) -> Result<PromotedPath> {
1740    let expected_name = format!("linux-{version}");
1741    let mut found_inner = false;
1742    for entry in std::fs::read_dir(staging.path()).with_context(|| "read staging dir entries")? {
1743        let entry = entry.with_context(|| "iterate staging dir entry")?;
1744        let name = entry.file_name();
1745        if name == std::ffi::OsStr::new(&expected_name) {
1746            found_inner = true;
1747        } else {
1748            anyhow::bail!(
1749                "tarball contains unexpected top-level entry {name:?}; \
1750                 expected only {expected_name}/"
1751            );
1752        }
1753    }
1754    if !found_inner {
1755        anyhow::bail!("expected directory {expected_name} after extraction");
1756    }
1757    let inner = staging.path().join(&expected_name);
1758    let source_dir = dest_dir.join(&expected_name);
1759    promote_path_noreplace(&inner, &source_dir, PublishedPathKind::Directory)
1760}
1761
1762/// Promote the single top-level directory a codeload archive extracts
1763/// out of `staging` into `dest_dir/{canonical}`, so it survives
1764/// `staging`'s `Drop`.
1765///
1766/// Unlike [`promote_staged_kernel_tree`], the top-dir name is not
1767/// `linux-{version}` — GitHub derives it from the ref (`linux-6.11.11`
1768/// for a tag, `linux-{sha}` for a commit, `linux-{branch}` for a
1769/// branch), so this promotes the SOLE entry by structure rather than by
1770/// a fixed name, renaming it to a caller-supplied `canonical` name that
1771/// keys off the resolved commit (collision-free across refs). A hostile
1772/// or malformed snapshot that deposits zero or several top-level
1773/// entries — or a top-level entry that is not a plain directory (a
1774/// regular file, or a symlink, which the directory-entry file-type
1775/// check rejects rather than following) — is rejected before anything
1776/// lands in `dest_dir`; the `TempDir`'s `Drop` sweeps every entry the
1777/// archive left.
1778#[cfg(test)]
1779fn promote_single_kernel_tree(
1780    staging: &tempfile::TempDir,
1781    dest_dir: &Path,
1782    canonical: &str,
1783) -> Result<PathBuf> {
1784    Ok(promote_single_kernel_tree_transaction(staging, dest_dir, canonical)?.commit())
1785}
1786
1787fn promote_single_kernel_tree_transaction(
1788    staging: &tempfile::TempDir,
1789    dest_dir: &Path,
1790    canonical: &str,
1791) -> Result<PromotedPath> {
1792    let mut entries = Vec::new();
1793    for entry in std::fs::read_dir(staging.path()).with_context(|| "read staging dir entries")? {
1794        entries.push(entry.with_context(|| "iterate staging dir entry")?);
1795    }
1796    if entries.len() != 1 {
1797        anyhow::bail!(
1798            "codeload archive must contain exactly one top-level entry; found {}",
1799            entries.len()
1800        );
1801    }
1802    let inner = entries[0].path();
1803    // Use the DIRECTORY-ENTRY file type (does NOT follow symlinks) so a
1804    // top-level symlink-to-directory is rejected rather than promoted:
1805    // `Path::is_dir()` would follow the link and accept an
1806    // attacker-chosen target, and renameat2 moves the symlink itself
1807    // (it never dereferences), leaving the build reading through it.
1808    let entry_type = entries[0]
1809        .file_type()
1810        .with_context(|| "stat codeload top-level entry")?;
1811    if !entry_type.is_dir() {
1812        anyhow::bail!(
1813            "codeload archive top-level entry is not a plain directory: {}",
1814            inner.display()
1815        );
1816    }
1817    let source_dir = dest_dir.join(canonical);
1818    promote_path_noreplace(&inner, &source_dir, PublishedPathKind::Directory)
1819}
1820
1821#[derive(Clone, Copy)]
1822enum PublishedPathKind {
1823    File,
1824    Directory,
1825}
1826
1827struct PromotedPath {
1828    path: PathBuf,
1829    kind: PublishedPathKind,
1830    committed: bool,
1831}
1832
1833impl PromotedPath {
1834    fn commit(mut self) -> PathBuf {
1835        self.committed = true;
1836        self.path.clone()
1837    }
1838
1839    fn rollback(mut self, primary: anyhow::Error) -> anyhow::Error {
1840        match self.remove() {
1841            Ok(()) => primary,
1842            Err(rollback_err) => primary.context(format!(
1843                "also failed to roll back transaction-owned publication {}: {rollback_err}",
1844                self.path.display()
1845            )),
1846        }
1847    }
1848
1849    fn remove(&mut self) -> std::io::Result<()> {
1850        let result = match self.kind {
1851            PublishedPathKind::File => std::fs::remove_file(&self.path),
1852            PublishedPathKind::Directory => std::fs::remove_dir_all(&self.path),
1853        };
1854        if result.is_ok()
1855            || result
1856                .as_ref()
1857                .is_err_and(|err| err.kind() == std::io::ErrorKind::NotFound)
1858        {
1859            self.committed = true;
1860            Ok(())
1861        } else {
1862            result
1863        }
1864    }
1865}
1866
1867impl Drop for PromotedPath {
1868    fn drop(&mut self) {
1869        if !self.committed {
1870            let _ = self.remove();
1871        }
1872    }
1873}
1874
1875/// Atomically publish a transaction-owned path without replacing an
1876/// existing destination. A signal landing across the rename removes
1877/// only the path this successful `RENAME_NOREPLACE` created; failure to
1878/// roll it back is attached to the cancellation error instead of being
1879/// silently discarded.
1880fn promote_path_noreplace(
1881    staged: &Path,
1882    destination: &Path,
1883    kind: PublishedPathKind,
1884) -> Result<PromotedPath> {
1885    ensure_source_operation_not_interrupted("before publishing acquired source")?;
1886    rustix::fs::renameat_with(
1887        rustix::fs::CWD,
1888        staged,
1889        rustix::fs::CWD,
1890        destination,
1891        rustix::fs::RenameFlags::NOREPLACE,
1892    )
1893    .map_err(|err| {
1894        anyhow!(
1895            "renameat2(RENAME_NOREPLACE) {} -> {}: {err}",
1896            staged.display(),
1897            destination.display()
1898        )
1899    })?;
1900
1901    if let Err(interrupted) =
1902        ensure_source_operation_not_interrupted("after publishing acquired source")
1903    {
1904        let promoted = PromotedPath {
1905            path: destination.to_path_buf(),
1906            kind,
1907            committed: false,
1908        };
1909        return Err(promoted.rollback(interrupted));
1910    }
1911    Ok(PromotedPath {
1912        path: destination.to_path_buf(),
1913        kind,
1914        committed: false,
1915    })
1916}
1917
1918/// Download an RC kernel tarball (.tar.gz) from git.kernel.org.
1919///
1920/// Streams the body through a [`DownloadStream`] watchdog so a
1921/// stalled connection surfaces as an error rather than blocking
1922/// indefinitely. RC tarballs are dynamically generated by gitweb
1923/// at request time and have no published `sha256sums` manifest, so
1924/// this path always logs a warning that the digest is unverified —
1925/// it is computed and surfaced for diagnostic value (operators can
1926/// pin it manually) but never compared to an authoritative source.
1927fn download_rc_tarball(
1928    client: &Client,
1929    version: &str,
1930    dest_dir: &Path,
1931    cli_label: &str,
1932    mp: Option<&crate::cli::FetchProgress>,
1933) -> Result<PathBuf> {
1934    let url = format!("https://git.kernel.org/torvalds/t/linux-{version}.tar.gz");
1935    tracing::info!(%url, "downloading RC kernel tarball (requires network)");
1936
1937    let response = get_with_transient_retry(
1938        client,
1939        &url,
1940        Some(DOWNLOAD_REQUEST_READ_TIMEOUT),
1941        "download",
1942        &TRANSIENT_HTTP_RETRY,
1943    )?;
1944    if response.status() == reqwest::StatusCode::NOT_FOUND {
1945        anyhow::bail!(
1946            "RC tarball not found: {url}\n  \
1947             RC releases are removed from git.kernel.org after the stable version ships."
1948        );
1949    }
1950    if !response.status().is_success() {
1951        anyhow::bail!("download {url}: HTTP {}", response.status());
1952    }
1953    reject_html_response(&response, &url)?;
1954    print_download_size(&response, &url, cli_label, mp);
1955    // RC tarballs are gitweb-generated and often arrive without a
1956    // Content-Length, so `total` is frequently `None` and the bar
1957    // degrades to a live byte counter (rate, no ETA).
1958    let total = response.content_length();
1959
1960    let status = |line: &str| match mp {
1961        Some(fp) => fp.println(line),
1962        None => crate::cli::print_status_line(line),
1963    };
1964    status(&format!("{cli_label}: extracting tarball (gzip)"));
1965    // Stage extraction inside `dest_dir` (same filesystem) so the
1966    // final atomic rename keeps `dest_dir` clean when a bad mirror
1967    // serves a wrong-version archive or sneaks stray top-level
1968    // entries past the archive boundary. RC tarballs have no
1969    // upstream sha256 manifest, so structural verification is the
1970    // only defence against a hostile gitweb response.
1971    let staging =
1972        tempfile::TempDir::new_in(dest_dir).with_context(|| "create extraction staging dir")?;
1973    let download_bar = mp.map(|fp| fp.download_bar(version, total));
1974    let stream = DownloadStream::with_progress(response, download_bar.as_ref().map(|b| b.bar()));
1975    let decoder = flate2::read::GzDecoder::new(stream);
1976    let mut archive = tar::Archive::new(decoder);
1977    ensure_source_operation_not_interrupted("before extracting RC kernel tarball")?;
1978    archive
1979        .unpack(staging.path())
1980        .with_context(|| "extract tarball")?;
1981    ensure_source_operation_not_interrupted("after extracting RC kernel tarball")?;
1982
1983    // Surface the streamed digest as a warning. RC tarballs have
1984    // no upstream manifest, so verification is impossible — but
1985    // emitting the hash gives an operator a value they can
1986    // capture for offline pinning if they want to detect drift on
1987    // re-fetch.
1988    let stream = archive.into_inner().into_inner();
1989    let (actual_hex, bytes_total) = stream.finalize();
1990    if let Some(bar) = &download_bar {
1991        bar.finish();
1992    }
1993    tracing::warn!(
1994        url = %url,
1995        bytes = bytes_total,
1996        sha256 = %actual_hex,
1997        "no expected sha256 available for {url} (RC tarballs are \
1998         dynamically generated by git.kernel.org and have no \
1999         published manifest); computed digest {actual_hex} over \
2000         {bytes_total} bytes is unverified",
2001    );
2002
2003    ensure_source_operation_not_interrupted("before promoting RC kernel source")?;
2004    let promoted = promote_staged_kernel_tree_transaction(&staging, dest_dir, version)?;
2005    if let Err(err) = ensure_source_operation_not_interrupted("after promoting RC kernel source") {
2006        return Err(promoted.rollback(err));
2007    }
2008    Ok(promoted.commit())
2009}
2010
2011/// Download a GitHub source snapshot for `git_ref` as a codeload
2012/// `tar.gz` and extract it, returning an [`AcquiredSource`] keyed as an
2013/// explicit immutable-SHA snapshot ([`git_cache_key`] over the resolved
2014/// `commit_hash`). Archive SHA, branch, and tag identities remain
2015/// distinct even when they currently point at the same commit.
2016///
2017/// GitHub serves a gzip snapshot for any tag/branch/commit via
2018/// codeload; the caller supplies the `archive_url`
2019/// ([`github_archive_url`]) and the pre-resolved `commit_hash`
2020/// ([`resolve_ref_commit`]) — the snapshot has no `.git`, so the
2021/// commit cannot be read back from the tree. Modeled on
2022/// [`download_rc_tarball`] (gzip decode; codeload carries no sha256
2023/// manifest, so extraction is structurally verified —
2024/// [`promote_single_kernel_tree_transaction`] rejects any top level that is not a
2025/// single plain directory (multi-entry, a file, or a symlink) — and
2026/// the streamed digest is logged, not compared).
2027pub(crate) fn download_github_archive(
2028    client: &Client,
2029    archive_url: &str,
2030    git_ref: &str,
2031    commit_hash: &str,
2032    dest_dir: &Path,
2033    cli_label: &str,
2034    mp: Option<&crate::cli::FetchProgress>,
2035) -> Result<AcquiredSource> {
2036    let progress_label = format!("{cli_label}: {git_ref}");
2037    let snapshot_progress = match mp {
2038        Some(fp) => fp.operation_progress(&progress_label, "fetching snapshot", "snapshot fetch"),
2039        None => crate::cli::progress::CloneProgress::standalone_operation(
2040            &progress_label,
2041            "fetching snapshot",
2042            "snapshot fetch",
2043        ),
2044    };
2045    // Create the phase before issuing the request. The blocking HTTP
2046    // header wait itself has no byte counter, so this live item is what
2047    // lets CloneProgress emit its first phase immediately and a
2048    // ten-second heartbeat for as long as the server stays silent.
2049    let request_phase = snapshot_progress.item_named("requesting snapshot");
2050    tracing::info!(%archive_url, "downloading GitHub codeload snapshot (requires network)");
2051    let response = get_with_transient_retry(
2052        client,
2053        archive_url,
2054        Some(DOWNLOAD_REQUEST_READ_TIMEOUT),
2055        "download",
2056        &TRANSIENT_HTTP_RETRY,
2057    )?;
2058    drop(request_phase);
2059    if response.status() == reqwest::StatusCode::NOT_FOUND {
2060        anyhow::bail!(
2061            "codeload snapshot not found: {archive_url}\n  \
2062             the ref may not exist on the remote, or the repo is private"
2063        );
2064    }
2065    if !response.status().is_success() {
2066        anyhow::bail!("download {archive_url}: HTTP {}", response.status());
2067    }
2068    reject_html_response(&response, archive_url)?;
2069    print_download_size(&response, archive_url, cli_label, mp);
2070    // codeload responses are dynamically generated and often arrive
2071    // without a Content-Length, so `total` is frequently `None` and the
2072    // bar degrades to a live byte counter.
2073    let total = response.content_length();
2074
2075    let status = |line: &str| match mp {
2076        Some(fp) => fp.println(line),
2077        None => crate::cli::print_status_line(line),
2078    };
2079    status(&format!("{cli_label}: extracting snapshot (gzip)"));
2080    // Stage extraction inside `dest_dir` (same filesystem) so the final
2081    // atomic rename keeps `dest_dir` clean when a bad response serves a
2082    // malformed archive or sneaks stray top-level entries. codeload
2083    // snapshots have no upstream sha256 manifest, so structural
2084    // verification (single top-level dir) is the only defence against a
2085    // hostile response.
2086    let staging =
2087        tempfile::TempDir::new_in(dest_dir).with_context(|| "create extraction staging dir")?;
2088    let short_hash: String = commit_hash.chars().take(7).collect();
2089    let download_bar = mp.map(|fp| fp.download_bar(git_ref, total));
2090    // tar::Archive streams decompression directly from the response, so
2091    // download and extraction intentionally overlap instead of writing
2092    // a second full archive to disk. Name that fused phase truthfully;
2093    // the reporter continues to heartbeat even while gzip/tar is doing
2094    // CPU or filesystem work between response reads.
2095    let stream_phase = snapshot_progress.item_named("streaming and extracting snapshot");
2096    let stream = DownloadStream::with_progress(response, download_bar.as_ref().map(|b| b.bar()));
2097    let decoder = flate2::read::GzDecoder::new(stream);
2098    let mut archive = tar::Archive::new(decoder);
2099    ensure_source_operation_not_interrupted("before extracting GitHub snapshot")?;
2100    archive
2101        .unpack(staging.path())
2102        .with_context(|| "extract snapshot")?;
2103    ensure_source_operation_not_interrupted("after extracting GitHub snapshot")?;
2104
2105    // Drain the watchdog to read the streamed digest. codeload has no
2106    // published manifest, so the digest cannot be verified — log it so
2107    // an operator can capture it for offline pinning. `into_inner` peels
2108    // the tar then the gz layer, recovering the `DownloadStream`.
2109    let stream = archive.into_inner().into_inner();
2110    let (actual_hex, bytes_total) = stream.finalize();
2111    drop(stream_phase);
2112    if let Some(bar) = &download_bar {
2113        bar.finish();
2114    }
2115    tracing::info!(
2116        url = %archive_url,
2117        bytes = bytes_total,
2118        sha256 = %actual_hex,
2119        "codeload snapshot extracted (unverified: codeload archives have \
2120         no published sha256 manifest)",
2121    );
2122
2123    // Name the promoted tree by the resolved commit so distinct refs
2124    // never collide in `dest_dir` (the tree is temporary — `is_temp`).
2125    let canonical = format!("linux-git-{short_hash}");
2126    ensure_source_operation_not_interrupted("before promoting GitHub snapshot")?;
2127    let promoted = promote_single_kernel_tree_transaction(&staging, dest_dir, &canonical)?;
2128    if let Err(err) = ensure_source_operation_not_interrupted("after promoting GitHub snapshot") {
2129        return Err(promoted.rollback(err));
2130    }
2131    let source_dir = promoted.path.clone();
2132    ensure_source_operation_not_interrupted("before reading GitHub snapshot metadata")?;
2133    let version = read_makefile_version(&source_dir);
2134    if let Err(err) =
2135        ensure_source_operation_not_interrupted("after reading GitHub snapshot metadata")
2136    {
2137        return Err(promoted.rollback(err));
2138    }
2139
2140    let acquired = AcquiredSource {
2141        source_dir,
2142        cache_key: git_cache_key(crate::kernel_path::GitRefKind::Sha, git_ref, commit_hash),
2143        version,
2144        kernel_source: crate::cache::KernelSource::git(short_hash, git_ref),
2145        is_temp: true,
2146        is_dirty: false,
2147        is_git: true,
2148    };
2149    let _source_dir = promoted.commit();
2150    snapshot_progress.finish();
2151    Ok(acquired)
2152}
2153
2154/// Download a kernel tarball (stable or RC) and extract it.
2155///
2156/// `cli_label` prefixes diagnostic status output (e.g. `"ktstr"` or
2157/// `"cargo ktstr"`).
2158///
2159/// `skip_sha256` propagates to `download_stable_tarball` only —
2160/// stable tarballs publish a `sha256sums.asc` manifest the flag
2161/// bypasses. RC tarballs (`download_rc_tarball`) have no published
2162/// manifest so verification is impossible regardless of the flag;
2163/// the RC path always runs unverified and emits its own warning,
2164/// so `skip_sha256` is a no-op on the RC arm. `--kernel <path>` and
2165/// `--kernel git+…` sources do not reach this function at all.
2166///
2167/// `mp` is the progress group the determinate download bar is added
2168/// to; `None` disables the bar (the single-shot `kernel build` paths
2169/// and unit tests pass `None`).
2170pub fn download_tarball(
2171    client: &Client,
2172    version: &str,
2173    dest_dir: &Path,
2174    cli_label: &str,
2175    skip_sha256: bool,
2176    mp: Option<&crate::cli::FetchProgress>,
2177) -> Result<AcquiredSource> {
2178    let (arch, _) = arch_info();
2179    let source_dir = if is_rc(version) {
2180        download_rc_tarball(client, version, dest_dir, cli_label, mp)?
2181    } else {
2182        match download_stable_tarball(client, version, dest_dir, cli_label, skip_sha256, mp) {
2183            Ok(dir) => dir,
2184            // Pruned tarball (EOL series or superseded point release):
2185            // cdn.kernel.org keeps only each maintained series' latest
2186            // .tar.xz. Recover the source from the stable tree's
2187            // `v{version}` tag via a shallow (depth-1) clone. The kernel
2188            // built from this source is cached by the caller under the
2189            // SAME `{version}-tarball-...` key returned below, so a
2190            // re-run hits that cache and never re-clones.
2191            Err(e) if e.downcast_ref::<TarballNotFound>().is_some() => {
2192                let tag = format!("v{version}");
2193                // A 404 says the tarball is gone, not why. cdn.kernel.org
2194                // keeps only the latest tarball per series, but the gregkh
2195                // GitHub mirror carries every `vX.Y.Z` release tag and
2196                // codeload serves the tag's snapshot as a tar.gz — no
2197                // clone, and a commit-pinned snapshot. Resolve the tag to
2198                // its commit first (kind-directed, so a tag never aliases
2199                // a same-named branch); a tag absent there means the
2200                // version simply does not exist — surface the friendly
2201                // "not found" suggestion (with the latest in-series patch)
2202                // instead of a cryptic fetch failure.
2203                let Some(commit_id) = resolve_ref_commit(
2204                    STABLE_MIRROR_URL,
2205                    &tag,
2206                    crate::kernel_path::GitRefKind::Tag,
2207                    cli_label,
2208                    mp,
2209                )?
2210                else {
2211                    anyhow::bail!("{}", version_not_found_msg(client, version));
2212                };
2213                let commit_hash = format!("{commit_id}");
2214                let archive_url = github_archive_url(STABLE_MIRROR_URL, &commit_hash)
2215                    .expect("STABLE_MIRROR_URL is a github.com URL");
2216                let msg = format!(
2217                    "{cli_label}: {version} not on cdn.kernel.org (pruned/EOL); \
2218                     using the GitHub source-snapshot fallback for gregkh mirror tag {tag}"
2219                );
2220                match mp {
2221                    Some(fp) => fp.println(&msg),
2222                    None => crate::cli::print_status_line(&msg),
2223                }
2224                download_github_archive(
2225                    client,
2226                    &archive_url,
2227                    &tag,
2228                    &commit_hash,
2229                    dest_dir,
2230                    cli_label,
2231                    mp,
2232                )?
2233                .source_dir
2234            }
2235            Err(e) => return Err(e),
2236        }
2237    };
2238
2239    Ok(AcquiredSource {
2240        source_dir,
2241        cache_key: format!("{version}-tarball-{arch}-kc{}", crate::cache_key_suffix()),
2242        version: Some(version.to_string()),
2243        kernel_source: crate::cache::KernelSource::Tarball,
2244        is_temp: true,
2245        is_dirty: false,
2246        is_git: true,
2247    })
2248}
2249
2250/// Download a single file from `url` to `dest`, streaming through the
2251/// `DownloadStream` watchdog + progress bar and verifying the
2252/// resulting SHA-256 against `expected_sha256` (hex, case-insensitive).
2253///
2254/// Used by prebuilt-distro-kernel acquisition
2255/// (`crate::distro::acquire`) to fetch each `.rpm`/`.deb` package and
2256/// its (up to ~1 GiB) debuginfo. Rides the same transient-retry seam
2257/// (`TRANSIENT_HTTP_RETRY`) and no-progress watchdog
2258/// (`DOWNLOAD_NO_PROGRESS_TIMEOUT`, which tracks body progress rather
2259/// than total time so a large but progressing download is never
2260/// aborted) as the kernel-source tarball path. Unlike the tarball
2261/// downloaders it does not extract — the bytes land verbatim at `dest`
2262/// for the caller to unpack. `label` names the package on its progress
2263/// bar; `cli_label` prefixes the size status line.
2264pub fn download_verified_file(
2265    url: &str,
2266    dest: &Path,
2267    expected_sha256: &str,
2268    label: &str,
2269    cli_label: &str,
2270    mp: Option<&crate::cli::FetchProgress>,
2271) -> Result<()> {
2272    download_verified_file_with_checksum(
2273        url,
2274        dest,
2275        DownloadChecksum::Sha256(expected_sha256),
2276        label,
2277        cli_label,
2278        mp,
2279    )
2280}
2281
2282/// Download a single file with the shared retry/watchdog/progress
2283/// pipeline and verify its Google Cloud Storage `md5Hash` (standard
2284/// base64). This is the GKE artifact counterpart to
2285/// [`download_verified_file`]; the transport and progress behavior are
2286/// deliberately identical.
2287pub(crate) fn download_verified_md5_file(
2288    url: &str,
2289    dest: &Path,
2290    expected_md5_base64: &str,
2291    label: &str,
2292    cli_label: &str,
2293    mp: Option<&crate::cli::FetchProgress>,
2294) -> Result<()> {
2295    download_verified_file_with_checksum(
2296        url,
2297        dest,
2298        DownloadChecksum::Md5Base64(expected_md5_base64),
2299        label,
2300        cli_label,
2301        mp,
2302    )
2303}
2304
2305#[derive(Clone, Copy)]
2306enum DownloadChecksum<'a> {
2307    Sha256(&'a str),
2308    Md5Base64(&'a str),
2309}
2310
2311fn download_verified_file_with_checksum(
2312    url: &str,
2313    dest: &Path,
2314    checksum: DownloadChecksum<'_>,
2315    label: &str,
2316    cli_label: &str,
2317    mp: Option<&crate::cli::FetchProgress>,
2318) -> Result<()> {
2319    let client = shared_client();
2320    tracing::info!(%url, "downloading verified kernel artifact (requires network)");
2321    let response = get_with_transient_retry(
2322        client,
2323        url,
2324        Some(DOWNLOAD_REQUEST_READ_TIMEOUT),
2325        "download",
2326        &TRANSIENT_HTTP_RETRY,
2327    )?;
2328    if response.status() == reqwest::StatusCode::NOT_FOUND {
2329        anyhow::bail!("package not found: {url}");
2330    }
2331    if !response.status().is_success() {
2332        anyhow::bail!("download {url}: HTTP {}", response.status());
2333    }
2334    reject_html_response(&response, url)?;
2335    print_download_size(&response, url, cli_label, mp);
2336    let total = response.content_length();
2337    let download_bar = mp.map(|fp| fp.download_bar(label, total));
2338    let parent = dest
2339        .parent()
2340        .ok_or_else(|| anyhow!("download destination has no parent: {}", dest.display()))?;
2341    let staging = tempfile::TempDir::new_in(parent)
2342        .with_context(|| format!("create download staging directory in {}", parent.display()))?;
2343    let staged_file = staging.path().join("artifact");
2344    let staged_result = (|| -> Result<()> {
2345        let mut stream =
2346            DownloadStream::with_progress(response, download_bar.as_ref().map(|b| b.bar()));
2347        let mut file = std::fs::File::create(&staged_file)
2348            .with_context(|| format!("create {}", staged_file.display()))?;
2349        std::io::copy(&mut stream, &mut file)
2350            .with_context(|| format!("stream {url} to {}", staged_file.display()))?;
2351        ensure_source_operation_not_interrupted("after writing downloaded artifact")?;
2352        if let Some(bar) = &download_bar {
2353            bar.finish();
2354        }
2355        match checksum {
2356            DownloadChecksum::Sha256(expected) => {
2357                let (actual, _bytes) = stream.finalize();
2358                verify_sha256(&actual, expected, url)?;
2359            }
2360            DownloadChecksum::Md5Base64(expected) => {
2361                let (actual, _bytes) = stream.finalize_md5_base64();
2362                verify_md5_base64(&actual, expected, url)?;
2363            }
2364        }
2365        drop(file);
2366        Ok(())
2367    })();
2368    if let Err(err) = staged_result {
2369        return Err(close_private_staging(staging, err));
2370    }
2371    let promoted = match promote_path_noreplace(&staged_file, dest, PublishedPathKind::File) {
2372        Ok(promoted) => promoted,
2373        Err(err) => return Err(close_private_staging(staging, err)),
2374    };
2375    drop(staging);
2376    promoted.commit();
2377    Ok(())
2378}
2379
2380fn verify_md5_base64(actual: &str, expected: &str, url: &str) -> Result<()> {
2381    if actual == expected {
2382        Ok(())
2383    } else {
2384        anyhow::bail!(
2385            "md5 mismatch for {url}: expected {expected}, got {actual}. \
2386             The object generation may have changed; retry so ktstr can \
2387             resolve fresh Google Cloud Storage metadata."
2388        );
2389    }
2390}
2391
2392/// Parse the patch level from a kernel version string.
2393/// "6.12.8" → Some(8), "7.0" → Some(0), "abc" → None.
2394fn patch_level(version: &str) -> Option<u32> {
2395    let parts: Vec<&str> = version.split('.').collect();
2396    match parts.len() {
2397        2 => Some(0), // "7.0" has patch level 0
2398        3 => parts[2].parse().ok(),
2399        _ => None,
2400    }
2401}
2402
2403/// Production URL for `releases.json`. Transport tests may pass a local URL
2404/// directly to [`fetch_releases`].
2405pub(crate) const RELEASES_URL: &str = "https://www.kernel.org/releases.json";
2406
2407/// Fetch `releases.json` from `url` and return a vector of
2408/// [`Release`] records. Issues an HTTP GET unconditionally — no
2409/// cache consultation.
2410///
2411/// Production callers reach this function via
2412/// [`cached_releases_with`] (or [`cached_releases`]) which pass
2413/// [`RELEASES_URL`]; the cache helper only invokes
2414/// `fetch_releases` on a cache miss for the singleton path or on
2415/// the bypass branch for non-singleton clients. Tests that need
2416/// to exercise the underlying GET directly — without the cache
2417/// layer — call this function with a locally-constructed `Client`
2418/// and a localhost URL pointed at a TcpListener-backed mock that
2419/// returns canned `releases.json` content.
2420pub(crate) fn fetch_releases(client: &Client, url: &str) -> Result<Vec<Release>> {
2421    tracing::info!(%url, "fetching kernel.org releases index (requires network)");
2422    let response = get_with_transient_retry(
2423        client,
2424        url,
2425        Some(SMALL_RESPONSE_REQUEST_TIMEOUT),
2426        "fetch",
2427        &TRANSIENT_HTTP_RETRY,
2428    )?;
2429    if !response.status().is_success() {
2430        anyhow::bail!("fetch {url}: HTTP {}", response.status());
2431    }
2432    let body = String::from_utf8(read_response_bytes(response, url)?)
2433        .with_context(|| "decode response body as UTF-8")?;
2434    parse_releases_body(&body)
2435}
2436
2437fn parse_releases_body(body: &str) -> Result<Vec<Release>> {
2438    let json: serde_json::Value =
2439        serde_json::from_str(body).with_context(|| "parse releases.json")?;
2440    let releases = json
2441        .get("releases")
2442        .and_then(|r| r.as_array())
2443        .ok_or_else(|| anyhow!("releases.json: missing releases array"))?;
2444    let input_rows = releases.len();
2445    let parsed: Vec<Release> = releases
2446        .iter()
2447        .filter_map(|r| {
2448            let moniker = r.get("moniker")?.as_str()?;
2449            let version = r.get("version")?.as_str()?;
2450            Some(Release {
2451                moniker: moniker.to_string(),
2452                version: version.to_string(),
2453            })
2454        })
2455        .collect();
2456    // Per-row tolerance: a corrupt row is silently dropped via the
2457    // filter_map `?` chain so a single bad entry does not abort the
2458    // whole fetch (see `fetch_releases_row_missing_moniker_drops_row`
2459    // and siblings). The drop is also a hazard: the truncated vector
2460    // gets cached in [`RELEASES_CACHE`] for the rest of the process
2461    // lifetime via the singleton path, so a transient malformed row
2462    // at fetch time persists as a partial snapshot for every later
2463    // cache-hit caller. Surface the drop count so an operator
2464    // tailing logs sees that releases.json arrived partial — without
2465    // this, the symptom (a missing version on resolve) is invisible
2466    // until it propagates as "version not found" elsewhere.
2467    let dropped = input_rows - parsed.len();
2468    if dropped > 0 {
2469        tracing::warn!(
2470            input_rows,
2471            parsed_rows = parsed.len(),
2472            dropped,
2473            "releases.json: dropped {dropped} of {input_rows} row(s) \
2474             missing moniker/version (or non-string values); cached \
2475             snapshot will reflect this for the process lifetime"
2476        );
2477    }
2478    Ok(parsed)
2479}
2480
2481/// Fetch the latest stable kernel version from kernel.org.
2482///
2483/// Selects from the `releases` array (moniker "stable" or "longterm"),
2484/// requiring patch version >= 8 to avoid brand-new major versions
2485/// that may have build issues on CI runners.
2486///
2487/// When `client` is the process-wide [`shared_client`] singleton,
2488/// routes through `RELEASES_CACHE`; other clients bypass the
2489/// cache via pointer-equality and exercise `fetch_releases`
2490/// directly — see `cached_releases_with` for details.
2491///
2492/// `cli_label` prefixes diagnostic status output (e.g. `"ktstr"` or
2493/// `"cargo ktstr"`).
2494pub fn fetch_latest_stable_version(client: &Client, cli_label: &str) -> Result<String> {
2495    crate::ktstr_status!("{cli_label}: fetching latest kernel version");
2496    let releases = cached_releases_with(client)?;
2497
2498    let mut best: Option<&str> = None;
2499    for r in &releases {
2500        if r.moniker != "stable" && r.moniker != "longterm" {
2501            continue;
2502        }
2503        if patch_level(&r.version).unwrap_or(0) < 8 {
2504            continue;
2505        }
2506        // Pick the first matching release — releases.json is ordered
2507        // newest first, so the first stable with patch >= 8 is the best.
2508        best = Some(r.version.as_str());
2509        break;
2510    }
2511
2512    let version =
2513        best.ok_or_else(|| anyhow!("no stable kernel with patch >= 8 found in releases.json"))?;
2514    crate::ktstr_status!("{cli_label}: latest stable kernel: {version}");
2515    Ok(version.to_string())
2516}
2517
2518/// Parse a version string into numeric components for comparison.
2519/// "6.14.2" → Some((6, 14, 2)), "6.14" → Some((6, 14, 0)),
2520/// "7.0" → Some((7, 0, 0)). Returns None for unparseable versions.
2521fn version_tuple(version: &str) -> Option<(u32, u32, u32)> {
2522    let parts: Vec<&str> = version.split('.').collect();
2523    match parts.len() {
2524        2 => {
2525            let major = parts[0].parse().ok()?;
2526            let minor = parts[1].parse().ok()?;
2527            Some((major, minor, 0))
2528        }
2529        3 => {
2530            let major = parts[0].parse().ok()?;
2531            let minor = parts[1].parse().ok()?;
2532            let patch = parts[2].parse().ok()?;
2533            Some((major, minor, patch))
2534        }
2535        _ => None,
2536    }
2537}
2538
2539/// Return true when `s` is a kernel major.minor prefix like
2540/// `"6.14"` (as opposed to a full patch version `"6.14.2"` or an rc
2541/// tag `"6.15-rc3"`). Callers use this to decide whether the input
2542/// needs prefix resolution via [`fetch_version_for_prefix`].
2543///
2544/// Accepts any string with fewer than 2 dots and no `-rc` substring,
2545/// so `"7"` (single-segment) and `""` both return true. This matches
2546/// the historical inline check used by kernel-build dispatchers.
2547pub fn is_major_minor_prefix(s: &str) -> bool {
2548    s.matches('.').count() < 2 && !s.contains("-rc")
2549}
2550
2551/// Resolve the highest version matching a prefix.
2552///
2553/// E.g., "6.12" → "6.12.81", "6" → "6.19.12" (highest 6.x.y).
2554///
2555/// Scans all monikers in releases.json except linux-next. On no active
2556/// match (an EOL or unreleased series, absent from releases.json),
2557/// resolves the highest `vX.Y.z` stable patch from the gregkh mirror's
2558/// git tags; if the series has NO stable point release yet (only a base
2559/// tag), falls back to the bare `{prefix}` mainline base — see
2560/// `latest_patch_from_git_tags`.
2561///
2562/// When `client` is the process-wide [`shared_client`] singleton,
2563/// routes through `RELEASES_CACHE`; other clients bypass the
2564/// cache via pointer-equality and exercise `fetch_releases`
2565/// directly — see `cached_releases_with` for details. Cache
2566/// scope is releases.json only; the EOL-series git-tag fallback in
2567/// `latest_patch_from_git_tags` always hits the network.
2568///
2569/// `cli_label` prefixes diagnostic status output (e.g. `"ktstr"` or
2570/// `"cargo ktstr"`).
2571pub fn fetch_version_for_prefix(client: &Client, prefix: &str, cli_label: &str) -> Result<String> {
2572    crate::ktstr_status!("{cli_label}: fetching latest {prefix}.x kernel version");
2573    let releases = cached_releases_with(client)?;
2574
2575    let mut best: Option<(&str, (u32, u32, u32))> = None;
2576    for r in &releases {
2577        if is_skippable_release_moniker(&r.moniker) {
2578            continue;
2579        }
2580        if !r.version.starts_with(prefix) {
2581            continue;
2582        }
2583        if r.version.len() != prefix.len() && r.version.as_bytes()[prefix.len()] != b'.' {
2584            continue;
2585        }
2586        let Some(tuple) = version_tuple(&r.version) else {
2587            continue;
2588        };
2589        if best.is_none() || tuple > best.unwrap().1 {
2590            best = Some((r.version.as_str(), tuple));
2591        }
2592    }
2593
2594    if let Some((version, _)) = best {
2595        crate::ktstr_status!("{cli_label}: latest {prefix}.x kernel: {version}");
2596        return Ok(version.to_string());
2597    }
2598
2599    crate::ktstr_status!(
2600        "{cli_label}: {prefix}.x not in releases.json (EOL or unreleased series); \
2601         resolving latest patch via the gregkh mirror tags"
2602    );
2603    match latest_patch_from_git_tags(STABLE_MIRROR_URL, prefix, cli_label)? {
2604        Some(version) => Ok(version),
2605        None => {
2606            // No stable point release for this series — fall back to the
2607            // mainline base (the `{prefix}` release itself, e.g. a series
2608            // just cut with no `.1` yet, per the "only if there is no
2609            // X.Y.z stable use X.Y mainline" rule). The base tarball is
2610            // fetched by the normal download path (cdn.kernel.org,
2611            // falling back to the gregkh mirror snapshot); torvalds is
2612            // the mainline authority the gregkh mirror tracks.
2613            crate::ktstr_status!(
2614                "{cli_label}: no {prefix}.x stable point release; using {prefix} mainline base"
2615            );
2616            Ok(prefix.to_string())
2617        }
2618    }
2619}
2620
2621/// Resolve a series' latest stable patch by ls-remote-ing the gregkh
2622/// GitHub mirror's `refs/tags/v{prefix}.{patch}` tags and taking the
2623/// highest patch. Returns `Ok(None)` when the series has NO stable
2624/// point release (no `v{prefix}.N` tag) — the caller then falls back to
2625/// the mainline base.
2626///
2627/// The gregkh mirror is the RELIABLE EOL-resolution source: it carries
2628/// every `vX.Y.Z` release tag (back to v2.6) and its codeload CDN
2629/// serves each tag's tarball, so resolution and the pruned-tarball
2630/// download (see [`download_tarball`]'s fallback) share ONE
2631/// comprehensive mirror. cdn.kernel.org cannot be used here: its
2632/// `v{major}.x/` directory index 404s, and its `sha256sums.asc` is
2633/// served inconsistently per CDN edge (200 from some nodes, 404 from
2634/// others — the 404 nodes break CI runners while the tarball fetch on
2635/// those same nodes still succeeds).
2636fn latest_patch_from_git_tags(url: &str, prefix: &str, cli_label: &str) -> Result<Option<String>> {
2637    crate::ktstr_status!("{cli_label}: resolving {prefix}.x release tags via {url}");
2638    let refs = ls_remote_refs(url, &format!("{cli_label}: {prefix}.x tags"))
2639        .with_context(|| format!("ls-remote {url} for {prefix}.x release tags"))?;
2640    match max_tag_patch(refs.iter().map(ref_full_name), prefix) {
2641        Some(patch) => {
2642            let version = format!("{prefix}.{patch}");
2643            crate::ktstr_status!(
2644                "{cli_label}: latest {prefix}.x kernel (from git tags): {version}"
2645            );
2646            Ok(Some(version))
2647        }
2648        None => Ok(None),
2649    }
2650}
2651
2652/// The advertised full ref name (`refs/...`), as raw bytes, of a
2653/// protocol handshake ref.
2654fn ref_full_name(r: &gix::protocol::handshake::Ref) -> &[u8] {
2655    use gix::protocol::handshake::Ref::{Direct, Peeled, Symbolic, Unborn};
2656    match r {
2657        Peeled { full_ref_name, .. }
2658        | Direct { full_ref_name, .. }
2659        | Symbolic { full_ref_name, .. }
2660        | Unborn { full_ref_name, .. } => full_ref_name.as_ref(),
2661    }
2662}
2663
2664/// Highest `{patch}` among `refs/tags/v{prefix}.{patch}` ref names.
2665///
2666/// gix folds an annotated tag's peeled entry into a single
2667/// `Ref::Peeled` whose `full_ref_name` is the BASE name — no `^{}`
2668/// suffix — and a lightweight tag arrives as a `Ref::Direct` with the
2669/// base name too, so every tag advertises its base
2670/// `refs/tags/v{prefix}.{patch}` name for the needle to match. The
2671/// `^{}` strip below is therefore a defensive no-op on real gix output
2672/// (it only affects a raw wire ref name gix never emits; the base
2673/// entry supplies the patch regardless). Pure (no network) so it is
2674/// unit-testable with synthetic ref names.
2675///
2676/// The trailing `.` in the `refs/tags/v{prefix}.` needle keeps a
2677/// `6.14` prefix from matching a `6.140` series, and the numeric-only
2678/// patch tail rejects `-rc` and other non-release tags.
2679fn max_tag_patch<'a>(ref_names: impl Iterator<Item = &'a [u8]>, prefix: &str) -> Option<u32> {
2680    let needle = format!("refs/tags/v{prefix}.");
2681    let mut best: Option<u32> = None;
2682    for name in ref_names {
2683        let Some(rest) = name.strip_prefix(needle.as_bytes()) else {
2684            continue;
2685        };
2686        let rest = rest.strip_suffix(b"^{}").unwrap_or(rest);
2687        if let Ok(s) = std::str::from_utf8(rest)
2688            && let Ok(patch) = s.parse::<u32>()
2689        {
2690            best = Some(best.map_or(patch, |b| b.max(patch)));
2691        }
2692    }
2693    best
2694}
2695
2696/// ls-remote the gregkh stable mirror ([`STABLE_MIRROR_URL`]) once and
2697/// cache the release version strings (`X.Y.Z`) parsed from its
2698/// `refs/tags/vX.Y.Z` advertisement, for `--include-eol` range
2699/// expansion. Returns EVERY release-tag version verbatim (including
2700/// `-rc*` and old series); the caller
2701/// (`crate::cli::select_series_latest_in_range`) does the
2702/// range / rc / per-series filtering. `None` on ls-remote failure —
2703/// not cached, so the next caller retries. gregkh/linux mirrors
2704/// linux-stable comprehensively (tags back to v2.6), so this surfaces
2705/// EOL series that `releases.json` has dropped.
2706pub(crate) fn cached_stable_tags() -> Option<&'static [String]> {
2707    if let Some(tags) = STABLE_TAGS_CACHE.get() {
2708        return Some(tags.as_slice());
2709    }
2710    let refs = ls_remote_refs(STABLE_MIRROR_URL, "ktstr: stable release tags").ok()?;
2711    let tags: Vec<String> = refs
2712        .iter()
2713        .filter_map(|r| {
2714            // Base tag name only: gix folds an annotated tag's peeled
2715            // entry into one `Ref::Peeled` carrying the base name, and a
2716            // lightweight tag is a `Ref::Direct` with the base name, so
2717            // `^{}` never appears on real gix output — the strip is a
2718            // defensive no-op. Non-`refs/tags/v*` refs are skipped.
2719            let name = ref_full_name(r);
2720            let v = name.strip_prefix(b"refs/tags/v")?;
2721            let v = v.strip_suffix(b"^{}").unwrap_or(v);
2722            std::str::from_utf8(v).ok().map(|s| s.to_string())
2723        })
2724        .collect();
2725    // Loser of a concurrent race discards its clone (both fetched the
2726    // same advertisement, so the cached content is equivalent).
2727    let _ = STABLE_TAGS_CACHE.set(tags);
2728    STABLE_TAGS_CACHE.get().map(|v| v.as_slice())
2729}
2730
2731/// Cache key for a git-acquired kernel: the explicit ref kind, a
2732/// fixed-seed ahash of the length-delimited raw ref bytes, the resolved
2733/// commit's full hash, the target arch, and the kconfig-fragment suffix.
2734///
2735/// The kind is part of the identity because `#tag=next`,
2736/// `#branch=next`, and `#sha=...` are distinct acquisition contracts
2737/// even when they currently resolve to the same commit. Hashing the raw
2738/// bytes avoids both filesystem-hostile ref spellings and collisions
2739/// introduced by sanitizing (`a/b` versus `a_b`) while keeping the
2740/// cache key well below `NAME_MAX`. The zero-seeded ahash matches
2741/// ktstr's other fast content-addressed caches; this is an identity
2742/// accelerator, not a cryptographic integrity boundary.
2743///
2744/// The FULL 40-hex commit hash keys the entry (not a 7-hex prefix): a
2745/// branch/tag tip moves over time, so the `{git_ref}` segment alone
2746/// cannot distinguish successive commits — only the hash does. A 7-hex
2747/// (28-bit) prefix would let a moved tip whose new commit shares the
2748/// first 7 hex with the cached old commit hit the stale entry and serve
2749/// the wrong kernel build under the new ref. The full id removes that
2750/// collision class; the probe and clone both render full lowercase hex
2751/// before any truncation, so keying on it is drift-free.
2752pub(crate) fn git_cache_key(
2753    ref_kind: crate::kernel_path::GitRefKind,
2754    git_ref: &str,
2755    commit_hash: &str,
2756) -> String {
2757    use crate::kernel_path::GitRefKind;
2758
2759    let (arch, _) = arch_info();
2760    let (kind_name, kind_byte) = match ref_kind {
2761        GitRefKind::Tag => ("tag", 0u8),
2762        GitRefKind::Branch => ("branch", 1u8),
2763        GitRefKind::Sha => ("sha", 2u8),
2764        GitRefKind::Unknown => ("unknown", 3u8),
2765    };
2766    let canonical_ref = (ref_kind == GitRefKind::Sha).then(|| git_ref.to_ascii_lowercase());
2767    let ref_bytes = canonical_ref.as_deref().unwrap_or(git_ref).as_bytes();
2768    let mut hasher = ahash::RandomState::with_seeds(0, 0, 0, 0).build_hasher();
2769    hasher.write(&[kind_byte]);
2770    hasher.write(&(ref_bytes.len() as u64).to_le_bytes());
2771    hasher.write(ref_bytes);
2772    let ref_hash = hasher.finish();
2773    format!(
2774        "git-{kind_name}-{ref_hash:016x}-{}-{arch}-kc{}",
2775        commit_hash.to_ascii_lowercase(),
2776        crate::cache_key_suffix()
2777    )
2778}
2779
2780/// If `url` is a GitHub remote, build the codeload archive URL for the
2781/// resolved `commit_hash`: `github.com/OWNER/REPO/archive/<commit>.tar.gz`
2782/// (302 → codeload.github.com, its CDN) serves a gzip source snapshot
2783/// for any commit — verified empirically. `resolve_git_kernel` invokes
2784/// this only for an explicit immutable SHA; branches and tags use the
2785/// exact shallow gix path even on GitHub. `None` for a non-GitHub URL
2786/// (self-hosted / GitLab / …), where commit-SHA acquisition is not
2787/// supported.
2788///
2789/// The caller supplies an immutable `commit_hash` (an explicit SHA, or
2790/// the strict stable-tag fallback's resolved commit), so the download
2791/// fetches the exact commit the cache/source record names.
2792/// `commit_hash` is lowercased to align with `git_cache_key`.
2793///
2794/// Accepts the https/http/ssh/git and scp-style GitHub remotes, each
2795/// with an optional trailing `/` and `.git`; the host is matched
2796/// case-insensitively (DNS hostnames are case-insensitive).
2797pub(crate) fn github_archive_url(url: &str, commit_hash: &str) -> Option<String> {
2798    // Match the github.com scheme+host CASE-INSENSITIVELY (DNS
2799    // hostnames are case-insensitive, so `GitHub.com` is a GitHub URL),
2800    // keeping the OWNER/REPO path verbatim. Accept the https/http/ssh/git
2801    // schemes (with an optional `git@` userinfo) and the scp-style
2802    // git@github.com:OWNER/REPO, each with an optional trailing `.git`.
2803    let mut path = None;
2804    for prefix in [
2805        "https://github.com/",
2806        "http://github.com/",
2807        "ssh://git@github.com/",
2808        "ssh://github.com/",
2809        "git://github.com/",
2810        "git@github.com:",
2811    ] {
2812        if url
2813            .get(..prefix.len())
2814            .is_some_and(|head| head.eq_ignore_ascii_case(prefix))
2815        {
2816            path = Some(&url[prefix.len()..]);
2817            break;
2818        }
2819    }
2820    let path = path?;
2821    // Trim trailing slashes (a common copy-paste artifact) before the
2822    // `.git` strip so `OWNER/REPO/` and `OWNER/REPO.git/` still resolve
2823    // to codeload rather than misrouting to the clone path.
2824    let path = path.trim_end_matches('/');
2825    let path = path.strip_suffix(".git").unwrap_or(path);
2826    // Exactly OWNER/REPO — reject deeper paths (a stray extra segment
2827    // is not a repo root, so fall through to the clone path).
2828    let mut segs = path.split('/');
2829    let owner = segs.next().filter(|s| !s.is_empty())?;
2830    let repo = segs.next().filter(|s| !s.is_empty())?;
2831    if segs.next().is_some() {
2832        return None;
2833    }
2834    // Always the resolved COMMIT (lowercased) — never a ref-name
2835    // snapshot — so the extracted tree matches git_cache_key's commit
2836    // exactly regardless of a concurrent branch-tip move. codeload
2837    // serves any commit case-insensitively.
2838    Some(format!(
2839        "https://github.com/{owner}/{repo}/archive/{}.tar.gz",
2840        commit_hash.to_ascii_lowercase()
2841    ))
2842}
2843
2844/// The object id the advertised ref named exactly `target` points at,
2845/// or `None` if no ref matches. For an annotated tag (`Ref::Peeled`)
2846/// this is the PEELED commit (`object`), never the tag object;
2847/// `Ref::Unborn` carries no commit and never matches. Used by the
2848/// kind-directed [`resolve_ref_commit`] so tag-peeling and
2849/// unborn-skipping stay consistent.
2850fn pick_ref_object(
2851    refs: &[gix::protocol::handshake::Ref],
2852    target: &str,
2853) -> Option<gix::hash::ObjectId> {
2854    refs.iter().find_map(|r| {
2855        let (name, object) = match r {
2856            gix::protocol::handshake::Ref::Peeled {
2857                full_ref_name,
2858                object,
2859                ..
2860            }
2861            | gix::protocol::handshake::Ref::Direct {
2862                full_ref_name,
2863                object,
2864            }
2865            | gix::protocol::handshake::Ref::Symbolic {
2866                full_ref_name,
2867                object,
2868                ..
2869            } => (full_ref_name, object),
2870            gix::protocol::handshake::Ref::Unborn { .. } => return None,
2871        };
2872        (*name == target).then_some(*object)
2873    })
2874}
2875
2876/// Resolve `git_ref` to its full commit hash under `ref_kind`, via a
2877/// kind-directed exact-ref discovery. This standalone helper serves
2878/// stable-tag fallback and metadata callers that genuinely need
2879/// ls-remote semantics. Normal branch/tag acquisition instead reads the
2880/// peeled id from its single receive `Prepare`, avoiding a second
2881/// handshake. An explicit SHA resolves offline.
2882///
2883/// A `Sha` ref IS the commit (lowercased to match `git_clone`'s
2884/// rendering) and resolves offline — no handshake. `Tag`/`Branch`
2885/// match ONLY the fully-qualified `refs/tags/{ref}` / `refs/heads/{ref}`
2886/// so a tag never aliases a same-named branch (a bare-name DWIM lookup
2887/// would resolve either). `Ok(None)` means the exact ref was not
2888/// advertised or the kind is `Unknown`. Transport, authentication,
2889/// protocol, and cancellation failures propagate to the caller.
2890pub(crate) fn resolve_ref_commit(
2891    url: &str,
2892    git_ref: &str,
2893    ref_kind: crate::kernel_path::GitRefKind,
2894    cli_label: &str,
2895    mp: Option<&crate::cli::FetchProgress>,
2896) -> Result<Option<gix::ObjectId>> {
2897    use crate::kernel_path::GitRefKind;
2898    let target = match ref_kind {
2899        GitRefKind::Sha => {
2900            let canonical = git_ref.to_ascii_lowercase();
2901            return gix::ObjectId::from_hex(canonical.as_bytes())
2902                .with_context(|| format!("parse explicit git commit {git_ref}"))
2903                .map(Some);
2904        }
2905        GitRefKind::Tag => format!("refs/tags/{git_ref}"),
2906        GitRefKind::Branch => format!("refs/heads/{git_ref}"),
2907        GitRefKind::Unknown => return Ok(None),
2908    };
2909    match gix_policy::classify_source(url).map_err(anyhow::Error::msg)? {
2910        gix_policy::InProcessSource::Local(path) => {
2911            return resolve_local_ref_commit(&path, &target, &SOURCE_OPERATION_INTERRUPTED);
2912        }
2913        gix_policy::InProcessSource::Http | gix_policy::InProcessSource::Https => {}
2914    }
2915    let progress_label = format!("{cli_label}: {git_ref}");
2916    let ref_progress = match mp {
2917        Some(fp) => fp.operation_progress(&progress_label, "resolving git ref", "ref discovery"),
2918        None => crate::cli::progress::CloneProgress::standalone_operation(
2919            &progress_label,
2920            "resolving git ref",
2921            "ref discovery",
2922        ),
2923    };
2924    let refs = discover_remote_refs(
2925        url,
2926        Some(&target),
2927        &ref_progress,
2928        &SOURCE_OPERATION_INTERRUPTED,
2929    )?;
2930    let object = pick_ref_object(&refs, &target);
2931    ref_progress.finish();
2932    Ok(object)
2933}
2934
2935fn resolve_local_ref_commit(
2936    repository_path: &Path,
2937    target: &str,
2938    interrupt: &std::sync::atomic::AtomicBool,
2939) -> Result<Option<gix::ObjectId>> {
2940    ensure_git_operation_not_interrupted(interrupt, "before opening local repository")?;
2941    let repo = gix::open_opts(repository_path, anon_open_opts())
2942        .with_context(|| format!("open local source repository {}", repository_path.display()))?;
2943    ensure_git_operation_not_interrupted(interrupt, "after opening local repository")?;
2944    let Some(mut reference) = repo
2945        .try_find_reference(target)
2946        .with_context(|| format!("find local source ref {target}"))?
2947    else {
2948        return Ok(None);
2949    };
2950    let commit_id = reference
2951        .peel_to_commit()
2952        .with_context(|| format!("peel local source ref {target} to a commit"))?
2953        .id;
2954    #[cfg(test)]
2955    if TEST_INTERRUPT_AFTER_REF_DISCOVERY.swap(false, Ordering::AcqRel) {
2956        interrupt.store(true, Ordering::Release);
2957    }
2958    ensure_git_operation_not_interrupted(interrupt, "after reading local ref")?;
2959    Ok(Some(commit_id))
2960}
2961
2962/// ls-remote `url` and return EVERY advertised ref WITHOUT fetching a
2963/// pack. Callers decide explicitly whether failure is fatal:
2964/// correctness-sensitive resolution propagates it, while optional
2965/// historical-tag enrichment degrades with `.ok()`.
2966///
2967/// The ad-hoc repo (`init_opts` on a tempdir, with repo-local git config
2968/// only — see `anon_open_opts`) carries no working tree and fetches no
2969/// pack. Remote-side ref-prefix filtering is
2970/// DISABLED: gix's default (`prefix_from_spec_as_filter_on_remote =
2971/// true`) derives protocol-v2 `ls-refs` `ref-prefix` filters from the
2972/// remote's fetch refspecs; an anonymous `remote_at` has none, and
2973/// `fetch_tags = Included` injects only `refs/tags/*`, so the server
2974/// would return TAGS ONLY and `refs/heads/*` would never arrive.
2975/// Disabling the filter returns all refs, so a branch, tag, or HEAD
2976/// all resolve.
2977fn ls_remote_refs(url: &str, progress_label: &str) -> Result<Vec<gix::protocol::handshake::Ref>> {
2978    let ref_progress = crate::cli::progress::CloneProgress::standalone_operation(
2979        progress_label,
2980        "resolving git refs",
2981        "ref discovery",
2982    );
2983    let refs = discover_remote_refs(url, None, &ref_progress, &SOURCE_OPERATION_INTERRUPTED)?;
2984    ref_progress.finish();
2985    Ok(refs)
2986}
2987
2988/// Discover either one exact fully-qualified ref or the remote's full
2989/// advertisement without fetching a pack.
2990///
2991/// `exact_ref = Some` installs one exact fetch refspec and leaves
2992/// protocol-v2 prefix filtering enabled, so a normal branch/tag cache
2993/// probe asks the server only for that namespace-qualified ref. The
2994/// `None` form deliberately disables filtering for stable-tag
2995/// enumeration. Both forms create their progress phase before
2996/// `connect`: the first blocking network wait is therefore visible and
2997/// receives CloneProgress's ten-second heartbeat.
2998fn discover_remote_refs(
2999    url: &str,
3000    exact_ref: Option<&str>,
3001    progress: &crate::cli::progress::CloneProgress,
3002    interrupt: &std::sync::atomic::AtomicBool,
3003) -> Result<Vec<gix::protocol::handshake::Ref>> {
3004    ensure_git_operation_not_interrupted(interrupt, "before preparing ref discovery")?;
3005    match gix_policy::classify_source(url).map_err(anyhow::Error::msg)? {
3006        gix_policy::InProcessSource::Http | gix_policy::InProcessSource::Https => {}
3007        gix_policy::InProcessSource::Local(path) => anyhow::bail!(
3008            "local ref discovery must open {} directly instead of constructing a remote transport",
3009            path.display()
3010        ),
3011    }
3012    let tmp = tempfile::TempDir::new().with_context(|| "create ref discovery staging dir")?;
3013    ensure_git_operation_not_interrupted(interrupt, "after preparing ref discovery")?;
3014    let repo = gix::ThreadSafeRepository::init_opts(
3015        tmp.path(),
3016        gix::create::Kind::WithWorktree,
3017        gix::create::Options::default(),
3018        anon_open_opts(),
3019    )
3020    .with_context(|| "prepare ref discovery repository")?
3021    .to_thread_local();
3022    ensure_git_operation_not_interrupted(interrupt, "after initializing ref discovery")?;
3023    let mut remote = repo
3024        .remote_at_without_url_rewrite(url)
3025        .with_context(|| format!("prepare ref discovery remote {url}"))?;
3026    ensure_git_operation_not_interrupted(interrupt, "after preparing ref discovery remote")?;
3027
3028    if let Some(source_ref) = exact_ref {
3029        // A private throwaway destination is enough to derive the exact
3030        // protocol-v2 ref-prefix; no ref is written because this is only
3031        // a ref-map operation.
3032        const DISCOVERY_REF: &str = "refs/ktstr/discovery";
3033        let refspec = format!("+{source_ref}:{DISCOVERY_REF}");
3034        remote = remote.with_fetch_tags(gix::remote::fetch::Tags::None);
3035        remote
3036            .replace_refspecs([refspec.as_str()], gix::remote::Direction::Fetch)
3037            .with_context(|| format!("set exact discovery refspec for {source_ref}"))?;
3038        ensure_git_operation_not_interrupted(interrupt, "after setting discovery refspec")?;
3039    }
3040
3041    // This phase exists before connect(), whose handshake has no
3042    // progress or interrupt parameter. The shared process interrupt is
3043    // checked on both sides of that non-interruptible boundary.
3044    let discovery_progress = progress.item_named("discovering refs");
3045    ensure_git_operation_not_interrupted(interrupt, "before connecting for ref discovery")?;
3046    let mut connection = remote
3047        .connect(gix::remote::Direction::Fetch)
3048        .with_context(|| format!("connect to {url} for ref discovery"))?;
3049    connection.set_credentials(gix_policy::reject_credentials);
3050    ensure_git_operation_not_interrupted(interrupt, "after connecting for ref discovery")?;
3051    let options = gix::remote::ref_map::Options {
3052        prefix_from_spec_as_filter_on_remote: exact_ref.is_some(),
3053        ..Default::default()
3054    };
3055    ensure_git_operation_not_interrupted(interrupt, "before reading remote refs")?;
3056    let (refmap, _handshake) = connection
3057        .ref_map(discovery_progress, options)
3058        .with_context(|| format!("read remote refs from {url}"))?;
3059    #[cfg(test)]
3060    if TEST_INTERRUPT_AFTER_REF_DISCOVERY.swap(false, Ordering::AcqRel) {
3061        interrupt.store(true, Ordering::Release);
3062    }
3063    ensure_git_operation_not_interrupted(interrupt, "after reading remote refs")?;
3064    Ok(refmap.remote_refs)
3065}
3066
3067/// Shared hermetic options for runtime ref discovery, remote acquisition,
3068/// and directly opened local repositories.
3069fn anon_open_opts() -> gix::open::Options {
3070    gix_policy::open_options()
3071}
3072
3073/// Shallow-clone a git repository at a BRANCH ref.
3074///
3075/// `cli_label` prefixes diagnostic status output (e.g. `"ktstr"` or
3076/// `"cargo ktstr"`).
3077///
3078/// `mp` is the progress group a determinate clone bar is added to.
3079/// Without a group, the clone still drives a hidden CloneProgress tree
3080/// and emits escape-free ten-second heartbeats, so single-shot
3081/// `kernel build` cannot disappear during a slow handshake or pack
3082/// index. The bar shows real object/file counts + ETA during the
3083/// receiving / resolving / checkout phases that gix reports a bounded
3084/// total for; see the `crate::cli::progress` module.
3085///
3086/// For a TAG ref use the test-visible `git_clone_tag` wrapper. Both entry points share the
3087/// same lower-level exact-ref fetch; the wrapper selects the native
3088/// `refs/heads/*` or `refs/tags/*` source namespace.
3089pub fn git_clone(
3090    url: &str,
3091    git_ref: &str,
3092    dest_dir: &Path,
3093    cli_label: &str,
3094    mp: Option<&crate::cli::FetchProgress>,
3095) -> Result<AcquiredSource> {
3096    git_clone_inner(
3097        url,
3098        git_ref,
3099        crate::kernel_path::GitRefKind::Branch,
3100        dest_dir,
3101        cli_label,
3102        mp,
3103    )
3104}
3105
3106/// Shallow-clone a git repository at a TAG ref (e.g. `v6.14.11`).
3107///
3108/// The exact fetch maps `refs/tags/{tag}` to ktstr's private fetched
3109/// ref, peels annotated tags to their commit for checkout, and retains
3110/// the original tag object under its native name for `git describe` /
3111/// `setlocalversion`. The `#tag=` source (via
3112/// [`git_clone_kinded`]) uses this path for every remote, including
3113/// GitHub. The pruned/EOL tarball recovery is a separately reported
3114/// archive fallback in [`download_tarball`].
3115#[cfg(test)]
3116pub(crate) fn git_clone_tag(
3117    url: &str,
3118    tag: &str,
3119    dest_dir: &Path,
3120    cli_label: &str,
3121    mp: Option<&crate::cli::FetchProgress>,
3122) -> Result<AcquiredSource> {
3123    git_clone_inner(
3124        url,
3125        tag,
3126        crate::kernel_path::GitRefKind::Tag,
3127        dest_dir,
3128        cli_label,
3129        mp,
3130    )
3131}
3132
3133/// Clone a git source at `git_ref`, dispatching on `ref_kind` to the
3134/// correct clone path. Branches and tags use this exact shallow gix
3135/// path uniformly, including well-formed GitHub remotes. Only an
3136/// explicit immutable GitHub `Sha` is routed to
3137/// [`download_github_archive`] by [`crate::cli::resolve_git_kernel`].
3138///
3139/// - `Tag` → [`git_clone_tag`] (exact `refs/tags/*` source).
3140/// - `Branch` → [`git_clone`] (exact `refs/heads/*` source).
3141/// - `Sha` → a hard error: gix cannot fetch a bare commit, and a
3142///   self-hosted server generally lacks allow-sha-in-want. The
3143///   actionable message points at GitHub (codeload serves any sha) or a
3144///   tag/branch.
3145/// - `Unknown` → a hard error; [`crate::kernel_path::KernelId::validate`]
3146///   rejects it upstream, so this is a defensive backstop.
3147#[cfg(test)]
3148pub(crate) fn git_clone_kinded(
3149    url: &str,
3150    git_ref: &str,
3151    ref_kind: crate::kernel_path::GitRefKind,
3152    dest_dir: &Path,
3153    cli_label: &str,
3154    mp: Option<&crate::cli::FetchProgress>,
3155) -> Result<AcquiredSource> {
3156    use crate::kernel_path::GitRefKind;
3157    match ref_kind {
3158        GitRefKind::Tag => git_clone_tag(url, git_ref, dest_dir, cli_label, mp),
3159        GitRefKind::Branch => git_clone(url, git_ref, dest_dir, cli_label, mp),
3160        GitRefKind::Sha => anyhow::bail!(
3161            "git+{url}#sha={git_ref}: fetching this source by commit sha is \
3162             not supported — gix cannot fetch a bare commit and the remote \
3163             lacks allow-sha-in-want. Use a github.com/OWNER/REPO URL \
3164             (codeload serves any commit) or pin a #tag= / #branch= instead."
3165        ),
3166        GitRefKind::Unknown => anyhow::bail!(
3167            "git+{url}: ref kind could not be determined; use #tag=NAME, \
3168             #branch=NAME, or #sha=<40-hex>"
3169        ),
3170    }
3171}
3172
3173/// Shared shallow-clone implementation for [`git_clone`] (branch) and
3174/// `git_clone_tag` (tag).
3175///
3176/// This deliberately uses gix's lower-level fetch API rather than
3177/// `clone::PrepareFetch`. The clone helper adds the remote's `HEAD` to
3178/// every fetch and, for its ordinary shallow branch path, changes tag
3179/// handling to `Tags::All`. On a kernel repository that turns a
3180/// depth-one branch checkout into a fetch and pack-index of every
3181/// release tag. An exact fully-qualified refspec keeps negotiation to
3182/// the one branch or tag the caller requested. Branches and tags use
3183/// the same fetch and checkout path; only their source namespace
3184/// differs.
3185fn git_clone_inner(
3186    url: &str,
3187    git_ref: &str,
3188    ref_kind: crate::kernel_path::GitRefKind,
3189    dest_dir: &Path,
3190    cli_label: &str,
3191    mp: Option<&crate::cli::FetchProgress>,
3192) -> Result<AcquiredSource> {
3193    match git_clone_inner_gated(url, git_ref, ref_kind, dest_dir, cli_label, mp, |_| {
3194        Ok(ExactRefGate::Fetch(()))
3195    })? {
3196        GitCloneOutcome::Fetched { acquired, .. } => Ok(acquired),
3197        GitCloneOutcome::Skipped { .. } => unreachable!("unconditional clone cannot skip"),
3198    }
3199}
3200
3201pub(crate) enum GitCloneOutcome<T> {
3202    Fetched { acquired: AcquiredSource, token: T },
3203    Skipped { commit_hash: String, token: T },
3204}
3205
3206/// Exact branch/tag acquisition with a cache/election callback executed
3207/// after the single gix ref-map and before receive. Used by
3208/// `resolve_git_kernel` to avoid a separate discovery handshake.
3209pub(crate) fn git_clone_kinded_gated<T>(
3210    url: &str,
3211    git_ref: &str,
3212    ref_kind: crate::kernel_path::GitRefKind,
3213    dest_dir: &Path,
3214    cli_label: &str,
3215    mp: Option<&crate::cli::FetchProgress>,
3216    gate: impl FnOnce(gix::ObjectId) -> Result<ExactRefGate<T>>,
3217) -> Result<GitCloneOutcome<T>> {
3218    use crate::kernel_path::GitRefKind;
3219    match ref_kind {
3220        GitRefKind::Branch | GitRefKind::Tag => {
3221            git_clone_inner_gated(url, git_ref, ref_kind, dest_dir, cli_label, mp, gate)
3222        }
3223        GitRefKind::Sha => anyhow::bail!(
3224            "git+{url}#sha={git_ref}: fetching this source by commit sha is \
3225             not supported — use a github.com/OWNER/REPO URL or pin a \
3226             #tag= / #branch= instead"
3227        ),
3228        GitRefKind::Unknown => anyhow::bail!(
3229            "git+{url}: ref kind could not be determined; use #tag=NAME, \
3230             #branch=NAME, or #sha=<40-hex>"
3231        ),
3232    }
3233}
3234
3235fn close_private_staging(staging: tempfile::TempDir, primary: anyhow::Error) -> anyhow::Error {
3236    match staging.close() {
3237        Ok(()) => primary,
3238        Err(cleanup_err) => primary.context(format!(
3239            "also failed to remove private source-acquisition staging directory: {cleanup_err}"
3240        )),
3241    }
3242}
3243
3244fn git_clone_inner_gated<T>(
3245    url: &str,
3246    git_ref: &str,
3247    ref_kind: crate::kernel_path::GitRefKind,
3248    dest_dir: &Path,
3249    cli_label: &str,
3250    mp: Option<&crate::cli::FetchProgress>,
3251    gate: impl FnOnce(gix::ObjectId) -> Result<ExactRefGate<T>>,
3252) -> Result<GitCloneOutcome<T>> {
3253    let cloning = format!("{cli_label}: cloning {url} (ref: {git_ref}, depth: 1)");
3254    match mp {
3255        Some(fp) => fp.println(&cloning),
3256        None => crate::cli::print_status_line(&cloning),
3257    }
3258
3259    let clone_dir = dest_dir.join("linux");
3260    let staging = tempfile::TempDir::new_in(dest_dir)
3261        .with_context(|| format!("create private clone staging dir in {}", dest_dir.display()))?;
3262
3263    // Drive a determinate clone bar from gix's progress tree (see
3264    // [`crate::cli::progress::CloneProgress`]). A shared group renders
3265    // coordinated bars; a command without one still gets the same
3266    // plain-text ten-second heartbeat through a standalone reporter.
3267    // Every phase polls the process-level signal bridge set by
3268    // cargo-ktstr's interrupt handler.
3269    let progress_label = format!("{cli_label}: {git_ref}");
3270    let clone_progress = match mp {
3271        Some(fp) => fp.clone_progress(&progress_label),
3272        None => crate::cli::progress::CloneProgress::standalone_operation(
3273            &progress_label,
3274            "cloning",
3275            "clone",
3276        ),
3277    };
3278    let fetched = match fetch_exact_ref_and_checkout_gated(
3279        url,
3280        git_ref,
3281        ref_kind,
3282        staging.path(),
3283        Some(&clone_progress),
3284        &SOURCE_OPERATION_INTERRUPTED,
3285        gate,
3286    ) {
3287        Ok(outcome) => outcome,
3288        Err(err) => {
3289            return Err(close_private_staging(staging, err));
3290        }
3291    };
3292    let (commit_id, token) = match fetched {
3293        ExactRefFetch::Skipped { commit_id, token } => {
3294            let commit_hash = format!("{commit_id}");
3295            staging.close().with_context(
3296                || "remove private clone staging directory after cache/election skip",
3297            )?;
3298            return Ok(GitCloneOutcome::Skipped { commit_hash, token });
3299        }
3300        ExactRefFetch::Fetched { commit_id, token } => (commit_id, token),
3301    };
3302
3303    let acquired = (|| -> Result<AcquiredSource> {
3304        // FULL commit hash keys the cache (see `git_cache_key` — a 7-hex
3305        // prefix risks a moved-tip collision serving a stale build); the
3306        // 7-hex `short_hash` is kept only for the human-facing source
3307        // record.
3308        let commit_hash = format!("{commit_id}");
3309        let short_hash = commit_hash.chars().take(7).collect::<String>();
3310        let cache_key = git_cache_key(ref_kind, git_ref, &commit_hash);
3311
3312        // Record the kernel version from the checked-out source
3313        // Makefile, as local_source does. The filesystem read is the
3314        // final non-interruptible boundary after index publication, so
3315        // observe the shared epoch on both sides before making this
3316        // source visible to the cache/build pipeline.
3317        ensure_source_operation_not_interrupted("before reading cloned source metadata")?;
3318        let version = read_makefile_version(staging.path());
3319        ensure_source_operation_not_interrupted("after reading cloned source metadata")?;
3320
3321        Ok(AcquiredSource {
3322            source_dir: clone_dir.clone(),
3323            cache_key,
3324            version,
3325            kernel_source: crate::cache::KernelSource::git(short_hash, git_ref),
3326            is_temp: true,
3327            is_dirty: false,
3328            is_git: true,
3329        })
3330    })();
3331    match acquired {
3332        Ok(acquired) => {
3333            let promoted = match promote_path_noreplace(
3334                staging.path(),
3335                &clone_dir,
3336                PublishedPathKind::Directory,
3337            ) {
3338                Ok(promoted) => promoted,
3339                Err(err) => return Err(close_private_staging(staging, err)),
3340            };
3341            #[cfg(test)]
3342            if TEST_INTERRUPT_AFTER_CLONE_PROMOTE.swap(false, Ordering::AcqRel) {
3343                SOURCE_OPERATION_INTERRUPTED.store(true, Ordering::Release);
3344            }
3345            if let Err(err) = ensure_source_operation_not_interrupted("after promoting exact clone")
3346            {
3347                drop(staging);
3348                return Err(promoted.rollback(err));
3349            }
3350            drop(staging);
3351            let source_dir = promoted.commit();
3352            debug_assert_eq!(source_dir, acquired.source_dir);
3353            clone_progress.finish();
3354            Ok(GitCloneOutcome::Fetched { acquired, token })
3355        }
3356        Err(err) => Err(close_private_staging(staging, err)),
3357    }
3358}
3359
3360/// Measured ceiling for one gix pack-index or checkout operation.
3361///
3362/// A live sched_ext `for-next` depth-one fetch on the 64-CPU
3363/// development host took 55.0s / 705MB with all CPUs, 57.6s / 454MB
3364/// with eight, and 58.0s / 591MB with sixteen. Eight therefore keeps
3365/// essentially all transfer throughput while avoiding hundreds of MB
3366/// of per-clone worker state. Smaller hosts use their available CPU
3367/// count instead.
3368const GIX_WORKERS_PER_OPERATION: usize = 8;
3369
3370/// Machine-wide, non-blocking lease for checkout's optional extra
3371/// workers. Every checkout always has its calling thread. It then tries
3372/// to flock up to seven numbered tokens in `KTSTR_LOCK_DIR` (or `/tmp`)
3373/// and uses one worker per acquired token. Contending processes simply
3374/// get fewer extras; no source acquisition is ever queued behind this
3375/// performance optimization.
3376struct GixCheckoutWorkerLease {
3377    extra_workers: Vec<std::os::fd::OwnedFd>,
3378}
3379
3380impl GixCheckoutWorkerLease {
3381    fn acquire() -> Self {
3382        let wanted = std::thread::available_parallelism()
3383            .map(usize::from)
3384            .unwrap_or(1)
3385            .min(GIX_WORKERS_PER_OPERATION)
3386            .saturating_sub(1);
3387        let lock_dir = crate::cache::resolve_lock_dir();
3388        Self::acquire_in(&lock_dir, wanted)
3389    }
3390
3391    fn acquire_in(lock_dir: &Path, wanted: usize) -> Self {
3392        let wanted = wanted.min(GIX_WORKERS_PER_OPERATION.saturating_sub(1));
3393        if let Err(err) = std::fs::create_dir_all(lock_dir) {
3394            tracing::warn!(
3395                %err,
3396                path = %lock_dir.display(),
3397                "could not create gix worker-token directory; checkout will use its caller thread",
3398            );
3399            return Self {
3400                extra_workers: Vec::new(),
3401            };
3402        }
3403        let mut extra_workers = Vec::with_capacity(wanted);
3404        // `wanted` is also the size of the machine-wide namespace.
3405        // Scanning all seven ceiling slots on a two-CPU host would let
3406        // seven processes each lease a different extra despite the
3407        // host-wide budget being one.
3408        for slot in 0..wanted {
3409            if extra_workers.len() == wanted {
3410                break;
3411            }
3412            let path = lock_dir.join(format!("ktstr-gix-checkout-worker-{slot}.lock"));
3413            match crate::flock::try_flock(&path, crate::flock::FlockMode::Exclusive) {
3414                Ok(Some(fd)) => extra_workers.push(fd),
3415                Ok(None) => {}
3416                Err(err) => {
3417                    tracing::warn!(
3418                        %err,
3419                        path = %path.display(),
3420                        "could not lease gix checkout worker token; continuing with fewer workers",
3421                    );
3422                }
3423            }
3424        }
3425        Self { extra_workers }
3426    }
3427
3428    fn workers(&self) -> usize {
3429        1 + self.extra_workers.len()
3430    }
3431}
3432
3433/// Fetch one fully-qualified branch or tag at depth one and materialize
3434/// its peeled commit into `clone_dir`.
3435pub(crate) enum ExactRefGate<T> {
3436    Fetch(T),
3437    Skip(T),
3438}
3439
3440enum ExactRefFetch<T> {
3441    Fetched { commit_id: gix::ObjectId, token: T },
3442    Skipped { commit_id: gix::ObjectId, token: T },
3443}
3444
3445#[cfg(test)]
3446fn fetch_exact_ref_and_checkout(
3447    url: &str,
3448    git_ref: &str,
3449    ref_kind: crate::kernel_path::GitRefKind,
3450    clone_dir: &Path,
3451    clone_progress: Option<&crate::cli::progress::CloneProgress>,
3452    interrupt: &std::sync::atomic::AtomicBool,
3453) -> Result<gix::ObjectId> {
3454    match fetch_exact_ref_and_checkout_gated(
3455        url,
3456        git_ref,
3457        ref_kind,
3458        clone_dir,
3459        clone_progress,
3460        interrupt,
3461        |_| Ok(ExactRefGate::Fetch(())),
3462    )? {
3463        ExactRefFetch::Fetched { commit_id, .. } => Ok(commit_id),
3464        ExactRefFetch::Skipped { .. } => unreachable!("unconditional exact fetch cannot skip"),
3465    }
3466}
3467
3468/// Prepare one exact gix fetch, expose its advertised peeled commit to
3469/// the caller before receiving a pack, and reuse that same `Prepare`
3470/// when the caller wins the content-key builder election.
3471fn fetch_exact_ref_and_checkout_gated<T>(
3472    url: &str,
3473    git_ref: &str,
3474    ref_kind: crate::kernel_path::GitRefKind,
3475    clone_dir: &Path,
3476    clone_progress: Option<&crate::cli::progress::CloneProgress>,
3477    interrupt: &std::sync::atomic::AtomicBool,
3478    gate: impl FnOnce(gix::ObjectId) -> Result<ExactRefGate<T>>,
3479) -> Result<ExactRefFetch<T>> {
3480    use crate::kernel_path::GitRefKind;
3481
3482    let source_ref = match ref_kind {
3483        GitRefKind::Branch => format!("refs/heads/{git_ref}"),
3484        GitRefKind::Tag => format!("refs/tags/{git_ref}"),
3485        GitRefKind::Sha | GitRefKind::Unknown => {
3486            anyhow::bail!("exact shallow fetch requires a branch or tag")
3487        }
3488    };
3489    if let gix_policy::InProcessSource::Local(repository_path) =
3490        gix_policy::classify_source(url).map_err(anyhow::Error::msg)?
3491    {
3492        return materialize_local_exact_ref_gated(
3493            &repository_path,
3494            &source_ref,
3495            clone_dir,
3496            clone_progress,
3497            interrupt,
3498            gate,
3499        );
3500    }
3501    // A fixed private destination means arbitrary valid branch/tag names
3502    // never have to be re-encoded as a local branch name.
3503    const FETCHED_REF: &str = "refs/ktstr/source";
3504    let refspec = format!("+{source_ref}:{FETCHED_REF}");
3505
3506    ensure_git_operation_not_interrupted(interrupt, "before preparing clone")?;
3507    let mut repo = gix::ThreadSafeRepository::init_opts(
3508        clone_dir,
3509        gix::create::Kind::WithWorktree,
3510        gix::create::Options::default(),
3511        anon_open_opts(),
3512    )
3513    .with_context(|| "prepare clone")?
3514    .to_thread_local();
3515    ensure_git_operation_not_interrupted(interrupt, "after preparing clone")?;
3516
3517    // Pack indexing runs on the caller thread. Checkout separately
3518    // leases machine-wide optional workers immediately before it starts;
3519    // no worker token is held across connect or ref mapping.
3520    ensure_git_operation_not_interrupted(interrupt, "before configuring pack workers")?;
3521    repo.config_snapshot_mut()
3522        .set_value(&gix::config::tree::Pack::THREADS, "1")
3523        .with_context(|| "configure single-threaded gix pack indexing")?;
3524    ensure_git_operation_not_interrupted(interrupt, "after configuring pack workers")?;
3525
3526    ensure_git_operation_not_interrupted(interrupt, "before preparing clone remote")?;
3527    let mut remote = repo
3528        .remote_at_without_url_rewrite(url)
3529        .with_context(|| "prepare clone remote")?
3530        .with_fetch_tags(gix::remote::fetch::Tags::None);
3531    ensure_git_operation_not_interrupted(interrupt, "after preparing clone remote")?;
3532    ensure_git_operation_not_interrupted(interrupt, "before setting exact fetch refspec")?;
3533    remote
3534        .replace_refspecs([refspec.as_str()], gix::remote::Direction::Fetch)
3535        .with_context(|| format!("set exact fetch refspec for {source_ref}"))?;
3536    ensure_git_operation_not_interrupted(interrupt, "after setting exact fetch refspec")?;
3537
3538    // prepare_fetch owns this item later, but create it before connect:
3539    // connect performs the first network handshake and accepts no
3540    // progress/interrupt handle of its own.
3541    let discovery_progress =
3542        clone_progress.map(|progress| progress.item_named("discovering exact ref"));
3543    ensure_git_operation_not_interrupted(interrupt, "before connecting")?;
3544    let connection = remote
3545        .connect(gix::remote::Direction::Fetch)
3546        .with_context(|| format!("connect to {url}"))?;
3547    let mut connection = connection;
3548    connection.set_credentials(gix_policy::reject_credentials);
3549    ensure_git_operation_not_interrupted(interrupt, "after connecting")?;
3550    ensure_git_operation_not_interrupted(interrupt, "before mapping exact remote ref")?;
3551    let prepare = match discovery_progress {
3552        Some(progress) => connection
3553            .prepare_fetch(progress, gix::remote::ref_map::Options::default())
3554            .with_context(|| format!("map exact remote ref {source_ref}"))?,
3555        None => connection
3556            .prepare_fetch(
3557                gix::progress::Discard,
3558                gix::remote::ref_map::Options::default(),
3559            )
3560            .with_context(|| format!("map exact remote ref {source_ref}"))?,
3561    };
3562    #[cfg(test)]
3563    TEST_EXACT_PREPARE_COUNT.fetch_add(1, Ordering::AcqRel);
3564    ensure_git_operation_not_interrupted(interrupt, "after mapping exact remote ref")?;
3565    let prepare = prepare.with_shallow(gix::remote::fetch::Shallow::DepthAtRemote(
3566        NonZeroU32::new(1).expect("1 is nonzero"),
3567    ));
3568    ensure_git_operation_not_interrupted(interrupt, "after configuring shallow fetch")?;
3569
3570    // `Prepare` already owns the one connect + ref-map handshake. Read
3571    // the exact mapping it will receive and elect/cache-probe before any
3572    // pack bytes move; the winner continues with this same value.
3573    let advertised_commit = prepare
3574        .ref_map()
3575        .mappings
3576        .iter()
3577        .find_map(|mapping| mapping.remote.peeled_id().map(ToOwned::to_owned))
3578        .with_context(|| format!("remote did not advertise exact ref {source_ref}"))?;
3579    ensure_git_operation_not_interrupted(interrupt, "before exact-ref cache election")?;
3580    let token = match gate(advertised_commit)? {
3581        ExactRefGate::Fetch(token) => token,
3582        ExactRefGate::Skip(token) => {
3583            return Ok(ExactRefFetch::Skipped {
3584                commit_id: advertised_commit,
3585                token,
3586            });
3587        }
3588    };
3589    ensure_git_operation_not_interrupted(interrupt, "after exact-ref cache election")?;
3590
3591    ensure_git_operation_not_interrupted(interrupt, "before receiving the pack")?;
3592    match clone_progress {
3593        Some(progress) => prepare
3594            .receive(
3595                progress.item_named("receiving and indexing exact ref"),
3596                interrupt,
3597            )
3598            .with_context(|| format!("fetch exact remote ref {source_ref}"))?,
3599        None => prepare
3600            .receive(gix::progress::Discard, interrupt)
3601            .with_context(|| format!("fetch exact remote ref {source_ref}"))?,
3602    };
3603    ensure_git_operation_not_interrupted(interrupt, "after receiving the pack")?;
3604    ensure_git_operation_not_interrupted(interrupt, "before reading fetched ref")?;
3605    let mut fetched_ref = repo
3606        .find_reference(FETCHED_REF)
3607        .with_context(|| format!("find fetched ref {source_ref}"))?;
3608    ensure_git_operation_not_interrupted(interrupt, "after reading fetched ref")?;
3609    let unpeeled_id = fetched_ref.id().detach();
3610    // Preserve exactly the requested ref in its native namespace. In
3611    // particular, retaining the selected tag keeps setlocalversion and
3612    // git-describe semantics without importing any unrelated tag.
3613    ensure_git_operation_not_interrupted(interrupt, "before preserving exact source ref")?;
3614    repo.reference(
3615        source_ref.as_str(),
3616        unpeeled_id,
3617        gix::refs::transaction::PreviousValue::Any,
3618        "clone: preserve exact source ref",
3619    )
3620    .with_context(|| format!("preserve fetched ref {source_ref}"))?;
3621    ensure_git_operation_not_interrupted(interrupt, "after preserving exact source ref")?;
3622    ensure_git_operation_not_interrupted(interrupt, "before peeling exact source ref")?;
3623    let commit_id = fetched_ref
3624        .peel_to_commit()
3625        .with_context(|| format!("peel fetched ref {source_ref} to a commit"))?
3626        .id;
3627    ensure_git_operation_not_interrupted(interrupt, "after peeling exact source ref")?;
3628    if commit_id != advertised_commit {
3629        anyhow::bail!(
3630            "exact ref {source_ref} changed during its single fetch handshake \
3631             (advertised {advertised_commit}, received {commit_id})"
3632        );
3633    }
3634
3635    // Both branch and tag clones are snapshots for ktstr, so a detached
3636    // HEAD is the uniform representation of the exact peeled commit.
3637    ensure_git_operation_not_interrupted(interrupt, "before publishing exact HEAD")?;
3638    repo.reference(
3639        "HEAD",
3640        commit_id,
3641        gix::refs::transaction::PreviousValue::Any,
3642        "clone: checkout exact ktstr source",
3643    )
3644    .with_context(|| "set cloned repository HEAD")?;
3645    ensure_git_operation_not_interrupted(interrupt, "after publishing exact HEAD")?;
3646
3647    match clone_progress {
3648        Some(progress) => {
3649            checkout_exact_commit(
3650                &repo,
3651                commit_id,
3652                progress.item_named("checking out exact ref"),
3653                interrupt,
3654            )?;
3655        }
3656        None => {
3657            checkout_exact_commit(&repo, commit_id, gix::progress::Discard, interrupt)?;
3658        }
3659    }
3660    Ok(ExactRefFetch::Fetched { commit_id, token })
3661}
3662
3663fn materialize_local_exact_ref_gated<T>(
3664    repository_path: &Path,
3665    source_ref: &str,
3666    clone_dir: &Path,
3667    clone_progress: Option<&crate::cli::progress::CloneProgress>,
3668    interrupt: &std::sync::atomic::AtomicBool,
3669    gate: impl FnOnce(gix::ObjectId) -> Result<ExactRefGate<T>>,
3670) -> Result<ExactRefFetch<T>> {
3671    const FETCHED_REF: &str = "refs/ktstr/source";
3672
3673    ensure_git_operation_not_interrupted(interrupt, "before opening local exact source")?;
3674    let source = gix::open_opts(repository_path, anon_open_opts())
3675        .with_context(|| format!("open local source repository {}", repository_path.display()))?;
3676    ensure_git_operation_not_interrupted(interrupt, "after opening local exact source")?;
3677    let mut reference = source
3678        .find_reference(source_ref)
3679        .with_context(|| format!("find local source ref {source_ref}"))?;
3680    let unpeeled_id = reference.id().detach();
3681    let commit_id = reference
3682        .peel_to_commit()
3683        .with_context(|| format!("peel local source ref {source_ref} to a commit"))?
3684        .id;
3685    ensure_git_operation_not_interrupted(interrupt, "before exact-ref cache election")?;
3686    let token = match gate(commit_id)? {
3687        ExactRefGate::Fetch(token) => token,
3688        ExactRefGate::Skip(token) => {
3689            return Ok(ExactRefFetch::Skipped { commit_id, token });
3690        }
3691    };
3692    ensure_git_operation_not_interrupted(interrupt, "after exact-ref cache election")?;
3693
3694    let _copy_progress =
3695        clone_progress.map(|progress| progress.item_named("copying local exact ref"));
3696    ensure_git_operation_not_interrupted(interrupt, "before preparing local clone")?;
3697    let repo = gix::ThreadSafeRepository::init_opts(
3698        clone_dir,
3699        gix::create::Kind::WithWorktree,
3700        gix::create::Options::default(),
3701        anon_open_opts(),
3702    )
3703    .with_context(|| "prepare local clone")?
3704    .to_thread_local();
3705    ensure_git_operation_not_interrupted(interrupt, "before copying local exact objects")?;
3706    copy_local_exact_ref_objects(&source, &repo, unpeeled_id, commit_id, interrupt)?;
3707    ensure_git_operation_not_interrupted(interrupt, "after copying local exact objects")?;
3708    drop(_copy_progress);
3709
3710    repo.reference(
3711        FETCHED_REF,
3712        unpeeled_id,
3713        gix::refs::transaction::PreviousValue::Any,
3714        "clone: stage exact local source ref",
3715    )
3716    .with_context(|| "stage exact local source ref")?;
3717    repo.reference(
3718        source_ref,
3719        unpeeled_id,
3720        gix::refs::transaction::PreviousValue::Any,
3721        "clone: preserve exact local source ref",
3722    )
3723    .with_context(|| format!("preserve local source ref {source_ref}"))?;
3724    repo.reference(
3725        "HEAD",
3726        commit_id,
3727        gix::refs::transaction::PreviousValue::Any,
3728        "clone: checkout exact local source",
3729    )
3730    .with_context(|| "set local clone HEAD")?;
3731    std::fs::write(repo.git_dir().join("shallow"), format!("{commit_id}\n"))
3732        .with_context(|| "write local clone shallow boundary")?;
3733    ensure_git_operation_not_interrupted(interrupt, "after publishing local exact refs")?;
3734
3735    match clone_progress {
3736        Some(progress) => checkout_exact_commit(
3737            &repo,
3738            commit_id,
3739            progress.item_named("checking out local exact ref"),
3740            interrupt,
3741        )?,
3742        None => checkout_exact_commit(&repo, commit_id, gix::progress::Discard, interrupt)?,
3743    }
3744    Ok(ExactRefFetch::Fetched { commit_id, token })
3745}
3746
3747fn copy_local_exact_ref_objects(
3748    source: &gix::Repository,
3749    destination: &gix::Repository,
3750    unpeeled_id: gix::ObjectId,
3751    commit_id: gix::ObjectId,
3752    interrupt: &std::sync::atomic::AtomicBool,
3753) -> Result<()> {
3754    let mut copied = std::collections::HashSet::new();
3755    let mut current = unpeeled_id;
3756    loop {
3757        ensure_git_operation_not_interrupted(interrupt, "while copying local exact ref")?;
3758        if !copied.insert(current) {
3759            anyhow::bail!("local exact ref contains a tag cycle at {current}");
3760        }
3761        let object = source
3762            .find_object(current)
3763            .with_context(|| format!("read local exact-ref object {current}"))?;
3764        write_local_object(destination, &object)?;
3765        match object.kind {
3766            gix::objs::Kind::Tag => {
3767                current = object
3768                    .try_into_tag()
3769                    .with_context(|| format!("decode local tag object {current}"))?
3770                    .target_id()
3771                    .with_context(|| format!("read target of local tag object {current}"))?
3772                    .detach();
3773            }
3774            gix::objs::Kind::Commit => {
3775                if current != commit_id {
3776                    anyhow::bail!(
3777                        "local exact ref peeled inconsistently: expected {commit_id}, found {current}"
3778                    );
3779                }
3780                let tree_id = object
3781                    .try_into_commit()
3782                    .with_context(|| format!("decode local commit {commit_id}"))?
3783                    .tree_id()
3784                    .with_context(|| format!("read tree of local commit {commit_id}"))?
3785                    .detach();
3786                copy_local_tree_objects(source, destination, tree_id, &mut copied, interrupt)?;
3787                return Ok(());
3788            }
3789            kind => anyhow::bail!("local exact ref resolves to {kind}, expected a tag or commit"),
3790        }
3791    }
3792}
3793
3794fn copy_local_tree_objects(
3795    source: &gix::Repository,
3796    destination: &gix::Repository,
3797    root_tree: gix::ObjectId,
3798    copied: &mut std::collections::HashSet<gix::ObjectId>,
3799    interrupt: &std::sync::atomic::AtomicBool,
3800) -> Result<()> {
3801    let mut pending = vec![root_tree];
3802    while let Some(object_id) = pending.pop() {
3803        #[cfg(test)]
3804        if TEST_INTERRUPT_DURING_LOCAL_COPY.swap(false, Ordering::AcqRel) {
3805            interrupt.store(true, Ordering::Release);
3806        }
3807        ensure_git_operation_not_interrupted(interrupt, "while copying local checkout tree")?;
3808        if !copied.insert(object_id) {
3809            continue;
3810        }
3811        let object = source
3812            .find_object(object_id)
3813            .with_context(|| format!("read local checkout object {object_id}"))?;
3814        write_local_object(destination, &object)?;
3815        match object.kind {
3816            gix::objs::Kind::Tree => {
3817                let tree = object
3818                    .try_into_tree()
3819                    .with_context(|| format!("decode local tree {object_id}"))?;
3820                for entry in tree.iter() {
3821                    let entry =
3822                        entry.with_context(|| format!("decode entry in local tree {object_id}"))?;
3823                    if entry.kind() != gix::objs::tree::EntryKind::Commit {
3824                        pending.push(entry.object_id());
3825                    }
3826                }
3827            }
3828            gix::objs::Kind::Blob => {}
3829            kind => anyhow::bail!(
3830                "local tree {root_tree} references unexpected {kind} object {object_id}"
3831            ),
3832        }
3833    }
3834    Ok(())
3835}
3836
3837fn write_local_object(destination: &gix::Repository, object: &gix::Object<'_>) -> Result<()> {
3838    let written = gix::objs::Write::write_buf(destination, object.kind, &object.data)
3839        .map_err(anyhow::Error::from_boxed)
3840        .with_context(|| format!("write local source object {}", object.id))?;
3841    if written != object.id {
3842        anyhow::bail!(
3843            "local source object {} changed identity while materializing (wrote {written})",
3844            object.id
3845        );
3846    }
3847    Ok(())
3848}
3849
3850fn ensure_git_operation_not_interrupted(
3851    interrupt: &std::sync::atomic::AtomicBool,
3852    phase: &str,
3853) -> Result<()> {
3854    if interrupt.load(Ordering::Acquire) {
3855        anyhow::bail!("git operation interrupted {phase}");
3856    }
3857    Ok(())
3858}
3859
3860#[cfg(test)]
3861static TEST_INTERRUPT_AFTER_CHECKOUT: std::sync::atomic::AtomicBool =
3862    std::sync::atomic::AtomicBool::new(false);
3863
3864#[cfg(test)]
3865static TEST_INTERRUPT_AFTER_INDEX_WRITE: std::sync::atomic::AtomicBool =
3866    std::sync::atomic::AtomicBool::new(false);
3867
3868#[cfg(test)]
3869static TEST_INTERRUPT_AFTER_REF_DISCOVERY: std::sync::atomic::AtomicBool =
3870    std::sync::atomic::AtomicBool::new(false);
3871
3872#[cfg(test)]
3873static TEST_INTERRUPT_DURING_LOCAL_COPY: std::sync::atomic::AtomicBool =
3874    std::sync::atomic::AtomicBool::new(false);
3875
3876#[cfg(test)]
3877static TEST_EXACT_PREPARE_COUNT: std::sync::atomic::AtomicUsize =
3878    std::sync::atomic::AtomicUsize::new(0);
3879
3880#[cfg(test)]
3881static TEST_INTERRUPT_AFTER_CLONE_PROMOTE: std::sync::atomic::AtomicBool =
3882    std::sync::atomic::AtomicBool::new(false);
3883
3884/// Materialize `commit_id` into the main worktree and persist its index.
3885fn checkout_exact_commit(
3886    repo: &gix::Repository,
3887    commit_id: gix::ObjectId,
3888    mut progress: impl gix::Progress,
3889    interrupt: &std::sync::atomic::AtomicBool,
3890) -> Result<()> {
3891    ensure_git_operation_not_interrupted(interrupt, "before locating exact clone worktree")?;
3892    let workdir = repo
3893        .workdir()
3894        .with_context(|| "exact clone repository has no worktree")?;
3895    ensure_git_operation_not_interrupted(interrupt, "after locating exact clone worktree")?;
3896    ensure_git_operation_not_interrupted(interrupt, "before resolving fetched commit tree")?;
3897    let tree_id = repo
3898        .find_object(commit_id)
3899        .with_context(|| format!("find fetched commit {commit_id}"))?
3900        .peel_to_tree()
3901        .with_context(|| format!("peel fetched commit {commit_id} to its tree"))?
3902        .id;
3903    ensure_git_operation_not_interrupted(interrupt, "after resolving fetched commit tree")?;
3904    ensure_git_operation_not_interrupted(interrupt, "before building exact clone index")?;
3905    let mut index = repo
3906        .index_from_tree(&tree_id)
3907        .with_context(|| format!("build index from fetched tree {tree_id}"))?;
3908    ensure_git_operation_not_interrupted(interrupt, "after building exact clone index")?;
3909    ensure_git_operation_not_interrupted(interrupt, "before building checkout options")?;
3910    let mut opts = repo
3911        .checkout_options(gix::worktree::stack::state::attributes::Source::IdMapping)
3912        .with_context(|| "build exact clone checkout options")?;
3913    ensure_git_operation_not_interrupted(interrupt, "after building checkout options")?;
3914    opts.destination_is_initially_empty = true;
3915    opts.keep_going = false;
3916    let checkout_workers = GixCheckoutWorkerLease::acquire();
3917    opts.thread_limit = Some(checkout_workers.workers());
3918
3919    progress.init(Some(index.entries().len()), gix::progress::count("files"));
3920    ensure_git_operation_not_interrupted(interrupt, "before building checkout object handle")?;
3921    let objects = repo
3922        .objects
3923        .clone()
3924        .into_arc()
3925        .with_context(|| "build thread-safe object handle for exact clone checkout")?;
3926    ensure_git_operation_not_interrupted(interrupt, "after building checkout object handle")?;
3927    ensure_git_operation_not_interrupted(interrupt, "before checkout")?;
3928    gix::worktree::state::checkout(
3929        &mut index,
3930        workdir,
3931        objects,
3932        &progress,
3933        &gix::progress::Discard,
3934        interrupt,
3935        opts,
3936    )
3937    .with_context(|| format!("check out fetched commit {commit_id}"))?;
3938    #[cfg(test)]
3939    if TEST_INTERRUPT_AFTER_CHECKOUT.swap(false, Ordering::AcqRel) {
3940        interrupt.store(true, Ordering::Release);
3941    }
3942    // gix checkout returns Ok with a partial worktree when its
3943    // interrupt flag becomes set. Never persist that partial index or
3944    // report success; the caller removes the entire clone directory.
3945    ensure_git_operation_not_interrupted(interrupt, "during checkout")?;
3946    drop(checkout_workers);
3947
3948    ensure_git_operation_not_interrupted(interrupt, "before writing exact clone index")?;
3949    index
3950        .write(Default::default())
3951        .with_context(|| "write exact clone index")?;
3952    #[cfg(test)]
3953    if TEST_INTERRUPT_AFTER_INDEX_WRITE.swap(false, Ordering::AcqRel) {
3954        interrupt.store(true, Ordering::Release);
3955    }
3956    // Publication is not complete until the persisted index has been
3957    // followed by one final observation of the shared signal epoch.
3958    // Otherwise a signal landing inside index.write() could return an
3959    // apparently successful clone whose caller proceeds to cache it.
3960    ensure_git_operation_not_interrupted(interrupt, "after writing exact clone index")?;
3961    Ok(())
3962}
3963
3964/// Use a local kernel source tree.
3965///
3966/// Dirty detection uses gix `tree_index_status` (HEAD-vs-index) and
3967/// `status().into_index_worktree_iter()` (index-vs-worktree) to check
3968/// for modifications to tracked files. Submodule checks are skipped
3969/// entirely. Untracked files do not affect the dirty flag.
3970///
3971/// When the tree is dirty, the HEAD commit does not describe the
3972/// source actually being built, so `git_hash` is dropped — no
3973/// commit identifies a dirty worktree. `is_dirty=true` carries that
3974/// fact forward; callers (see [`crate::cli`]) use it to bypass the
3975/// kernel cache entirely.
3976///
3977/// No diagnostic output: all operator-visible messaging for a
3978/// local source is routed through `kernel_build_pipeline`'s
3979/// cache-skip hint (`DIRTY_TREE_CACHE_SKIP_HINT` /
3980/// `NON_GIT_TREE_CACHE_SKIP_HINT`), which has the full context
3981/// to emit a single informational line rather than two redundant
3982/// warnings. Sibling entries (`download_tarball`, `git_clone`)
3983/// still take a `cli_label` because they genuinely print
3984/// progress lines — `local_source` does not.
3985pub fn local_source(source_path: &Path) -> Result<AcquiredSource> {
3986    let (arch, _) = arch_info();
3987
3988    if !source_path.is_dir() {
3989        anyhow::bail!("{}: not a directory", source_path.display());
3990    }
3991
3992    let canonical = source_path
3993        .canonicalize()
3994        .with_context(|| format!("canonicalize {}", source_path.display()))?;
3995
3996    let LocalSourceState {
3997        short_hash,
3998        is_dirty,
3999        is_git,
4000    } = inspect_local_source_state(&canonical)?;
4001
4002    // User .config is folded into the cache key so two builds of the
4003    // same HEAD with different `.config` files do NOT collide on the
4004    // same key — see [`config_hash_for_key`] for the encoding.
4005    // Read at `local_source` time (rather than at the post-build
4006    // store site) so cache LOOKUP and cache STORE see the same key.
4007    let user_config_hash = config_hash_for_key(&canonical);
4008
4009    let cache_key =
4010        compose_local_cache_key(arch, &short_hash, &canonical, user_config_hash.as_deref());
4011
4012    // Record the kernel version from the source-tree Makefile so the
4013    // tmpfs-fraction gate (TmpfsFraction::for_kernel_version, via the
4014    // cache metadata.json sidecar) recognizes a locally-built honoring
4015    // kernel — symmetric with the tarball path. None when the Makefile
4016    // is unreadable/unparsable, which keeps the conservative 50% default.
4017    let version = read_makefile_version(&canonical);
4018
4019    Ok(AcquiredSource {
4020        source_dir: canonical.clone(),
4021        cache_key,
4022        version,
4023        kernel_source: crate::cache::KernelSource::Local {
4024            source_tree_path: Some(canonical),
4025            git_hash: short_hash,
4026        },
4027        is_temp: false,
4028        is_dirty,
4029        is_git,
4030    })
4031}
4032
4033/// Parse the kernel `MAJOR.MINOR.PATCH` version from a source tree's
4034/// top-level `Makefile` (`VERSION` / `PATCHLEVEL` / `SUBLEVEL`) — the
4035/// authoritative version of a locally-built kernel, mirroring the
4036/// version a tarball acquisition records. Returns `None` if the
4037/// `Makefile` is unreadable or any of the three fields is absent or
4038/// non-numeric, so the caller records no version and the rootfs tmpfs
4039/// fraction conservatively defaults to 50% (the honoring gate
4040/// `TmpfsFraction::for_kernel_version` keys on a positively-known
4041/// version). `EXTRAVERSION` (e.g. `-rc7`) is intentionally ignored: the
4042/// gate keys on `MAJOR.MINOR.PATCH` only.
4043fn read_makefile_version(source_dir: &Path) -> Option<String> {
4044    let text = std::fs::read_to_string(source_dir.join("Makefile")).ok()?;
4045    // Each field is a top-of-file `NAME = N` assignment; take the first
4046    // matching line and require a bare integer (a trailing comment or
4047    // non-numeric value yields None for that field, hence overall None).
4048    let field = |name: &str| -> Option<u16> {
4049        text.lines().find_map(|line| {
4050            line.trim()
4051                .strip_prefix(name)?
4052                .trim_start()
4053                .strip_prefix('=')?
4054                .trim()
4055                .parse::<u16>()
4056                .ok()
4057        })
4058    };
4059    Some(format!(
4060        "{}.{}.{}",
4061        field("VERSION")?,
4062        field("PATCHLEVEL")?,
4063        field("SUBLEVEL")?
4064    ))
4065}
4066
4067/// Result of [`inspect_local_source_state`] — git hash and dirty/git
4068/// classification of a canonical source-tree path. Pulled out of
4069/// [`local_source`] so the post-build dirty re-check (a second call
4070/// from [`crate::cli::kernel_build_pipeline`]) reuses the exact same
4071/// gix path.
4072#[derive(Debug, Clone)]
4073pub struct LocalSourceState {
4074    /// HEAD short hash (7 chars). `None` when the tree is dirty
4075    /// (HEAD doesn't describe the actual source) or non-git (no
4076    /// HEAD at all). Mirrors the `git_hash` field on
4077    /// [`AcquiredSource::kernel_source`] for [`crate::cache::KernelSource::Local`].
4078    pub short_hash: Option<String>,
4079    /// Tracked-file dirt: HEAD-vs-index disagreement OR
4080    /// index-vs-worktree disagreement. Always `true` for non-git
4081    /// trees (dirty detection is impossible without git, so the
4082    /// pessimistic stance is dirty).
4083    pub is_dirty: bool,
4084    /// `true` when `gix::discover` succeeded (the tree is a git
4085    /// repo); `false` otherwise. Lets the cache-skip hint branch
4086    /// on whether `commit` / `stash` is actionable.
4087    pub is_git: bool,
4088}
4089
4090/// Inspect a canonical source-tree path for git hash + dirty state.
4091///
4092/// Submodule checks are skipped (false positives on kernel trees
4093/// with uninitialized submodules). The non-git arm returns
4094/// `(None, true, false)` so the caller's cache-skip hint can
4095/// distinguish "dirty git repo" from "not a git repo at all".
4096///
4097/// Called twice per build by [`crate::cli::kernel_build_pipeline`]:
4098/// once at acquire time (via [`local_source`]) and again after
4099/// `make` returns to detect mid-build worktree edits, branch flips,
4100/// or commits that would otherwise let a racing-write build land in
4101/// the cache under a stale identity. Both calls share the same gix
4102/// path so the post-build comparison is apples-to-apples.
4103///
4104/// Non-atomic against concurrent git operations: the probe runs
4105/// six sequential gix calls (`discover` → `head_id` → `head_tree`
4106/// → `index_or_empty` → `tree_index_status` → `status`), each a
4107/// separate filesystem read with no transactional bracket. A
4108/// concurrent `git commit`, `git add`, or worktree write between
4109/// any two calls can produce internally-inconsistent results —
4110/// e.g. `head_id` reads commit C0, a peer commit lands C1, then
4111/// `head_tree` reads C1's root tree and the diff against the
4112/// post-add index reports unexpected dirt. Git itself serializes
4113/// its own writes via per-resource lockfiles under `.git/`
4114/// (`index.lock` for staging operations, `HEAD.lock` and
4115/// `refs/heads/<branch>.lock` for ref updates), so peer `git`
4116/// processes wait on whichever lockfile their operation touches;
4117/// the genuinely-unsynchronized class is worktree-only writes
4118/// (autoformatter, IDE-on-save) which the index-worktree status
4119/// step catches regardless of timing.
4120///
4121/// The disposition is intentionally pessimistic so inconsistency is
4122/// safe: any `Err` propagates to the caller, which treats it as a
4123/// rebuild signal (`MidWaitState::ProbeFailed` in the mid-wait
4124/// caller); any spurious dirty signal falls into DirtyEdit /
4125/// HashAdvanced, both forcing a rebuild. The cost of a false-
4126/// positive rebuild is one extra `make`; the cost of a false-
4127/// negative would be a cache slot keyed on a HEAD that no longer
4128/// describes the source — the asymmetry is the reason for the
4129/// pessimistic disposition. Callers should treat the returned
4130/// state as a best-effort approximation of probe-time, not an
4131/// instantaneous snapshot.
4132pub fn inspect_local_source_state(canonical: &Path) -> Result<LocalSourceState> {
4133    let (short_hash, is_dirty, is_git) = match gix::discover(canonical) {
4134        Ok(repo) => {
4135            let head = repo.head_id().with_context(|| "read HEAD")?;
4136            let short_hash = format!("{}", head).chars().take(7).collect::<String>();
4137
4138            // tree_index_status compares a TREE id against the index;
4139            // the HEAD commit id is not itself a tree, so peel HEAD
4140            // to its root tree before diffing or the diff silently
4141            // returns an error and index dirt goes undetected.
4142            let head_tree = repo.head_tree().with_context(|| "read HEAD tree")?;
4143            let head_tree_id = head_tree.id;
4144
4145            // Check HEAD-vs-index for tracked file changes.
4146            let mut index_dirty = false;
4147            let index = repo.index_or_empty().with_context(|| "open index")?;
4148            let _ = repo.tree_index_status(
4149                &head_tree_id,
4150                &index,
4151                None,
4152                gix::status::tree_index::TrackRenames::Disabled,
4153                |_, _, _| {
4154                    index_dirty = true;
4155                    Ok::<_, std::convert::Infallible>(std::ops::ControlFlow::Break(()))
4156                },
4157            );
4158
4159            // Check index-vs-worktree for modified tracked files,
4160            // skipping submodules entirely (Ignore::All).
4161            let worktree_dirty = if !index_dirty {
4162                repo.status(gix::progress::Discard)
4163                    .with_context(|| "status")?
4164                    .index_worktree_rewrites(None)
4165                    .index_worktree_submodules(gix::status::Submodule::Given {
4166                        ignore: gix::submodule::config::Ignore::All,
4167                        check_dirty: false,
4168                    })
4169                    .index_worktree_options_mut(|opts| {
4170                        opts.dirwalk_options = None;
4171                        crate::git_status::configure_index_worktree_parallelism(opts);
4172                    })
4173                    .into_index_worktree_iter(Vec::new())
4174                    .map(crate::git_status::consume_has_any)
4175                    .unwrap_or(false)
4176            } else {
4177                false
4178            };
4179
4180            let is_dirty = index_dirty || worktree_dirty;
4181            // Drop the HEAD hash when dirty — the commit does not
4182            // describe the actual source being built, so publishing
4183            // it via git_hash / cache_key would misidentify the
4184            // build input.
4185            let hash = if is_dirty { None } else { Some(short_hash) };
4186            (hash, is_dirty, true)
4187        }
4188        Err(_) => {
4189            // The downstream kernel_build_pipeline (cli::kernel_build_pipeline)
4190            // emits `NON_GIT_TREE_CACHE_SKIP_HINT` — a single
4191            // informational line that names both the cause and the
4192            // remediation paths — once the is_dirty=true branch
4193            // decides to skip the cache. Emitting a second
4194            // "not a git repository" warning here duplicated that
4195            // content for every non-git `--kernel <path>` run. The
4196            // `(None, true, false)` tuple silently communicates
4197            // the non-git state to the cache-skip decision site;
4198            // no separate stderr line is needed on this path.
4199            (None, true, false)
4200        }
4201    };
4202    Ok(LocalSourceState {
4203        short_hash,
4204        is_dirty,
4205        is_git,
4206    })
4207}
4208
4209/// Compose the cache key for a local source given its arch, optional
4210/// HEAD short hash, canonical source path, and optional user
4211/// `.config` hash.
4212///
4213/// Three shapes:
4214/// - `local-{hash7}-{arch}-kc{suffix}` — clean git tree, no user
4215///   `.config` (plain `make defconfig` path or no config file yet)
4216/// - `local-{hash7}-{arch}-cfg{user_config}-kc{suffix}` — clean git
4217///   tree with a user `.config` whose hash differs from `defconfig`
4218/// - `local-unknown-{path_hash}-{arch}-kc{suffix}` — dirty / non-git
4219///   tree (HEAD does not describe the source; the path-derived
4220///   crc32 salt keeps two distinct dirty trees from colliding on the
4221///   same `local-unknown-...` slot)
4222///
4223/// `path_hash` is the full 8-char (32-bit) lowercase-hex CRC32 of
4224/// the canonical source-path bytes. CRC32 keeps the per-path
4225/// disambiguator stable across runs without pulling in a
4226/// crypto-grade hash for what is fundamentally a slot disambiguator.
4227///
4228/// `user_config_hash` is `None` whenever the source tree has no
4229/// `.config` file yet (the build will run `make defconfig` and
4230/// produce one). This collapses the user-config branch back into the
4231/// hash-only key so a fresh checkout's first build still hits a
4232/// later cache lookup keyed without the cfg segment.
4233pub fn compose_local_cache_key(
4234    arch: &str,
4235    short_hash: &Option<String>,
4236    canonical: &Path,
4237    user_config_hash: Option<&str>,
4238) -> String {
4239    let suffix = crate::cache_key_suffix();
4240    match short_hash {
4241        Some(hash) => match user_config_hash {
4242            Some(cfg) => format!("local-{hash}-{arch}-cfg{cfg}-kc{suffix}"),
4243            None => format!("local-{hash}-{arch}-kc{suffix}"),
4244        },
4245        None => {
4246            let path_hash = canonical_path_hash(canonical);
4247            format!("local-unknown-{path_hash}-{arch}-kc{suffix}")
4248        }
4249    }
4250}
4251
4252/// CRC32 of the canonical source-path bytes, lowercase hex
4253/// (full 8-char width — the entire 32-bit value). Disambiguates
4254/// `local-unknown-...` cache keys and per-source-tree lockfile
4255/// names across distinct dirty / non-git source trees so two
4256/// parallel `cargo ktstr test --kernel ./linux-a` and
4257/// `--kernel ./linux-b` runs can't write each other's vmlinux into
4258/// the same cache slot or share a single source-tree flock.
4259///
4260/// Full 32 bits (8 hex chars) of CRC32 keep collision risk
4261/// negligible against the practical population (handful of source
4262/// trees per host) while staying human-readable. The earlier
4263/// 6-char (24-bit) form left ~6× the collision surface for the
4264/// same key shape; truncation served no purpose other than visual
4265/// brevity. Path bytes are taken via `OsStr::as_encoded_bytes` so
4266/// a non-UTF-8 component (rare on Linux but possible) doesn't lose
4267/// entropy through a UTF-8 lossy conversion.
4268pub(crate) fn canonical_path_hash(canonical: &Path) -> String {
4269    let bytes = canonical.as_os_str().as_encoded_bytes();
4270    format!("{:08x}", crc32fast::hash(bytes))
4271}
4272
4273/// Read `<canonical>/.config` and return its CRC32 as a lowercase
4274/// hex string suitable for embedding in the cache key. Returns
4275/// `None` when no `.config` exists (a fresh tree before the build
4276/// runs `make defconfig`).
4277///
4278/// Distinct from the `config_hash` written into [`crate::cache::KernelMetadata`]
4279/// at store time — that records the FINAL `.config` after
4280/// configuration runs, for diagnostic display in `kernel list`.
4281/// This helper records the PRE-BUILD `.config` so the cache key
4282/// reflects what the operator's tree currently has on disk; the
4283/// same `.config` content always maps to the same key, even if the
4284/// downstream `make olddefconfig` step elaborates additional
4285/// defaults.
4286fn config_hash_for_key(canonical: &Path) -> Option<String> {
4287    let config_path = canonical.join(".config");
4288    let data = std::fs::read(&config_path).ok()?;
4289    Some(format!("{:08x}", crc32fast::hash(&data)))
4290}
4291
4292#[cfg(test)]
4293#[path = "fetch_tests.rs"]
4294mod tests;