ktstr/fetch.rs
1//! Kernel source acquisition: tarball download, GitHub codeload
2//! snapshot, git clone, local tree.
3//!
4//! The acquisition entry points each return an [`AcquiredSource`]
5//! carrying the source directory, cache key, and metadata the caller
6//! needs to proceed to configuration and build: [`download_tarball`]
7//! (kernel.org stable/RC), `download_github_archive` (a GitHub codeload
8//! commit snapshot), `git_clone_kinded` (a kind-directed shallow clone
9//! that dispatches to `git_clone_tag` / [`git_clone`]), and
10//! [`local_source`] (an on-disk tree).
11
12use std::collections::{HashSet, VecDeque};
13use std::io::Read;
14use std::net::{SocketAddr, ToSocketAddrs};
15use std::num::NonZeroU32;
16use std::path::{Path, PathBuf};
17use std::sync::OnceLock;
18use std::sync::atomic::Ordering;
19use std::time::{Duration, Instant};
20use std::{hash::BuildHasher, hash::Hasher};
21
22use ::gix;
23use anyhow::{Context, Result, anyhow};
24use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
25use md5::{Digest as _, Md5};
26use reqwest::Url;
27use reqwest::blocking::{Client, ClientBuilder};
28use sha2::{Digest, Sha256};
29
30#[path = "../build_support/gix_policy.rs"]
31mod gix_policy;
32
33/// Process-wide [`reqwest::blocking::Client`] lazily initialized on
34/// first access via [`shared_client`]. Keeping a single `Client`
35/// instance across the fetch-family reuses its TCP connection pool
36/// and TLS session cache across repeated calls to the same host
37/// within a CLI run. Cross-host fetches in the same run still
38/// re-handshake because reqwest's connection pool keys on host.
39static SHARED_CLIENT: OnceLock<Client> = OnceLock::new();
40
41/// Process interruption observed by every in-process source-acquisition
42/// operation. cargo-ktstr's signal handler updates this through the
43/// compatibility entry point [`set_git_operation_interrupted`]; binaries
44/// retaining the default signal disposition terminate directly and do not
45/// need the bridge.
46static SOURCE_OPERATION_INTERRUPTED: std::sync::atomic::AtomicBool =
47 std::sync::atomic::AtomicBool::new(false);
48
49/// Bridge a process-level signal epoch into in-process source acquisition.
50///
51/// This performs one atomic store and is safe to call from
52/// cargo-ktstr's SIGINT/SIGTERM handler. `false` starts a new guard epoch;
53/// `true` stops active and subsequent HTTP request/backoff/read/extract work
54/// as well as gix ref-map/receive/checkout work.
55#[doc(hidden)]
56pub fn set_git_operation_interrupted(interrupted: bool) {
57 SOURCE_OPERATION_INTERRUPTED.store(interrupted, Ordering::SeqCst);
58}
59
60/// Read the process-level source-acquisition interruption bridge.
61#[doc(hidden)]
62pub fn git_operation_interrupted() -> bool {
63 SOURCE_OPERATION_INTERRUPTED.load(Ordering::SeqCst)
64}
65
66fn ensure_source_operation_not_interrupted(phase: &str) -> Result<()> {
67 if SOURCE_OPERATION_INTERRUPTED.load(Ordering::Acquire) {
68 anyhow::bail!("source acquisition interrupted {phase}");
69 }
70 Ok(())
71}
72
73fn interrupted_io_error(phase: &str) -> std::io::Error {
74 std::io::Error::new(
75 std::io::ErrorKind::Interrupted,
76 format!("source acquisition interrupted {phase}"),
77 )
78}
79
80/// Connect-phase timeout for [`shared_client`]: bounds the TCP + TLS
81/// handshake before reqwest gives up on a dead peer or blackholed route.
82const SHARED_CLIENT_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
83
84/// Default blocking-operation timeout for [`shared_client`].
85///
86/// Reqwest's blocking client applies this budget to connection, request-write,
87/// and response-read waits. Keeping it explicit here means a response that
88/// delivered headers but then stopped producing body bytes cannot wedge a
89/// caller that forgot a per-request override. Small metadata/index/probe
90/// requests also pass this value explicitly; large archive and package
91/// downloads override it with [`DOWNLOAD_REQUEST_READ_TIMEOUT`].
92const SMALL_RESPONSE_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
93
94/// Return the process-wide shared [`reqwest::blocking::Client`]. First
95/// call constructs it via `Client::builder()` with
96/// `SHARED_CLIENT_CONNECT_TIMEOUT` and
97/// `SMALL_RESPONSE_REQUEST_TIMEOUT` applied; every subsequent call returns
98/// a reference to the same instance. This helper is for top-level CLI entries
99/// that want the default client.
100///
101/// Tests that need to verify a network round-trip (rather than a
102/// cache hit) must NOT pass `shared_client()` to a cache-routed
103/// helper (`cached_releases`, `cached_releases_with`,
104/// [`fetch_latest_stable_version`], [`fetch_version_for_prefix`]) —
105/// `RELEASES_CACHE` may already be populated by a peer test, in
106/// which case the helper returns cached data and the network is
107/// never touched. Construct a local `Client` and pass it to the
108/// cache-routed helper to skip the cache; the pointer-equality gate
109/// in `cached_releases_with` routes a non-singleton client to a
110/// direct `fetch_releases` call against `RELEASES_URL` (the
111/// production URL — the bypass skips the cache, NOT the URL). For
112/// full URL injection, call `fetch_releases` directly. Cache-routing
113/// tests inject the fetch operation into `cached_releases_with_fetch`
114/// so they can prove the bypass without scheduling a local HTTP
115/// server.
116///
117/// # Panics
118///
119/// Panics on the first call if `Client::builder().build()` fails to
120/// construct a client. Documented failure modes include TLS backend
121/// initialization (e.g. rustls/native-tls subsystem unreachable) and
122/// system-resolver config load failure; both are treated as setup
123/// bugs rather than runtime errors. The
124/// `expect` here, rather than propagating the error, mirrors the
125/// inherited behavior of `reqwest::blocking::Client::new()` (which
126/// is itself an infallible wrapper around `builder().build().expect`).
127pub fn shared_client() -> &'static Client {
128 SHARED_CLIENT.get_or_init(|| {
129 default_http_client_builder()
130 .build()
131 .expect("build shared reqwest client")
132 })
133}
134
135/// Construct the common reqwest client builder used by the process-wide
136/// client and by retry-only DNS-edge clients.
137///
138/// Keeping the builder policy in one place makes a routed retry retain the
139/// shared client's connect timeout, TLS defaults, and automatic system-proxy
140/// handling. A routed client adds only a per-origin DNS override; it does not
141/// rewrite the URL, Host header, TLS SNI, or proxy configuration.
142fn default_http_client_builder() -> ClientBuilder {
143 Client::builder()
144 .connect_timeout(SHARED_CLIENT_CONNECT_TIMEOUT)
145 .timeout(SMALL_RESPONSE_REQUEST_TIMEOUT)
146}
147
148/// Process-wide cache of the parsed `releases.json` payload.
149/// Populated by [`cached_releases_with`] on its first successful
150/// singleton-path fetch; every subsequent singleton call returns a
151/// clone of the cached vector without re-issuing the HTTP request.
152/// Lifetime matches the process — `releases.json` does not change
153/// underneath a single CLI invocation, so a per-process cache
154/// cannot serve stale data in any way the user would notice.
155///
156/// Failures are NOT cached: a transient kernel.org outage that
157/// errors the first call must allow a later caller to retry, since
158/// the underlying network condition may have cleared. Storing
159/// `Vec<Release>` rather than `Result<Vec<Release>>` enforces this
160/// at the type level — there's no way to populate the cache with
161/// a failure.
162///
163/// Companion to [`SHARED_CLIENT`]: both amortize per-invocation
164/// network cost across the resolve pipeline. Without this cache,
165/// `cargo ktstr test --kernel 6.10..6.12 --kernel 6.14..6.16`
166/// fetches `releases.json` twice — once per Range spec — under
167/// the rayon par_iter that drives `resolve_kernel_set`. With
168/// the cache the first Range to reach `expand_kernel_range`
169/// populates the slot; the second observes the populated slot
170/// and skips the network entirely.
171static RELEASES_CACHE: OnceLock<Vec<Release>> = OnceLock::new();
172
173/// Cache for the gregkh stable-mirror release tags — the `X.Y.Z`
174/// version strings parsed from its `refs/tags/vX.Y.Z` advertisement.
175/// Companion to [`RELEASES_CACHE`]: `--include-eol` may expand several
176/// `A..B` ranges under one `resolve_kernel_set`, and each would
177/// otherwise re-ls-remote the mirror. Populated on the first successful
178/// enumeration; a failed ls-remote leaves it empty so the next caller
179/// retries (`Vec`, not `Result`, mirroring `RELEASES_CACHE`).
180static STABLE_TAGS_CACHE: OnceLock<Vec<String>> = OnceLock::new();
181
182/// Fetch `releases.json` via the process-wide [`shared_client`],
183/// routing through [`RELEASES_CACHE`].
184///
185/// Thin wrapper for callers that don't already thread a `&Client`
186/// — top-level CLI entries like [`crate::cli::expand_kernel_range`]
187/// (under the rayon-driven `cargo ktstr` resolve pipeline) and
188/// `crate::cli::fetch_active_prefixes` (the EOL-annotation pass).
189/// Caching, race semantics, and fault-injection routing are all
190/// documented on [`cached_releases_with`].
191pub(crate) fn cached_releases() -> Result<Vec<Release>> {
192 cached_releases_with(shared_client())
193}
194
195/// Pointer-equality against the [`OnceLock`]-backed
196/// [`shared_client`] singleton is the correct predicate because
197/// `shared_client()` returns a stable `&'static Client` address.
198/// The [`cached_releases_with`] gate uses this predicate to
199/// decide whether to consult [`RELEASES_CACHE`]: the singleton
200/// hits the cache, every other (test-constructed) `Client`
201/// bypasses it and exercises the underlying [`fetch_releases`]
202/// path.
203///
204/// Caveat: `shared_client().clone()` produces a distinct
205/// `Client` at a different address even though it shares the
206/// singleton's connection pool via the inner `Arc`, so the
207/// clone bypasses the cache. Always pass `shared_client()`
208/// directly — never a clone — when cache routing is desired.
209///
210/// Side-effect-free when [`SHARED_CLIENT`] is uninitialized:
211/// no client can equal a not-yet-allocated singleton, so we
212/// return `false` without triggering `get_or_init` — tests
213/// that pass a local `Client` before any production code path
214/// has touched the singleton skip the construction entirely.
215fn is_shared_client(client: &Client) -> bool {
216 match SHARED_CLIENT.get() {
217 Some(singleton) => std::ptr::eq(client, singleton),
218 None => false,
219 }
220}
221
222/// Unified cache-aware entry point for `releases.json`. Routes
223/// the process-wide [`shared_client`] singleton through
224/// [`RELEASES_CACHE`]; any other (test-constructed) `Client`
225/// bypasses [`RELEASES_CACHE`] and calls [`fetch_releases`] with
226/// [`RELEASES_URL`] directly — the cache is skipped but the
227/// production URL is used.
228///
229/// Used by every in-file caller that already threads a `&Client`
230/// — [`fetch_latest_stable_version`], [`fetch_version_for_prefix`],
231/// [`latest_in_series`] — so production callers reuse
232/// [`RELEASES_CACHE`] and tests still get cache-bypass via the
233/// pointer-equality gate. [`cached_releases`] is the no-`Client`
234/// wrapper for top-level CLI entries.
235///
236/// This wrapper is the production entry point and
237/// pins the URL to [`RELEASES_URL`]; production code MUST go
238/// through this wrapper. A singleton call with a non-RELEASES_URL
239/// would otherwise populate [`RELEASES_CACHE`] with
240/// non-production data and corrupt every later production
241/// call — the singleton-path branch in
242/// [`cached_releases_with_url`] guards against this in both
243/// dev (`debug_assert!`) and release builds (fall back to
244/// bypass), but routing every production call through this
245/// wrapper makes the misuse impossible by construction.
246/// Caching, race semantics, and the bypass-vs-cache routing
247/// are fully documented on [`cached_releases_with_url`].
248fn cached_releases_with(client: &Client) -> Result<Vec<Release>> {
249 cached_releases_with_url(client, RELEASES_URL)
250}
251
252/// URL-explicit form of [`cached_releases_with`]. Production reaches this
253/// through the wrapper above, which pins `url` to [`RELEASES_URL`]. The
254/// routing core below accepts an injected fetch operation for deterministic
255/// cache/bypass tests.
256///
257/// Cache contract is identical to [`cached_releases_with`]:
258/// non-singleton clients bypass [`RELEASES_CACHE`] and call
259/// [`fetch_releases`] with `url`; the singleton routes through
260/// the cache only when `url == RELEASES_URL` (consulting via
261/// `OnceLock::get`, populating via `OnceLock::set` on miss). A
262/// singleton call with a non-RELEASES_URL trips the
263/// `debug_assert!` in dev builds and falls back to the bypass
264/// behavior in release builds — fetches directly via `url`,
265/// returns the result, never touches [`RELEASES_CACHE`]. The
266/// cache only ever stores data fetched from the singleton +
267/// RELEASES_URL combination, so a test that injects a mock URL
268/// on either branch cannot pollute the production cache.
269///
270/// Failures are propagated without populating [`RELEASES_CACHE`],
271/// so a transient kernel.org outage on the first call lets the
272/// next caller retry. Storing `Vec<Release>` (not
273/// `Result<Vec<Release>>`) enforces this at the type level.
274///
275/// Concurrent population on the singleton path is safe via the
276/// `OnceLock::set` race: the loser's `set` returns `Err(clone)`
277/// (the cloned vector that was passed in is moved back), the
278/// returned `Err` is discarded via `let _ = …`, and the loser
279/// returns its own original `fresh` vector. Both winner and
280/// loser return content-equivalent data since both fetched the
281/// same `releases.json`. Worst case under concurrent first
282/// calls: both callers issue the network round-trip, only one
283/// populates [`RELEASES_CACHE`]; every later call — from any
284/// thread — observes the populated slot via the `get` fast-path
285/// and skips the network.
286fn cached_releases_with_url(client: &Client, url: &str) -> Result<Vec<Release>> {
287 cached_releases_with_fetch(client, url, fetch_releases)
288}
289
290/// Cache-routing core with an injectable fetch operation.
291///
292/// Production passes [`fetch_releases`]. Keeping the network operation behind
293/// this narrow seam lets cache-routing tests prove singleton/bypass behavior
294/// without starting a localhost HTTP runtime whose response thread can be
295/// starved by a full CI storm.
296fn cached_releases_with_fetch<F>(client: &Client, url: &str, fetch: F) -> Result<Vec<Release>>
297where
298 F: FnOnce(&Client, &str) -> Result<Vec<Release>>,
299{
300 // Non-singleton clients bypass the cache (test fault injection).
301 if !is_shared_client(client) {
302 return fetch(client, url);
303 }
304 // Cache-poison guard: the singleton path populates
305 // RELEASES_CACHE on miss. A test author that mistakenly
306 // passes a non-production URL with shared_client() would
307 // fill the cache with non-production data and corrupt every
308 // later production call (which reaches the cache via
309 // get-fast-path). Catch the misuse at debug-build time —
310 // production callers always thread RELEASES_URL through the
311 // `cached_releases_with` wrapper, so the assertion is a
312 // no-op for them; only a future test author wiring this
313 // function up with shared_client() and a mock URL would trip
314 // it.
315 debug_assert!(
316 url == RELEASES_URL,
317 "cached_releases_with_url: shared_client() must use RELEASES_URL \
318 to avoid RELEASES_CACHE pollution — got url={url:?}, expected \
319 RELEASES_URL ({RELEASES_URL:?}). Tests that need URL injection \
320 must pass a non-singleton Client (which takes the bypass branch \
321 above and never touches the cache).",
322 );
323 // Release-build guard: `debug_assert!` is stripped in
324 // optimized builds, so a non-RELEASES_URL on the singleton
325 // path would otherwise reach the populate-on-miss path below
326 // and persistently poison RELEASES_CACHE for every later
327 // production caller. Mirror the bypass-branch behavior
328 // (fetch directly, do not touch the cache) so the misuse
329 // degrades to a slow per-call fetch instead of a permanently
330 // wrong cache. The debug_assert above still fires loudly in
331 // dev builds; this branch only catches the misuse that
332 // slipped through to release.
333 if url != RELEASES_URL {
334 return fetch(client, url);
335 }
336 if let Some(cached) = RELEASES_CACHE.get() {
337 return Ok(cached.clone());
338 }
339 let fresh = fetch(client, url)?;
340 // Race-loss: `set` returns `Err(clone)` carrying back the
341 // clone we passed in; we discard it and return the original
342 // `fresh` below. See the rustdoc above for full semantics.
343 let _ = RELEASES_CACHE.set(fresh.clone());
344 Ok(fresh)
345}
346
347/// Downloaded/cloned kernel source ready for building.
348#[non_exhaustive]
349pub struct AcquiredSource {
350 /// Path to the kernel source directory.
351 pub source_dir: PathBuf,
352 /// Cache key for this source (e.g. "6.14.2-tarball-x86_64-kc{kconfig_hash}").
353 pub cache_key: String,
354 /// Version string if known (e.g. "6.14.2", "6.15-rc3").
355 pub version: Option<String>,
356 /// How the source was acquired, with per-variant payload
357 /// (git hash/ref for `Git`, source tree path and git hash for
358 /// `Local`).
359 pub kernel_source: crate::cache::KernelSource,
360 /// Whether the source is a temporary directory that should be
361 /// cleaned up after building.
362 pub is_temp: bool,
363 /// For local sources: whether the working tree is dirty.
364 /// Dirty trees must not be cached.
365 pub is_dirty: bool,
366 /// For local sources: whether the source is an actual git
367 /// repository. `true` when `gix::discover` succeeded and the
368 /// crate could compute index + worktree dirty state; `false`
369 /// for non-git source trees (tarball-extracted, rsync'd,
370 /// hand-assembled) where dirty detection is impossible and
371 /// the source is always cache-skipped pessimistically. Lets
372 /// the cache-skip hint branch on whether `commit` / `stash`
373 /// are actionable remediations (they aren't for non-git
374 /// sources).
375 ///
376 /// For non-local sources (tarball, git clone) the field is
377 /// set to `true` by convention — these paths are always
378 /// `is_dirty = false`, so the cache-skip branch that reads
379 /// `is_git` is never reached and the value is inert. Pinning
380 /// to `true` (rather than leaving the field meaningless)
381 /// keeps the invariant "is_git is meaningful only when
382 /// is_dirty is true, but always set" so a future code path
383 /// that reaches `is_git` outside the cache-skip context does
384 /// not trip on an `is_git = false` under a known-good source.
385 pub is_git: bool,
386}
387
388/// Target architecture string and boot image name.
389pub fn arch_info() -> (&'static str, &'static str) {
390 #[cfg(target_arch = "x86_64")]
391 {
392 ("x86_64", "bzImage")
393 }
394 #[cfg(target_arch = "aarch64")]
395 {
396 ("aarch64", "Image")
397 }
398}
399
400/// Parse a version string into its major version for URL construction.
401///
402/// "6.14.2" -> 6, "6.15-rc3" -> 6.
403fn major_version(version: &str) -> Result<u32> {
404 let major_str = version
405 .split('.')
406 .next()
407 .ok_or_else(|| anyhow!("invalid version: {version}"))?;
408 major_str
409 .parse::<u32>()
410 .with_context(|| format!("invalid major version in {version}"))
411}
412
413/// Determine if a version string represents an RC release.
414///
415/// RC releases use a different URL pattern and gzip compression
416/// (vs xz for stable).
417fn is_rc(version: &str) -> bool {
418 version.contains("-rc")
419}
420
421/// One (`moniker`, `version`) row from kernel.org's `releases.json`.
422///
423/// A named struct instead of a bare `(String, String)` tuple so every
424/// call site reads its field by name (`r.moniker`, `r.version`) rather
425/// than positional destructuring — the two strings are trivially
426/// swappable at a tuple-destructure call site, and a silent swap
427/// would mis-drive `is_skippable_release_moniker` while the
428/// now-misnamed "moniker" string flows into `version_prefix`
429/// downstream. Naming the fields removes that class of bug at the
430/// type-checker level and shows up in IDE hints on every iteration
431/// site.
432///
433/// Both fields are owned `String` (not `&str`) because the values are
434/// parsed out of a `reqwest::Response` body whose lifetime ends when
435/// `fetch_releases` returns; downstream callers iterate the vector
436/// long after that borrow would dangle.
437#[derive(Clone, Debug)]
438pub(crate) struct Release {
439 /// releases.json `moniker` field — stable / longterm / mainline /
440 /// linux-next / etc. Consumed by
441 /// [`is_skippable_release_moniker`] and by
442 /// [`fetch_latest_stable_version`]'s stable/longterm filter.
443 pub moniker: String,
444 /// releases.json `version` field — e.g. `"6.14.2"`, `"6.15-rc3"`,
445 /// `"6.16-rc2-next-20260420"`. Consumed by
446 /// [`version_tuple`], [`patch_level`], and
447 /// `cli::version_prefix`.
448 pub version: String,
449}
450
451/// Is this releases.json moniker one that the version-resolution
452/// pipeline should skip?
453///
454/// `linux-next` is a rolling integration branch whose version strings
455/// carry a date suffix rather than a stable tag, so it does not fit
456/// the major.minor.patch resolution model used by `latest_in_series`,
457/// `fetch_version_for_prefix`, and `cli::fetch_active_prefixes`. The
458/// release iteration in all three sites filters it out; this helper
459/// is the single point of truth for that decision so a future moniker
460/// that also warrants skipping can be added in one place.
461pub(crate) fn is_skippable_release_moniker(moniker: &str) -> bool {
462 moniker == "linux-next"
463}
464
465/// Find the latest version in the same major.minor series from releases.json.
466///
467/// Returns `Some("6.14.10")` for prefix `"6.14"` if that series exists in
468/// releases.json. Returns `None` if the series is not found (EOL or invalid).
469fn latest_in_series(client: &Client, version: &str) -> Option<String> {
470 let prefix = {
471 let parts: Vec<&str> = version.split('.').collect();
472 if parts.len() >= 2 {
473 format!("{}.{}", parts[0], parts[1])
474 } else {
475 return None;
476 }
477 };
478
479 // Routes through [`RELEASES_CACHE`] for the singleton; see
480 // [`cached_releases_with`] for the bypass gate.
481 let releases = cached_releases_with(client).ok()?;
482 let mut best: Option<(String, (u32, u32, u32))> = None;
483 for r in &releases {
484 if is_skippable_release_moniker(&r.moniker) {
485 continue;
486 }
487 if !r.version.starts_with(&prefix) {
488 continue;
489 }
490 if r.version.len() != prefix.len() && r.version.as_bytes()[prefix.len()] != b'.' {
491 continue;
492 }
493 if let Some(tuple) = version_tuple(&r.version)
494 && (best.is_none() || tuple > best.as_ref().unwrap().1)
495 {
496 best = Some((r.version.clone(), tuple));
497 }
498 }
499 best.map(|(v, _)| v)
500}
501
502/// Build a user-facing error message for a version that was not found.
503///
504/// Suggests the latest version in the same major.minor series when
505/// releases.json contains one.
506fn version_not_found_msg(client: &Client, version: &str) -> String {
507 let parts: Vec<&str> = version.split('.').collect();
508 let prefix = if parts.len() >= 2 {
509 format!("{}.{}", parts[0], parts[1])
510 } else {
511 version.to_string()
512 };
513 match latest_in_series(client, version) {
514 Some(latest) if latest != version => {
515 format!("version {version} not found. latest {prefix}.x: {latest}")
516 }
517 _ => format!("version {version} not found"),
518 }
519}
520
521/// Reject responses where the server returned HTML instead of a binary
522/// archive. Some CDN error pages return 200 with text/html.
523fn reject_html_response(response: &reqwest::blocking::Response, url: &str) -> Result<()> {
524 if let Some(ct) = response.headers().get(reqwest::header::CONTENT_TYPE)
525 && let Ok(ct_str) = ct.to_str()
526 && ct_str.contains("text/html")
527 {
528 anyhow::bail!(
529 "download {url}: server returned HTML instead of tarball (URL may be invalid)"
530 );
531 }
532 Ok(())
533}
534
535/// Print download size from Content-Length header if available.
536///
537/// `cli_label` prefixes the diagnostic line so the message matches the
538/// binary the user invoked (`"ktstr"` vs `"cargo ktstr"`).
539fn print_download_size(
540 response: &reqwest::blocking::Response,
541 url: &str,
542 cli_label: &str,
543 mp: Option<&crate::cli::FetchProgress>,
544) {
545 let line = if let Some(len) = response.content_length() {
546 let mib = len as f64 / (1024.0 * 1024.0);
547 format!("{cli_label}: downloading {url} ({mib:.1} MiB)")
548 } else {
549 format!("{cli_label}: downloading {url}")
550 };
551 // Route through the progress group so the line coordinates with
552 // concurrent bars on a TTY (and still reaches piped/CI stderr when
553 // the group is hidden); raw `eprintln!` when no group is present.
554 match mp {
555 Some(fp) => fp.println(&line),
556 None => crate::cli::print_status_line(&line),
557 }
558}
559
560/// Maximum tolerated stretch of "no body bytes received" before a
561/// streaming download is declared stalled. Catches a TCP connection
562/// that completed handshake (so connect_timeout doesn't fire) but
563/// then silently stops delivering body data — a common CDN failure
564/// mode where keepalive holds the socket open while the upstream
565/// origin is unreachable. The 60s value is generous enough that a
566/// real slow uplink delivering chunks every few seconds never
567/// triggers it, but tight enough that a wedged connection surfaces
568/// before the run's overall test timeout.
569const DOWNLOAD_NO_PROGRESS_TIMEOUT: Duration = Duration::from_secs(60);
570
571/// Streaming `Read` adapter for kernel tarball downloads.
572///
573/// Wraps the [`reqwest::blocking::Response`] body to do two things
574/// the bare response cannot:
575///
576/// 1. **Body-progress watchdog.** Tracks `last_progress` (the
577/// instant of the last successful read with `n > 0`) and errors
578/// when more than [`DOWNLOAD_NO_PROGRESS_TIMEOUT`] elapses
579/// between byte-producing reads. Without this, a CDN edge that
580/// keepalives the socket but stops delivering body bytes would
581/// only surface after reqwest's per-request read timeout
582/// ([`DOWNLOAD_REQUEST_READ_TIMEOUT`], 300s), which bounds a
583/// single stalled `read()`; the watchdog applies the tighter
584/// 60s no-progress bound across successive reads. The check fires
585/// BEFORE the inner `read()` so a stalled inner reader cannot
586/// out-block the watchdog.
587///
588/// 2. **Streaming checksums.** Updates SHA-256 and MD5 hashers with
589/// every byte that flows past, so kernel.org/distro SHA-256 and
590/// Google Cloud Storage MD5 metadata can both use this one
591/// watchdog/progress stream without a second pass over the data.
592///
593/// Sits between [`reqwest::blocking::Response`] and the
594/// decompression layer (`XzDecoder` / `GzDecoder`); both
595/// decompressors expose `into_inner()` so the wrapper can be
596/// recovered after extraction completes (see
597/// [`Self::finalize`]).
598struct DownloadStream<R: Read> {
599 /// Underlying reqwest response body. Owned because `XzDecoder`
600 /// and `GzDecoder` take ownership of their inner reader, so
601 /// the wrapper must hold the response by value rather than by
602 /// reference.
603 inner: R,
604 /// Running SHA-256 hasher updated on every byte-producing read.
605 /// Consumed by [`DownloadStream::finalize`] (which takes `self`
606 /// by value); the call site recovers the wrapper from inside
607 /// the decoder + tar archive chain via `into_inner` before
608 /// finalizing.
609 sha256: Sha256,
610 /// Running MD5 hasher for GCS `md5Hash` verification. GCS exposes
611 /// this digest as standard base64 rather than hexadecimal.
612 md5: Md5,
613 /// Total body bytes read so far. Surfaced in the watchdog
614 /// error message so an operator triaging "no progress" can see
615 /// how many bytes did arrive before the stall — distinguishing
616 /// "connection dropped after a few bytes" from "connection
617 /// dropped after most of the payload".
618 bytes_total: u64,
619 /// `Instant` of the last successful read with `n > 0`. Set at
620 /// construction (not on first read) so a connection that wins
621 /// the handshake but never delivers any body bytes still
622 /// trips the watchdog after [`DOWNLOAD_NO_PROGRESS_TIMEOUT`]
623 /// rather than waiting for an indeterminate pre-data window.
624 last_progress: Instant,
625 /// Tolerated stretch of zero-progress time. Pinned at
626 /// construction from [`DOWNLOAD_NO_PROGRESS_TIMEOUT`]; held in
627 /// the struct rather than read from the constant on every
628 /// `read()` so a future per-call override (e.g. shorter
629 /// timeouts in tests) lands without touching the watchdog
630 /// logic.
631 no_progress_timeout: Duration,
632 /// Optional indicatif download bar, advanced by `inc(n)` on
633 /// every byte-producing read in lockstep with `bytes_total`.
634 /// `None` is the no-bar path (non-TTY, or no progress group
635 /// threaded in) and carries zero per-read overhead beyond the
636 /// `Option` check. Advancing here — the single byte-accounting
637 /// site — guarantees `bar.position() == finalize().1`, so the
638 /// bar can never drift from the bytes the hasher and watchdog
639 /// observed.
640 progress: Option<indicatif::ProgressBar>,
641}
642
643impl<R: Read> DownloadStream<R> {
644 /// Construct a streaming wrapper around `inner` with the production
645 /// no-progress budget, optionally attaching an indicatif progress
646 /// bar. `last_progress` is set to "now" so the watchdog clock starts
647 /// at construction; the downstream decoder may take an indeterminate
648 /// time before the first `read()`, but any actual progress resets
649 /// the clock. The optional bar is advanced by `inc(n)` on every
650 /// byte-producing read (see the `progress` field); `progress = None`
651 /// is the non-TTY / no-group path (no bar). The bar is a pure
652 /// observer — it never affects the watchdog gate or the streaming
653 /// sha256, so a stalled or truncated download still surfaces its
654 /// error unchanged.
655 fn with_progress(inner: R, progress: Option<indicatif::ProgressBar>) -> Self {
656 Self {
657 inner,
658 sha256: Sha256::new(),
659 md5: Md5::new(),
660 bytes_total: 0,
661 last_progress: Instant::now(),
662 no_progress_timeout: DOWNLOAD_NO_PROGRESS_TIMEOUT,
663 progress,
664 }
665 }
666
667 /// Consume the wrapper and return `(hex_digest, bytes_total)`.
668 /// Lowercase hex matches the format kernel.org publishes in
669 /// `sha256sums.asc`, so the caller can do a direct
670 /// `eq_ignore_ascii_case` comparison without re-encoding.
671 fn finalize(self) -> (String, u64) {
672 (hex::encode(self.sha256.finalize()), self.bytes_total)
673 }
674
675 /// Consume the wrapper and return the GCS-style standard-base64
676 /// MD5 digest plus the byte total.
677 fn finalize_md5_base64(self) -> (String, u64) {
678 (
679 BASE64_STANDARD.encode(self.md5.finalize()),
680 self.bytes_total,
681 )
682 }
683}
684
685impl<R: Read> Read for DownloadStream<R> {
686 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
687 if SOURCE_OPERATION_INTERRUPTED.load(Ordering::Acquire) {
688 return Err(interrupted_io_error("before reading response body"));
689 }
690 // Watchdog gate: trip BEFORE delegating to the inner reader
691 // so a stalled inner read does not get a fresh chance to
692 // run after the no-progress window has already expired. The
693 // wrapper cannot interrupt a `read()` that is currently
694 // blocked in a syscall — that protection comes from the
695 // per-request timeout configured via
696 // `RequestBuilder::timeout` — but it can refuse to issue
697 // the next call once the cumulative no-progress window
698 // crosses the bound.
699 let elapsed = self.last_progress.elapsed();
700 if elapsed > self.no_progress_timeout {
701 return Err(std::io::Error::new(
702 std::io::ErrorKind::TimedOut,
703 format!(
704 "download stalled: no body bytes for {}s after {} bytes received",
705 elapsed.as_secs(),
706 self.bytes_total,
707 ),
708 ));
709 }
710 match self.inner.read(buf) {
711 Ok(0) => {
712 // EOF: do NOT update last_progress — a 0-byte read
713 // is not progress, and updating here would let a
714 // decoder that polls past EOF reset the watchdog
715 // indefinitely.
716 Ok(0)
717 }
718 Ok(n) => {
719 // A signal can land while the inner blocking read owns this
720 // thread. Reject the bytes before feeding a decoder, checksum,
721 // or destination file so the caller's transaction rolls back
722 // instead of publishing after cancellation.
723 if SOURCE_OPERATION_INTERRUPTED.load(Ordering::Acquire) {
724 return Err(interrupted_io_error("while reading response body"));
725 }
726 self.sha256.update(&buf[..n]);
727 self.md5.update(&buf[..n]);
728 self.bytes_total += n as u64;
729 self.last_progress = Instant::now();
730 // Advance the bar in lockstep with `bytes_total` (same
731 // `n`, same reads) so `position()` and `finalize().1`
732 // never diverge. No-op when no bar is attached.
733 if let Some(pb) = &self.progress {
734 pb.inc(n as u64);
735 }
736 Ok(n)
737 }
738 Err(e) => Err(e),
739 }
740 }
741}
742
743/// Per-request body-stream timeout passed to
744/// [`reqwest::blocking::RequestBuilder::timeout`] for tarball
745/// downloads. The blocking client treats this as a per-`read()`
746/// deadline (reset on every successful read), so it complements the
747/// [`DownloadStream`] watchdog: reqwest's deadline kills a single
748/// stalled syscall, and the watchdog observes the cumulative
749/// no-progress window across multiple reads. Set generously
750/// (5 minutes) because a slow but progressing connection can
751/// legitimately take that long for a single read on a large CDN
752/// chunk; the watchdog provides the tighter 60s no-progress bound.
753const DOWNLOAD_REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(300);
754
755/// Retry policy for [`get_with_transient_retry`]: attempt count and
756/// the unit the exponential backoff scales from (sleep =
757/// `backoff_unit << attempt`, i.e. 2s/4s/8s for a 1s unit — the same
758/// timing as `build_helpers::retry_with_backoff`, which covers the
759/// build-script blob downloads; this covers the runtime kernel-source
760/// fetches). A struct rather than two bare params so the production
761/// sites share one const and the fetch_tests seam can inject a
762/// zero-backoff policy without sleeping through real backoff.
763struct HttpRetry {
764 attempts: u32,
765 backoff_unit: Duration,
766}
767
768/// Production retry policy: 4 attempts, 2s/4s/8s between them.
769/// Matches `MAX_TARBALL_ATTEMPTS`/`MAX_CLONE_ATTEMPTS` in build.rs so
770/// every network fetch in the project tolerates the same outage
771/// window (~14s) before giving up.
772const TRANSIENT_HTTP_RETRY: HttpRetry = HttpRetry {
773 attempts: 4,
774 backoff_unit: Duration::from_secs(1),
775};
776
777/// Whether an HTTP status is worth retrying: 429 (rate limit) and the
778/// gateway trio 502/503/504. kernel.org fronts cdn.kernel.org with
779/// Varnish, which returns bursts of `503` with `retry-after: 0`
780/// during cache churn — the exact failure that killed every
781/// `just kernel-build` job in one CI run. 500 is deliberately NOT
782/// retried: it signals a server bug rather than a transient edge
783/// condition, and the existing contract (pinned by
784/// `download_stable_tarball_500_is_hard_error_without_marker`) treats
785/// it as an immediate hard error.
786fn is_transient_http_status(status: reqwest::StatusCode) -> bool {
787 matches!(status.as_u16(), 429 | 502 | 503 | 504)
788}
789
790/// Whether a retryable status identifies a failed HTTP gateway/CDN edge.
791///
792/// 429 remains retryable but deliberately does not select another address:
793/// changing peers to evade a server's rate limit is incorrect. The gateway
794/// trio instead says that a connectable peer could not serve the request, so
795/// another address published for the same hostname is the useful next route.
796fn is_dns_edge_failure_status(status: reqwest::StatusCode) -> bool {
797 matches!(status.as_u16(), 502..=504)
798}
799
800/// Remaining DNS addresses for one origin during a single retry loop.
801struct HttpRetryRoutes {
802 host: String,
803 port: u16,
804 remaining: VecDeque<SocketAddr>,
805}
806
807impl HttpRetryRoutes {
808 fn matches(&self, host: &str, port: u16) -> bool {
809 self.host == host && self.port == port
810 }
811
812 fn matches_url(&self, url: &Url) -> bool {
813 url.host_str()
814 .zip(url.port_or_known_default())
815 .is_some_and(|(host, port)| self.matches(host, port))
816 }
817
818 fn mark_failed(&mut self, addr: SocketAddr) {
819 self.remaining
820 .retain(|candidate| candidate.ip() != addr.ip());
821 }
822}
823
824/// Resolve every address currently published for an HTTP origin.
825fn resolve_http_origin(host: &str, port: u16) -> std::io::Result<Vec<SocketAddr>> {
826 (host, port).to_socket_addrs().map(Iterator::collect)
827}
828
829/// Build a fresh client with an ordered set of origin-address candidates while
830/// retaining the hostname in the request URL.
831///
832/// Passing the whole remaining tail lets reqwest retain its connector-level
833/// multi-address/Happy-Eyeballs fallback within one attempt. The URL hostname
834/// still supplies the Host header and TLS SNI. If a system proxy intercepts
835/// the request, reqwest connects to that proxy instead; these origin
836/// candidates then remain advisory and do not claim which upstream peer the
837/// proxy selects.
838fn build_routed_http_client(host: &str, addrs: &[SocketAddr]) -> reqwest::Result<Client> {
839 default_http_client_builder()
840 .resolve_to_addrs(host, addrs)
841 .build()
842}
843
844/// Replace the retry-address queue with a fresh resolution of `request_url`.
845///
846/// Returning `false` means the URL has no routable HTTP origin (host or known
847/// port), so the caller should preserve the ordinary same-client retry.
848fn reset_http_retry_routes<R>(
849 routes: &mut Option<HttpRetryRoutes>,
850 request_url: &Url,
851 resolve: &R,
852) -> std::io::Result<bool>
853where
854 R: Fn(&str, u16) -> std::io::Result<Vec<SocketAddr>>,
855{
856 let Some(host) = request_url.host_str() else {
857 *routes = None;
858 return Ok(false);
859 };
860 let Some(port) = request_url.port_or_known_default() else {
861 *routes = None;
862 return Ok(false);
863 };
864
865 let mut seen = HashSet::new();
866 let remaining: VecDeque<_> = resolve(host, port)?
867 .into_iter()
868 .filter(|addr| seen.insert(*addr))
869 .collect();
870 let has_candidates = !remaining.is_empty();
871 *routes = Some(HttpRetryRoutes {
872 host: host.to_owned(),
873 port,
874 remaining,
875 });
876 Ok(has_candidates)
877}
878
879/// Populate or update the alternate-address queue after a gateway response.
880fn record_failed_http_edge<R>(
881 routes: &mut Option<HttpRetryRoutes>,
882 response_url: &Url,
883 failed_remote: Option<SocketAddr>,
884 resolve: &R,
885) -> std::io::Result<bool>
886where
887 R: Fn(&str, u16) -> std::io::Result<Vec<SocketAddr>>,
888{
889 let Some(host) = response_url.host_str() else {
890 return Ok(false);
891 };
892 let Some(port) = response_url.port_or_known_default() else {
893 return Ok(false);
894 };
895
896 if routes
897 .as_ref()
898 .is_none_or(|existing| !existing.matches(host, port))
899 {
900 reset_http_retry_routes(routes, response_url, resolve)?;
901 }
902 if let (Some(existing), Some(failed)) = (routes.as_mut(), failed_remote) {
903 existing.mark_failed(failed);
904 }
905
906 Ok(routes
907 .as_ref()
908 .is_some_and(|existing| !existing.remaining.is_empty()))
909}
910
911/// Prepare the next routed attempt after a transport error.
912///
913/// [`next_routed_http_client`] removes the current primary before issuing the
914/// request, leaving only untried candidates in `routes.remaining`. When the
915/// error URL still names that origin, preserve the advanced tail rather than
916/// re-resolving and retrying the same primary again. A missing route set
917/// (initial-client failure) or a different error origin (redirect failure)
918/// gets a fresh resolution for the URL that actually failed.
919fn prepare_routes_after_transport_error<R>(
920 routes: &mut Option<HttpRetryRoutes>,
921 failed_url: &Url,
922 resolve: &R,
923) -> std::io::Result<bool>
924where
925 R: Fn(&str, u16) -> std::io::Result<Vec<SocketAddr>>,
926{
927 if routes
928 .as_ref()
929 .is_some_and(|existing| existing.matches_url(failed_url))
930 {
931 return Ok(routes
932 .as_ref()
933 .is_some_and(|existing| !existing.remaining.is_empty()));
934 }
935 reset_http_retry_routes(routes, failed_url, resolve)
936}
937
938/// Route-relevant facts from one failed HTTP attempt.
939///
940/// Keeping this independent of reqwest's concrete response/error objects gives
941/// the retry loop one transition point for both failure classes and lets tests
942/// drive the route state deterministically without scheduling local sockets.
943enum HttpRetryRouteFailure<'a> {
944 Response {
945 status: reqwest::StatusCode,
946 response_url: &'a Url,
947 remote_addr: Option<SocketAddr>,
948 },
949 Transport {
950 failed_url: &'a Url,
951 },
952}
953
954/// Update retry-route state from one failed attempt.
955///
956/// Non-gateway responses (notably 429) deliberately leave routing unchanged.
957/// Gateway responses discard the observed peer after resolving the response
958/// origin, while transport errors initialize or preserve the candidate tail
959/// according to [`prepare_routes_after_transport_error`].
960fn prepare_http_retry_route<R>(
961 routes: &mut Option<HttpRetryRoutes>,
962 failure: HttpRetryRouteFailure<'_>,
963 resolve: &R,
964) -> std::io::Result<bool>
965where
966 R: Fn(&str, u16) -> std::io::Result<Vec<SocketAddr>>,
967{
968 match failure {
969 HttpRetryRouteFailure::Response {
970 status,
971 response_url,
972 remote_addr,
973 } if is_dns_edge_failure_status(status) => {
974 record_failed_http_edge(routes, response_url, remote_addr, resolve)
975 }
976 HttpRetryRouteFailure::Response { .. } => Ok(false),
977 HttpRetryRouteFailure::Transport { failed_url } => {
978 prepare_routes_after_transport_error(routes, failed_url, resolve)
979 }
980 }
981}
982
983/// Build a retry client for the next untried primary address plus every
984/// remaining fallback candidate.
985///
986/// Client-construction failure is non-fatal: skipping that primary address
987/// preserves the original retry contract rather than replacing an HTTP outage
988/// with a TLS/backend setup error.
989fn next_routed_http_client<B>(
990 routes: &mut Option<HttpRetryRoutes>,
991 build_client: &B,
992) -> Option<(Client, Vec<SocketAddr>)>
993where
994 B: Fn(&str, &[SocketAddr]) -> reqwest::Result<Client>,
995{
996 let routes = routes.as_mut()?;
997 while let Some(primary) = routes.remaining.pop_front() {
998 let candidates: Vec<_> = std::iter::once(primary)
999 .chain(routes.remaining.iter().copied())
1000 .collect();
1001 match build_client(&routes.host, &candidates) {
1002 Ok(client) => return Some((client, candidates)),
1003 Err(err) => {
1004 tracing::warn!(
1005 host = %routes.host,
1006 ?candidates,
1007 %err,
1008 "failed to build an HTTP retry client for resolved candidates; \
1009 trying a shorter candidate tail",
1010 );
1011 }
1012 }
1013 }
1014 None
1015}
1016
1017/// GET `url`, retrying transient failures (transport errors and
1018/// [`is_transient_http_status`] statuses) per `retry`.
1019///
1020/// The failure surface is IDENTICAL to a bare
1021/// `client.get(url).send()`: every non-transient response — success,
1022/// 404, or any other status — is returned to the caller untouched, so
1023/// each call site keeps its own status routing (the 404 →
1024/// [`TarballNotFound`] git-fallback marker, the RC/codeload bespoke
1025/// 404 messages) and its exact error wording. A transient status on
1026/// the FINAL attempt is likewise returned (the caller's status gate
1027/// bails with the same `HTTP 503` text an unretried call would have
1028/// produced); a transport error on the final attempt propagates with
1029/// the same `"{what} {url}"` context the call sites previously
1030/// attached. Retries are purely additive.
1031///
1032/// `what` is the call site's error-context verb ("download" /
1033/// "fetch") so the transport-error context matches the wording each
1034/// site used before retries existed.
1035///
1036/// Only the GET is retried — response-body streaming failures are
1037/// out of scope (the tarball sites stream through [`DownloadStream`]
1038/// with its own watchdog, and a mid-stream abort after a 200 leaves
1039/// staging state a plain re-GET could not resume anyway).
1040fn get_with_transient_retry(
1041 client: &Client,
1042 url: &str,
1043 timeout: Option<Duration>,
1044 what: &str,
1045 retry: &HttpRetry,
1046) -> Result<reqwest::blocking::Response> {
1047 get_with_transient_retry_and_headers(
1048 client,
1049 url,
1050 timeout,
1051 what,
1052 retry,
1053 &reqwest::header::HeaderMap::new(),
1054 )
1055}
1056
1057/// [`get_with_transient_retry`] with headers reapplied to every attempt.
1058///
1059/// This is used by the ranged URL-existence probe. Keeping request
1060/// construction inside the retry loop is important: a retry must carry the
1061/// same `Range` contract as the first attempt instead of silently becoming a
1062/// full-body GET.
1063fn get_with_transient_retry_and_headers(
1064 client: &Client,
1065 url: &str,
1066 timeout: Option<Duration>,
1067 what: &str,
1068 retry: &HttpRetry,
1069 headers: &reqwest::header::HeaderMap,
1070) -> Result<reqwest::blocking::Response> {
1071 get_with_transient_retry_and_routes(
1072 client,
1073 url,
1074 timeout,
1075 what,
1076 retry,
1077 headers,
1078 HttpRouteFailover {
1079 enabled: is_shared_client(client),
1080 resolve: resolve_http_origin,
1081 build_client: build_routed_http_client,
1082 },
1083 )
1084}
1085
1086struct HttpRouteFailover<R, B> {
1087 enabled: bool,
1088 resolve: R,
1089 build_client: B,
1090}
1091
1092/// Implementation seam for [`get_with_transient_retry`].
1093///
1094/// Production enables address failover only for [`shared_client`], whose
1095/// builder configuration is known and can be reproduced safely. A
1096/// caller-provided client may carry custom roots, proxies, or redirect policy,
1097/// so its retries remain on that exact client. Tests inject deterministic
1098/// address resolution and a no-proxy routed-client builder here.
1099fn get_with_transient_retry_and_routes<R, B>(
1100 client: &Client,
1101 url: &str,
1102 timeout: Option<Duration>,
1103 what: &str,
1104 retry: &HttpRetry,
1105 headers: &reqwest::header::HeaderMap,
1106 route_failover: HttpRouteFailover<R, B>,
1107) -> Result<reqwest::blocking::Response>
1108where
1109 R: Fn(&str, u16) -> std::io::Result<Vec<SocketAddr>>,
1110 B: Fn(&str, &[SocketAddr]) -> reqwest::Result<Client>,
1111{
1112 assert!(retry.attempts > 0, "HttpRetry.attempts must be >= 1");
1113 let mut routes = None;
1114 let mut routed_client: Option<(Client, Vec<SocketAddr>)> = None;
1115
1116 for attempt in 1..=retry.attempts {
1117 ensure_source_operation_not_interrupted("before HTTP request")?;
1118 let request_client = routed_client
1119 .as_ref()
1120 .map(|(client, _)| client)
1121 .unwrap_or(client);
1122 let mut req = request_client.get(url).headers(headers.clone());
1123 if let Some(t) = timeout {
1124 req = req.timeout(t);
1125 }
1126 let mut advance_route = false;
1127 match req.send() {
1128 Ok(response) => {
1129 ensure_source_operation_not_interrupted("after HTTP response headers")?;
1130 let status = response.status();
1131 if !is_transient_http_status(status) || attempt == retry.attempts {
1132 return Ok(response);
1133 }
1134 let remote_addr = response.remote_addr();
1135 if route_failover.enabled {
1136 match prepare_http_retry_route(
1137 &mut routes,
1138 HttpRetryRouteFailure::Response {
1139 status,
1140 response_url: response.url(),
1141 remote_addr,
1142 },
1143 &route_failover.resolve,
1144 ) {
1145 Ok(has_routes) => advance_route = has_routes,
1146 Err(err) => tracing::warn!(
1147 url = %response.url(),
1148 %err,
1149 "failed to resolve alternate HTTP addresses; retrying normally",
1150 ),
1151 }
1152 }
1153 tracing::warn!(
1154 %url, %status, ?remote_addr, attempt, max_attempts = retry.attempts,
1155 "{what} hit a transient HTTP status; retrying",
1156 );
1157 }
1158 Err(e) => {
1159 if attempt == retry.attempts {
1160 return Err(anyhow::Error::new(e).context(format!("{what} {url}")));
1161 }
1162 if route_failover.enabled {
1163 let failed_url = e.url().cloned().or_else(|| Url::parse(url).ok());
1164 if let Some(failed_url) = failed_url {
1165 match prepare_http_retry_route(
1166 &mut routes,
1167 HttpRetryRouteFailure::Transport {
1168 failed_url: &failed_url,
1169 },
1170 &route_failover.resolve,
1171 ) {
1172 Ok(has_routes) => advance_route = has_routes,
1173 Err(resolve_err) => tracing::warn!(
1174 url = %failed_url,
1175 err = %resolve_err,
1176 "failed to refresh HTTP candidates after a transport error; \
1177 retrying normally",
1178 ),
1179 }
1180 }
1181 }
1182 tracing::warn!(
1183 %url, error_url = ?e.url(), err = %e,
1184 attempt, max_attempts = retry.attempts,
1185 "{what} failed in transport; retrying",
1186 );
1187 }
1188 }
1189 // Reached only on a transient failure with attempts left:
1190 // both match arms return on the final attempt.
1191 source_operation_backoff(retry.backoff_unit * (1u32 << attempt))?;
1192 if advance_route {
1193 routed_client = next_routed_http_client(&mut routes, &route_failover.build_client);
1194 if let Some((_, candidates)) = routed_client.as_ref() {
1195 tracing::warn!(
1196 %url,
1197 ?candidates,
1198 next_attempt = attempt + 1,
1199 "retrying with alternate resolved origin candidates; \
1200 an active proxy may select a different upstream route",
1201 );
1202 }
1203 }
1204 }
1205 unreachable!("the attempt == retry.attempts arms above return on the final iteration")
1206}
1207
1208/// Sleep between HTTP attempts without making cancellation wait for the whole
1209/// exponential-backoff interval. The 100 ms quantum matches the flock wait
1210/// poller and is far below the shortest production backoff (2 s).
1211fn source_operation_backoff(duration: Duration) -> Result<()> {
1212 const QUANTUM: Duration = Duration::from_millis(100);
1213 let deadline = Instant::now() + duration;
1214 loop {
1215 ensure_source_operation_not_interrupted("during HTTP retry backoff")?;
1216 let now = Instant::now();
1217 if now >= deadline {
1218 return Ok(());
1219 }
1220 std::thread::sleep((deadline - now).min(QUANTUM));
1221 }
1222}
1223
1224/// GET `url` through the process-wide [`shared_client`] with the
1225/// standard [`TRANSIENT_HTTP_RETRY`] policy and return the full
1226/// response body as bytes.
1227///
1228/// The distro repo-metadata resolver ([`crate::distro::repo`]) fetches
1229/// repomd.xml, primary.xml.{gz,zst,xz}, `Packages.gz`, and `mirror.list`
1230/// through this so every metadata fetch rides the same retry/backoff
1231/// seam as the kernel-source downloads. `what` is the error-context
1232/// verb ("fetch"). A non-success status is a hard error carrying the
1233/// status; the caller adds its own context. The explicit
1234/// [`SMALL_RESPONSE_REQUEST_TIMEOUT`] bounds both the wait for response
1235/// headers and every blocking body read, including the headers-then-stall
1236/// failure mode.
1237pub(crate) fn fetch_metadata_bytes(url: &str, what: &str) -> Result<Vec<u8>> {
1238 fetch_metadata_bytes_with(
1239 shared_client(),
1240 url,
1241 what,
1242 &TRANSIENT_HTTP_RETRY,
1243 SMALL_RESPONSE_REQUEST_TIMEOUT,
1244 )
1245}
1246
1247/// Injectable core of [`fetch_metadata_bytes`] for timeout/retry tests.
1248fn fetch_metadata_bytes_with(
1249 client: &Client,
1250 url: &str,
1251 what: &str,
1252 retry: &HttpRetry,
1253 timeout: Duration,
1254) -> Result<Vec<u8>> {
1255 let response = get_with_transient_retry(client, url, Some(timeout), what, retry)?;
1256 if !response.status().is_success() {
1257 anyhow::bail!("{what} {url}: HTTP {}", response.status());
1258 }
1259 read_response_bytes(response, url)
1260}
1261
1262/// GET `url` like [`fetch_metadata_bytes`] but decode the body as
1263/// UTF-8 text — for the plain-text metadata endpoints (`mirror.list`,
1264/// `meta-release-lts`).
1265pub(crate) fn fetch_metadata_text(url: &str, what: &str) -> Result<String> {
1266 let bytes = fetch_metadata_bytes(url, what)?;
1267 String::from_utf8(bytes).with_context(|| format!("decode body of {url} as UTF-8"))
1268}
1269
1270/// Ranged GET used by live distro-resolution tests to prove a resolved
1271/// package URL still exists upstream without downloading the package.
1272///
1273/// The range header is rebuilt on every transient retry, and the small-response
1274/// timeout bounds both header and body waits. Returning the final status keeps
1275/// 404 and other permanent failures visible to the caller's assertion.
1276#[cfg(test)]
1277pub(crate) fn probe_url_status(url: &str) -> Result<reqwest::StatusCode> {
1278 probe_url_status_with(
1279 shared_client(),
1280 url,
1281 &TRANSIENT_HTTP_RETRY,
1282 SMALL_RESPONSE_REQUEST_TIMEOUT,
1283 )
1284}
1285
1286#[cfg(test)]
1287fn probe_url_status_with(
1288 client: &Client,
1289 url: &str,
1290 retry: &HttpRetry,
1291 timeout: Duration,
1292) -> Result<reqwest::StatusCode> {
1293 let mut headers = reqwest::header::HeaderMap::new();
1294 headers.insert(
1295 reqwest::header::RANGE,
1296 reqwest::header::HeaderValue::from_static("bytes=0-0"),
1297 );
1298 let response =
1299 get_with_transient_retry_and_headers(client, url, Some(timeout), "probe", retry, &headers)?;
1300 Ok(response.status())
1301}
1302
1303/// Construct the cdn.kernel.org `sha256sums.asc` URL for a stable
1304/// major series:
1305/// `https://cdn.kernel.org/pub/linux/kernel/v{major}.x/sha256sums.asc`.
1306/// Single source of truth for the manifest URL shape, used by
1307/// [`resolve_expected_sha256`] (production) and shared with the
1308/// URL-injection test seam so the two never drift.
1309fn sha256sums_url(major: u32) -> String {
1310 format!("https://cdn.kernel.org/pub/linux/kernel/v{major}.x/sha256sums.asc")
1311}
1312
1313/// GET the cleartext SHA-256 manifest at `url` and return its body.
1314///
1315/// Returns the file body as a `String` on success. Any error
1316/// (transport failure, non-2xx status, non-UTF-8 body) is
1317/// propagated; the caller treats failure as "no expected hash
1318/// available" and downgrades verification to a warning.
1319///
1320/// Takes the full `url` rather than a `major` so the GET-and-status
1321/// mechanics are reachable with an injected URL (a localhost mock)
1322/// without a real cdn.kernel.org round-trip — mirrors the
1323/// [`fetch_releases`] / [`cached_releases_with_url`] seam. Production
1324/// reaches this only via [`resolve_expected_sha256_from_url`], whose
1325/// URL is pinned by [`sha256sums_url`].
1326fn fetch_sha256sums_from_url(client: &Client, url: &str) -> Result<String> {
1327 tracing::info!(%url, "fetching kernel tarball sha256sums (requires network)");
1328 let response = get_with_transient_retry(
1329 client,
1330 url,
1331 Some(SMALL_RESPONSE_REQUEST_TIMEOUT),
1332 "fetch",
1333 &TRANSIENT_HTTP_RETRY,
1334 )?;
1335 if !response.status().is_success() {
1336 anyhow::bail!("fetch {url}: HTTP {}", response.status());
1337 }
1338 let bytes = read_response_bytes(response, url)?;
1339 String::from_utf8(bytes).with_context(|| format!("decode body of {url} as UTF-8"))
1340}
1341
1342/// Read a short HTTP response through the same cancellation-aware
1343/// streaming adapter as large archives. Metadata endpoints do not need
1344/// a visible byte bar, but they must observe the shared signal epoch
1345/// before and after every blocking body read.
1346fn read_response_bytes(response: reqwest::blocking::Response, url: &str) -> Result<Vec<u8>> {
1347 let mut stream = DownloadStream::with_progress(response, None);
1348 let mut bytes = Vec::new();
1349 stream
1350 .read_to_end(&mut bytes)
1351 .with_context(|| format!("read body of {url}"))?;
1352 ensure_source_operation_not_interrupted("after reading HTTP response body")?;
1353 Ok(bytes)
1354}
1355
1356/// Extract the SHA-256 hex digest for `target_filename` from the
1357/// cleartext-signed `sha256sums.asc` body.
1358///
1359/// kernel.org publishes `sha256sums.asc` as a PGP-cleartext-signed
1360/// document: a `-----BEGIN PGP SIGNED MESSAGE-----` header, an
1361/// optional `Hash:` line, a blank line, the cleartext body
1362/// (`<64-hex-chars> <filename>` per line), then a
1363/// `-----BEGIN PGP SIGNATURE-----` block. We only need the
1364/// cleartext body — signature verification is a separate concern
1365/// (the user-facing instruction is "If no expected hash available,
1366/// log warning", not "require signature").
1367///
1368/// Returns `Some(lowercase_hex)` on first match. Returns `None` if
1369/// the target filename does not appear in the manifest (e.g. the
1370/// upstream rotated or removed the entry).
1371fn parse_sha256_for_file(manifest: &str, target_filename: &str) -> Option<String> {
1372 // Strip the PGP signature trailer if present. Everything after
1373 // the signature marker is binary noise that never contains
1374 // checksum lines.
1375 let body = manifest
1376 .split_once("-----BEGIN PGP SIGNATURE-----")
1377 .map(|(before, _)| before)
1378 .unwrap_or(manifest);
1379 for line in body.lines() {
1380 let line = line.trim();
1381 // sha256sum format: `<64-hex-chars><whitespace><filename>`.
1382 // Split on whitespace; require exactly two tokens and a
1383 // 64-char hex first token.
1384 let mut parts = line.split_whitespace();
1385 let Some(hash) = parts.next() else { continue };
1386 let Some(name) = parts.next() else { continue };
1387 if name != target_filename {
1388 continue;
1389 }
1390 if hash.len() != 64 || !hash.chars().all(|c| c.is_ascii_hexdigit()) {
1391 continue;
1392 }
1393 return Some(hash.to_ascii_lowercase());
1394 }
1395 None
1396}
1397
1398/// Verify `actual_hex` against `expected_hex` (case-insensitive).
1399/// Returns `Ok(())` on match, `Err` with a diagnostic message on
1400/// mismatch. Pulled out of the call site so the comparison logic
1401/// has one home and the diagnostic carries both digests in lowercase
1402/// hex for direct copy-paste reuse.
1403fn verify_sha256(actual_hex: &str, expected_hex: &str, url: &str) -> Result<()> {
1404 if actual_hex.eq_ignore_ascii_case(expected_hex) {
1405 Ok(())
1406 } else {
1407 anyhow::bail!(
1408 "sha256 mismatch for {url}: expected {}, got {}. \
1409 If cdn.kernel.org updated this tarball in-place, \
1410 retry with --skip-sha256 to bypass verification.",
1411 expected_hex.to_ascii_lowercase(),
1412 actual_hex.to_ascii_lowercase(),
1413 );
1414 }
1415}
1416
1417/// Resolve the expected SHA-256 digest for a stable tarball from
1418/// cdn.kernel.org's `sha256sums.asc` manifest.
1419///
1420/// Three outcomes:
1421/// - `Some(hex)` — manifest fetched and the entry for `tarball_name`
1422/// was parsed cleanly.
1423/// - `None` with no warning (only when `skip_sha256 = true`) —
1424/// operator explicitly opted out of verification; emits a single
1425/// security-sensitive bypass warning instead.
1426/// - `None` with a per-cause warning (manifest fetch failed, or
1427/// manifest fetched but entry missing) — best-effort fallback so
1428/// a transient cdn.kernel.org outage / schema drift does not
1429/// gate the whole download.
1430///
1431/// The fallback path is deliberately permissive: we trade strict
1432/// authentication for build availability. A network-path attacker
1433/// who can deny `sha256sums.asc` while serving a poisoned
1434/// `linux-{version}.tar.xz` could exploit this; operators who
1435/// require strict verification should pin the source via a
1436/// `--kernel <path>` or `--kernel git+…` source rather than the
1437/// download path. The bypass warnings
1438/// surface on the operator's diagnostic stream so the lost
1439/// guarantee is visible to ops triage.
1440///
1441/// Extracted from [`download_stable_tarball`] so the gate is
1442/// directly unit-testable without mocking network calls — the
1443/// caller-supplied `client` reaches a `Client::get` only when
1444/// `skip_sha256 == false`, so a `skip_sha256 = true` test does not
1445/// need a configured `Client`.
1446fn resolve_expected_sha256(
1447 client: &Client,
1448 major: u32,
1449 tarball_name: &str,
1450 skip_sha256: bool,
1451) -> Option<String> {
1452 resolve_expected_sha256_from_url(client, &sha256sums_url(major), tarball_name, skip_sha256)
1453}
1454
1455/// URL-explicit HTTP wrapper for [`resolve_expected_sha256`]. Production
1456/// reaches this with [`sha256sums_url`]; the routing core below accepts an
1457/// injected fetch operation for deterministic no-network tests.
1458fn resolve_expected_sha256_from_url(
1459 client: &Client,
1460 sha256sums_url: &str,
1461 tarball_name: &str,
1462 skip_sha256: bool,
1463) -> Option<String> {
1464 resolve_expected_sha256_with(tarball_name, skip_sha256, || {
1465 fetch_sha256sums_from_url(client, sha256sums_url)
1466 })
1467}
1468
1469/// Checksum-resolution core with an injectable manifest fetch.
1470///
1471/// Production supplies the real retrying HTTP operation through
1472/// [`resolve_expected_sha256_from_url`]. The seam keeps parsing and
1473/// warn-and-continue routing directly testable without making a synthetic
1474/// localhost server compete with thousands of CI test processes.
1475fn resolve_expected_sha256_with<F>(
1476 tarball_name: &str,
1477 skip_sha256: bool,
1478 fetch_manifest: F,
1479) -> Option<String>
1480where
1481 F: FnOnce() -> Result<String>,
1482{
1483 if skip_sha256 {
1484 tracing::warn!(
1485 tarball = %tarball_name,
1486 "--skip-sha256: bypassing checksum verification — the \
1487 downloaded tarball will not be authenticated against \
1488 cdn.kernel.org's sha256sums.asc manifest. Use only when \
1489 upstream has updated a tarball in-place and the manifest \
1490 is mismatched.",
1491 );
1492 return None;
1493 }
1494 // Best-effort expected-hash lookup: any failure (network,
1495 // status, parse, missing entry) downgrades to a warning so the
1496 // download still proceeds. The warning surfaces the cause so an
1497 // operator triaging "kernel build went weird" can spot that
1498 // verification was skipped.
1499 match fetch_manifest() {
1500 Ok(manifest) => match parse_sha256_for_file(&manifest, tarball_name) {
1501 Some(hex) => Some(hex),
1502 None => {
1503 tracing::warn!(
1504 tarball = %tarball_name,
1505 "sha256sums.asc fetched but no entry for {tarball_name}; \
1506 download will proceed without checksum verification. \
1507 Pass --skip-sha256 to bypass the manifest fetch when \
1508 the entry is known to be absent.",
1509 );
1510 None
1511 }
1512 },
1513 Err(err) => {
1514 tracing::warn!(
1515 error = %format!("{err:#}"),
1516 "failed to fetch sha256sums.asc; download will proceed \
1517 without checksum verification. Pass --skip-sha256 to \
1518 bypass the manifest fetch when the manifest is known \
1519 to be unavailable.",
1520 );
1521 None
1522 }
1523 }
1524}
1525
1526/// GitHub mirror of the linux-stable tree — comprehensive (stable +
1527/// base-release `vX.Y.Z` tags back to v2.6) and the authoritative
1528/// source for tags whose `.tar.xz` is no longer on cdn.kernel.org.
1529///
1530/// cdn.kernel.org keeps only the LATEST tarball of each series
1531/// currently in `releases.json`; every superseded point release AND
1532/// every tag of an EOL series is pruned (a GET for the tarball 404s,
1533/// verified empirically — and HEAD is not a dependable existence probe
1534/// on the CDN). The gregkh mirror still carries every `vX.Y.Z` tag, and
1535/// codeload serves each tag's snapshot as a `tar.gz`, so a codeload
1536/// download recovers the source a pruned tarball would have provided —
1537/// no clone. Its `ls-refs` advertises every release tag, which
1538/// `--include-eol` enumerates to surface EOL series absent from
1539/// `releases.json` (see [`cached_stable_tags`]) and which
1540/// [`fetch_version_for_prefix`] resolves for an EOL/unreleased series.
1541/// github.com advertises allow-sha + a ref-prefix filter and a codeload
1542/// CDN; git.kernel.org offers neither. Used by [`download_tarball`]'s
1543/// [`TarballNotFound`] fallback and the prefix resolver.
1544const STABLE_MIRROR_URL: &str = "https://github.com/gregkh/linux";
1545
1546/// Marker error attached to a stable-tarball download failure when
1547/// cdn.kernel.org returns HTTP 404.
1548///
1549/// A 404 means the tarball is pruned — an EOL series (absent from
1550/// `releases.json`) or a superseded point release (the CDN retains
1551/// only each maintained series' latest). [`download_tarball`] detects
1552/// this via `downcast_ref` (the context-aware anyhow accessor — a
1553/// `chain().any(..is::<T>())` walk would MISS a context-wrapped
1554/// marker) and falls back to a codeload snapshot of the tag from the
1555/// gregkh mirror ([`STABLE_MIRROR_URL`]). Any other HTTP status is a
1556/// hard error with no fallback.
1557#[derive(Debug)]
1558struct TarballNotFound;
1559
1560impl std::fmt::Display for TarballNotFound {
1561 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1562 f.write_str("stable tarball pruned from cdn.kernel.org (EOL or superseded point release)")
1563 }
1564}
1565
1566impl std::error::Error for TarballNotFound {}
1567
1568/// Download a stable kernel tarball (.tar.xz) from cdn.kernel.org.
1569///
1570/// Returns a [`TarballNotFound`] error (downcast-detectable) when the
1571/// CDN 404s the tarball — see that type for the pruning semantics and
1572/// [`download_tarball`] for the git-tag fallback it triggers.
1573///
1574/// Streams the body through a [`DownloadStream`] watchdog so a
1575/// stalled connection (no body bytes for
1576/// [`DOWNLOAD_NO_PROGRESS_TIMEOUT`]) surfaces as an error rather
1577/// than blocking indefinitely. Computes SHA-256 over the streamed
1578/// bytes and verifies against the digest in
1579/// `sha256sums.asc` for the matching `linux-{version}.tar.xz`
1580/// entry; if the manifest fetch / parse fails (transient outage,
1581/// schema drift, missing entry), logs a warning and continues
1582/// without verification rather than failing the whole download.
1583///
1584/// `skip_sha256 = true` bypasses the manifest fetch entirely and
1585/// emits a single bypass warning. Intended for the case where
1586/// cdn.kernel.org has updated a tarball in-place (a new point
1587/// release reusing the same URL) and the manifest is stale or
1588/// mismatched. Unverified downloads are a security-sensitive
1589/// fallback — the bypass warning surfaces the lost guarantee on
1590/// the operator's diagnostic stream.
1591fn download_stable_tarball(
1592 client: &Client,
1593 version: &str,
1594 dest_dir: &Path,
1595 cli_label: &str,
1596 skip_sha256: bool,
1597 mp: Option<&crate::cli::FetchProgress>,
1598) -> Result<PathBuf> {
1599 let major = major_version(version)?;
1600 let url = format!("https://cdn.kernel.org/pub/linux/kernel/v{major}.x/linux-{version}.tar.xz");
1601 download_stable_tarball_from_url(client, &url, version, dest_dir, cli_label, skip_sha256, mp)
1602}
1603
1604/// URL-injectable core of [`download_stable_tarball`]: the GET, the
1605/// 404→[`TarballNotFound`] / other-status→hard-error status gate, and
1606/// the stream→verify→extract pipeline, against an arbitrary tarball
1607/// `url`. Production reaches this only via [`download_stable_tarball`],
1608/// which pins the cdn.kernel.org URL; the seam exists so the status
1609/// routing (404 marker vs hard error) is unit-testable against a
1610/// localhost mock without a real cdn round-trip — mirrors
1611/// [`resolve_expected_sha256_from_url`] / [`fetch_releases`].
1612fn download_stable_tarball_from_url(
1613 client: &Client,
1614 url: &str,
1615 version: &str,
1616 dest_dir: &Path,
1617 cli_label: &str,
1618 skip_sha256: bool,
1619 mp: Option<&crate::cli::FetchProgress>,
1620) -> Result<PathBuf> {
1621 let major = major_version(version)?;
1622 let tarball_name = format!("linux-{version}.tar.xz");
1623
1624 let expected_sha256 = resolve_expected_sha256(client, major, &tarball_name, skip_sha256);
1625
1626 tracing::info!(%url, "downloading stable kernel tarball (requires network)");
1627 let response = get_with_transient_retry(
1628 client,
1629 url,
1630 Some(DOWNLOAD_REQUEST_READ_TIMEOUT),
1631 "download",
1632 &TRANSIENT_HTTP_RETRY,
1633 )?;
1634 if !response.status().is_success() {
1635 if response.status() == reqwest::StatusCode::NOT_FOUND {
1636 // Pruned tarball (EOL series or superseded point release).
1637 // Return the downcast-detectable marker so `download_tarball`
1638 // falls back to a codeload snapshot of the tag from the
1639 // gregkh mirror (`STABLE_MIRROR_URL`) rather than failing
1640 // outright.
1641 return Err(anyhow::Error::new(TarballNotFound));
1642 }
1643 anyhow::bail!("download {url}: HTTP {}", response.status());
1644 }
1645 reject_html_response(&response, url)?;
1646 print_download_size(&response, url, cli_label, mp);
1647 // Capture the total before `response` is moved into the stream so a
1648 // determinate (percent + ETA) bar can be built; `None` when the
1649 // server sent no Content-Length, in which case the bar degrades to
1650 // a live byte counter.
1651 let total = response.content_length();
1652
1653 // Route status lines through the progress group (see
1654 // `print_download_size`); `eprintln!` when no group is threaded in.
1655 let status = |line: &str| match mp {
1656 Some(fp) => fp.println(line),
1657 None => crate::cli::print_status_line(line),
1658 };
1659 status(&format!("{cli_label}: extracting tarball (xz)"));
1660 // Stage extraction inside `dest_dir` (same filesystem) so the
1661 // final `RENAME_NOREPLACE` into place is atomic and a verification
1662 // failure leaves `dest_dir` untouched. A bad mirror that serves
1663 // a wrong-version archive — or sneaks stray top-level entries
1664 // alongside `linux-{version}/` — gets caught after extraction
1665 // but before anything lands in `dest_dir`. The TempDir's Drop
1666 // sweeps every entry the malicious archive deposited.
1667 let staging =
1668 tempfile::TempDir::new_in(dest_dir).with_context(|| "create extraction staging dir")?;
1669 let download_bar = mp.map(|fp| fp.download_bar(version, total));
1670 let stream = DownloadStream::with_progress(response, download_bar.as_ref().map(|b| b.bar()));
1671 let decoder = xz2::read::XzDecoder::new(stream);
1672 let mut archive = tar::Archive::new(decoder);
1673 ensure_source_operation_not_interrupted("before extracting stable kernel tarball")?;
1674 archive
1675 .unpack(staging.path())
1676 .with_context(|| "extract tarball")?;
1677 ensure_source_operation_not_interrupted("after extracting stable kernel tarball")?;
1678
1679 // Recover the watchdog wrapper from inside the decoder/archive
1680 // chain to read the streaming digest. `into_inner` on tar +
1681 // xz2 each peel one layer of the chain. Done after a successful
1682 // unpack so we don't compute over a partial stream.
1683 let stream = archive.into_inner().into_inner();
1684 let (actual_hex, bytes_total) = stream.finalize();
1685 // Download is complete (every byte streamed) — clear the bar
1686 // before emitting the verification status so the two don't overlap.
1687 if let Some(bar) = &download_bar {
1688 bar.finish();
1689 }
1690 if let Some(expected) = expected_sha256.as_deref() {
1691 verify_sha256(&actual_hex, expected, url)?;
1692 status(&format!(
1693 "{cli_label}: sha256 verified ({bytes_total} bytes, hash {actual_hex})"
1694 ));
1695 } else if !skip_sha256 {
1696 // Skip path already emitted its bespoke bypass warning
1697 // before the download; firing again here under "no
1698 // expected sha256 available" would mislead — that wording
1699 // implies a fallback, not an explicit operator opt-out.
1700 tracing::warn!(
1701 url = %url,
1702 bytes = bytes_total,
1703 sha256 = %actual_hex,
1704 "no expected sha256 available for {url}; computed digest \
1705 {actual_hex} over {bytes_total} bytes is unverified",
1706 );
1707 }
1708
1709 ensure_source_operation_not_interrupted("before promoting stable kernel source")?;
1710 let promoted = promote_staged_kernel_tree_transaction(&staging, dest_dir, version)?;
1711 if let Err(err) =
1712 ensure_source_operation_not_interrupted("after promoting stable kernel source")
1713 {
1714 return Err(promoted.rollback(err));
1715 }
1716 Ok(promoted.commit())
1717}
1718
1719/// Verify a kernel tarball's staged extraction contains exactly one
1720/// top-level entry named `linux-{version}/` and atomically rename it
1721/// into `dest_dir/linux-{version}`. Bails — leaving `dest_dir`
1722/// untouched — when the staging dir holds a stray entry, when the
1723/// expected inner directory is missing, or when the rename fails.
1724/// The caller's `TempDir` outlives this helper, so its Drop sweeps
1725/// any residual staging contents whether this returns Ok or Err.
1726#[cfg(test)]
1727fn promote_staged_kernel_tree(
1728 staging: &tempfile::TempDir,
1729 dest_dir: &Path,
1730 version: &str,
1731) -> Result<PathBuf> {
1732 Ok(promote_staged_kernel_tree_transaction(staging, dest_dir, version)?.commit())
1733}
1734
1735fn promote_staged_kernel_tree_transaction(
1736 staging: &tempfile::TempDir,
1737 dest_dir: &Path,
1738 version: &str,
1739) -> Result<PromotedPath> {
1740 let expected_name = format!("linux-{version}");
1741 let mut found_inner = false;
1742 for entry in std::fs::read_dir(staging.path()).with_context(|| "read staging dir entries")? {
1743 let entry = entry.with_context(|| "iterate staging dir entry")?;
1744 let name = entry.file_name();
1745 if name == std::ffi::OsStr::new(&expected_name) {
1746 found_inner = true;
1747 } else {
1748 anyhow::bail!(
1749 "tarball contains unexpected top-level entry {name:?}; \
1750 expected only {expected_name}/"
1751 );
1752 }
1753 }
1754 if !found_inner {
1755 anyhow::bail!("expected directory {expected_name} after extraction");
1756 }
1757 let inner = staging.path().join(&expected_name);
1758 let source_dir = dest_dir.join(&expected_name);
1759 promote_path_noreplace(&inner, &source_dir, PublishedPathKind::Directory)
1760}
1761
1762/// Promote the single top-level directory a codeload archive extracts
1763/// out of `staging` into `dest_dir/{canonical}`, so it survives
1764/// `staging`'s `Drop`.
1765///
1766/// Unlike [`promote_staged_kernel_tree`], the top-dir name is not
1767/// `linux-{version}` — GitHub derives it from the ref (`linux-6.11.11`
1768/// for a tag, `linux-{sha}` for a commit, `linux-{branch}` for a
1769/// branch), so this promotes the SOLE entry by structure rather than by
1770/// a fixed name, renaming it to a caller-supplied `canonical` name that
1771/// keys off the resolved commit (collision-free across refs). A hostile
1772/// or malformed snapshot that deposits zero or several top-level
1773/// entries — or a top-level entry that is not a plain directory (a
1774/// regular file, or a symlink, which the directory-entry file-type
1775/// check rejects rather than following) — is rejected before anything
1776/// lands in `dest_dir`; the `TempDir`'s `Drop` sweeps every entry the
1777/// archive left.
1778#[cfg(test)]
1779fn promote_single_kernel_tree(
1780 staging: &tempfile::TempDir,
1781 dest_dir: &Path,
1782 canonical: &str,
1783) -> Result<PathBuf> {
1784 Ok(promote_single_kernel_tree_transaction(staging, dest_dir, canonical)?.commit())
1785}
1786
1787fn promote_single_kernel_tree_transaction(
1788 staging: &tempfile::TempDir,
1789 dest_dir: &Path,
1790 canonical: &str,
1791) -> Result<PromotedPath> {
1792 let mut entries = Vec::new();
1793 for entry in std::fs::read_dir(staging.path()).with_context(|| "read staging dir entries")? {
1794 entries.push(entry.with_context(|| "iterate staging dir entry")?);
1795 }
1796 if entries.len() != 1 {
1797 anyhow::bail!(
1798 "codeload archive must contain exactly one top-level entry; found {}",
1799 entries.len()
1800 );
1801 }
1802 let inner = entries[0].path();
1803 // Use the DIRECTORY-ENTRY file type (does NOT follow symlinks) so a
1804 // top-level symlink-to-directory is rejected rather than promoted:
1805 // `Path::is_dir()` would follow the link and accept an
1806 // attacker-chosen target, and renameat2 moves the symlink itself
1807 // (it never dereferences), leaving the build reading through it.
1808 let entry_type = entries[0]
1809 .file_type()
1810 .with_context(|| "stat codeload top-level entry")?;
1811 if !entry_type.is_dir() {
1812 anyhow::bail!(
1813 "codeload archive top-level entry is not a plain directory: {}",
1814 inner.display()
1815 );
1816 }
1817 let source_dir = dest_dir.join(canonical);
1818 promote_path_noreplace(&inner, &source_dir, PublishedPathKind::Directory)
1819}
1820
1821#[derive(Clone, Copy)]
1822enum PublishedPathKind {
1823 File,
1824 Directory,
1825}
1826
1827struct PromotedPath {
1828 path: PathBuf,
1829 kind: PublishedPathKind,
1830 committed: bool,
1831}
1832
1833impl PromotedPath {
1834 fn commit(mut self) -> PathBuf {
1835 self.committed = true;
1836 self.path.clone()
1837 }
1838
1839 fn rollback(mut self, primary: anyhow::Error) -> anyhow::Error {
1840 match self.remove() {
1841 Ok(()) => primary,
1842 Err(rollback_err) => primary.context(format!(
1843 "also failed to roll back transaction-owned publication {}: {rollback_err}",
1844 self.path.display()
1845 )),
1846 }
1847 }
1848
1849 fn remove(&mut self) -> std::io::Result<()> {
1850 let result = match self.kind {
1851 PublishedPathKind::File => std::fs::remove_file(&self.path),
1852 PublishedPathKind::Directory => std::fs::remove_dir_all(&self.path),
1853 };
1854 if result.is_ok()
1855 || result
1856 .as_ref()
1857 .is_err_and(|err| err.kind() == std::io::ErrorKind::NotFound)
1858 {
1859 self.committed = true;
1860 Ok(())
1861 } else {
1862 result
1863 }
1864 }
1865}
1866
1867impl Drop for PromotedPath {
1868 fn drop(&mut self) {
1869 if !self.committed {
1870 let _ = self.remove();
1871 }
1872 }
1873}
1874
1875/// Atomically publish a transaction-owned path without replacing an
1876/// existing destination. A signal landing across the rename removes
1877/// only the path this successful `RENAME_NOREPLACE` created; failure to
1878/// roll it back is attached to the cancellation error instead of being
1879/// silently discarded.
1880fn promote_path_noreplace(
1881 staged: &Path,
1882 destination: &Path,
1883 kind: PublishedPathKind,
1884) -> Result<PromotedPath> {
1885 ensure_source_operation_not_interrupted("before publishing acquired source")?;
1886 rustix::fs::renameat_with(
1887 rustix::fs::CWD,
1888 staged,
1889 rustix::fs::CWD,
1890 destination,
1891 rustix::fs::RenameFlags::NOREPLACE,
1892 )
1893 .map_err(|err| {
1894 anyhow!(
1895 "renameat2(RENAME_NOREPLACE) {} -> {}: {err}",
1896 staged.display(),
1897 destination.display()
1898 )
1899 })?;
1900
1901 if let Err(interrupted) =
1902 ensure_source_operation_not_interrupted("after publishing acquired source")
1903 {
1904 let promoted = PromotedPath {
1905 path: destination.to_path_buf(),
1906 kind,
1907 committed: false,
1908 };
1909 return Err(promoted.rollback(interrupted));
1910 }
1911 Ok(PromotedPath {
1912 path: destination.to_path_buf(),
1913 kind,
1914 committed: false,
1915 })
1916}
1917
1918/// Download an RC kernel tarball (.tar.gz) from git.kernel.org.
1919///
1920/// Streams the body through a [`DownloadStream`] watchdog so a
1921/// stalled connection surfaces as an error rather than blocking
1922/// indefinitely. RC tarballs are dynamically generated by gitweb
1923/// at request time and have no published `sha256sums` manifest, so
1924/// this path always logs a warning that the digest is unverified —
1925/// it is computed and surfaced for diagnostic value (operators can
1926/// pin it manually) but never compared to an authoritative source.
1927fn download_rc_tarball(
1928 client: &Client,
1929 version: &str,
1930 dest_dir: &Path,
1931 cli_label: &str,
1932 mp: Option<&crate::cli::FetchProgress>,
1933) -> Result<PathBuf> {
1934 let url = format!("https://git.kernel.org/torvalds/t/linux-{version}.tar.gz");
1935 tracing::info!(%url, "downloading RC kernel tarball (requires network)");
1936
1937 let response = get_with_transient_retry(
1938 client,
1939 &url,
1940 Some(DOWNLOAD_REQUEST_READ_TIMEOUT),
1941 "download",
1942 &TRANSIENT_HTTP_RETRY,
1943 )?;
1944 if response.status() == reqwest::StatusCode::NOT_FOUND {
1945 anyhow::bail!(
1946 "RC tarball not found: {url}\n \
1947 RC releases are removed from git.kernel.org after the stable version ships."
1948 );
1949 }
1950 if !response.status().is_success() {
1951 anyhow::bail!("download {url}: HTTP {}", response.status());
1952 }
1953 reject_html_response(&response, &url)?;
1954 print_download_size(&response, &url, cli_label, mp);
1955 // RC tarballs are gitweb-generated and often arrive without a
1956 // Content-Length, so `total` is frequently `None` and the bar
1957 // degrades to a live byte counter (rate, no ETA).
1958 let total = response.content_length();
1959
1960 let status = |line: &str| match mp {
1961 Some(fp) => fp.println(line),
1962 None => crate::cli::print_status_line(line),
1963 };
1964 status(&format!("{cli_label}: extracting tarball (gzip)"));
1965 // Stage extraction inside `dest_dir` (same filesystem) so the
1966 // final atomic rename keeps `dest_dir` clean when a bad mirror
1967 // serves a wrong-version archive or sneaks stray top-level
1968 // entries past the archive boundary. RC tarballs have no
1969 // upstream sha256 manifest, so structural verification is the
1970 // only defence against a hostile gitweb response.
1971 let staging =
1972 tempfile::TempDir::new_in(dest_dir).with_context(|| "create extraction staging dir")?;
1973 let download_bar = mp.map(|fp| fp.download_bar(version, total));
1974 let stream = DownloadStream::with_progress(response, download_bar.as_ref().map(|b| b.bar()));
1975 let decoder = flate2::read::GzDecoder::new(stream);
1976 let mut archive = tar::Archive::new(decoder);
1977 ensure_source_operation_not_interrupted("before extracting RC kernel tarball")?;
1978 archive
1979 .unpack(staging.path())
1980 .with_context(|| "extract tarball")?;
1981 ensure_source_operation_not_interrupted("after extracting RC kernel tarball")?;
1982
1983 // Surface the streamed digest as a warning. RC tarballs have
1984 // no upstream manifest, so verification is impossible — but
1985 // emitting the hash gives an operator a value they can
1986 // capture for offline pinning if they want to detect drift on
1987 // re-fetch.
1988 let stream = archive.into_inner().into_inner();
1989 let (actual_hex, bytes_total) = stream.finalize();
1990 if let Some(bar) = &download_bar {
1991 bar.finish();
1992 }
1993 tracing::warn!(
1994 url = %url,
1995 bytes = bytes_total,
1996 sha256 = %actual_hex,
1997 "no expected sha256 available for {url} (RC tarballs are \
1998 dynamically generated by git.kernel.org and have no \
1999 published manifest); computed digest {actual_hex} over \
2000 {bytes_total} bytes is unverified",
2001 );
2002
2003 ensure_source_operation_not_interrupted("before promoting RC kernel source")?;
2004 let promoted = promote_staged_kernel_tree_transaction(&staging, dest_dir, version)?;
2005 if let Err(err) = ensure_source_operation_not_interrupted("after promoting RC kernel source") {
2006 return Err(promoted.rollback(err));
2007 }
2008 Ok(promoted.commit())
2009}
2010
2011/// Download a GitHub source snapshot for `git_ref` as a codeload
2012/// `tar.gz` and extract it, returning an [`AcquiredSource`] keyed as an
2013/// explicit immutable-SHA snapshot ([`git_cache_key`] over the resolved
2014/// `commit_hash`). Archive SHA, branch, and tag identities remain
2015/// distinct even when they currently point at the same commit.
2016///
2017/// GitHub serves a gzip snapshot for any tag/branch/commit via
2018/// codeload; the caller supplies the `archive_url`
2019/// ([`github_archive_url`]) and the pre-resolved `commit_hash`
2020/// ([`resolve_ref_commit`]) — the snapshot has no `.git`, so the
2021/// commit cannot be read back from the tree. Modeled on
2022/// [`download_rc_tarball`] (gzip decode; codeload carries no sha256
2023/// manifest, so extraction is structurally verified —
2024/// [`promote_single_kernel_tree_transaction`] rejects any top level that is not a
2025/// single plain directory (multi-entry, a file, or a symlink) — and
2026/// the streamed digest is logged, not compared).
2027pub(crate) fn download_github_archive(
2028 client: &Client,
2029 archive_url: &str,
2030 git_ref: &str,
2031 commit_hash: &str,
2032 dest_dir: &Path,
2033 cli_label: &str,
2034 mp: Option<&crate::cli::FetchProgress>,
2035) -> Result<AcquiredSource> {
2036 let progress_label = format!("{cli_label}: {git_ref}");
2037 let snapshot_progress = match mp {
2038 Some(fp) => fp.operation_progress(&progress_label, "fetching snapshot", "snapshot fetch"),
2039 None => crate::cli::progress::CloneProgress::standalone_operation(
2040 &progress_label,
2041 "fetching snapshot",
2042 "snapshot fetch",
2043 ),
2044 };
2045 // Create the phase before issuing the request. The blocking HTTP
2046 // header wait itself has no byte counter, so this live item is what
2047 // lets CloneProgress emit its first phase immediately and a
2048 // ten-second heartbeat for as long as the server stays silent.
2049 let request_phase = snapshot_progress.item_named("requesting snapshot");
2050 tracing::info!(%archive_url, "downloading GitHub codeload snapshot (requires network)");
2051 let response = get_with_transient_retry(
2052 client,
2053 archive_url,
2054 Some(DOWNLOAD_REQUEST_READ_TIMEOUT),
2055 "download",
2056 &TRANSIENT_HTTP_RETRY,
2057 )?;
2058 drop(request_phase);
2059 if response.status() == reqwest::StatusCode::NOT_FOUND {
2060 anyhow::bail!(
2061 "codeload snapshot not found: {archive_url}\n \
2062 the ref may not exist on the remote, or the repo is private"
2063 );
2064 }
2065 if !response.status().is_success() {
2066 anyhow::bail!("download {archive_url}: HTTP {}", response.status());
2067 }
2068 reject_html_response(&response, archive_url)?;
2069 print_download_size(&response, archive_url, cli_label, mp);
2070 // codeload responses are dynamically generated and often arrive
2071 // without a Content-Length, so `total` is frequently `None` and the
2072 // bar degrades to a live byte counter.
2073 let total = response.content_length();
2074
2075 let status = |line: &str| match mp {
2076 Some(fp) => fp.println(line),
2077 None => crate::cli::print_status_line(line),
2078 };
2079 status(&format!("{cli_label}: extracting snapshot (gzip)"));
2080 // Stage extraction inside `dest_dir` (same filesystem) so the final
2081 // atomic rename keeps `dest_dir` clean when a bad response serves a
2082 // malformed archive or sneaks stray top-level entries. codeload
2083 // snapshots have no upstream sha256 manifest, so structural
2084 // verification (single top-level dir) is the only defence against a
2085 // hostile response.
2086 let staging =
2087 tempfile::TempDir::new_in(dest_dir).with_context(|| "create extraction staging dir")?;
2088 let short_hash: String = commit_hash.chars().take(7).collect();
2089 let download_bar = mp.map(|fp| fp.download_bar(git_ref, total));
2090 // tar::Archive streams decompression directly from the response, so
2091 // download and extraction intentionally overlap instead of writing
2092 // a second full archive to disk. Name that fused phase truthfully;
2093 // the reporter continues to heartbeat even while gzip/tar is doing
2094 // CPU or filesystem work between response reads.
2095 let stream_phase = snapshot_progress.item_named("streaming and extracting snapshot");
2096 let stream = DownloadStream::with_progress(response, download_bar.as_ref().map(|b| b.bar()));
2097 let decoder = flate2::read::GzDecoder::new(stream);
2098 let mut archive = tar::Archive::new(decoder);
2099 ensure_source_operation_not_interrupted("before extracting GitHub snapshot")?;
2100 archive
2101 .unpack(staging.path())
2102 .with_context(|| "extract snapshot")?;
2103 ensure_source_operation_not_interrupted("after extracting GitHub snapshot")?;
2104
2105 // Drain the watchdog to read the streamed digest. codeload has no
2106 // published manifest, so the digest cannot be verified — log it so
2107 // an operator can capture it for offline pinning. `into_inner` peels
2108 // the tar then the gz layer, recovering the `DownloadStream`.
2109 let stream = archive.into_inner().into_inner();
2110 let (actual_hex, bytes_total) = stream.finalize();
2111 drop(stream_phase);
2112 if let Some(bar) = &download_bar {
2113 bar.finish();
2114 }
2115 tracing::info!(
2116 url = %archive_url,
2117 bytes = bytes_total,
2118 sha256 = %actual_hex,
2119 "codeload snapshot extracted (unverified: codeload archives have \
2120 no published sha256 manifest)",
2121 );
2122
2123 // Name the promoted tree by the resolved commit so distinct refs
2124 // never collide in `dest_dir` (the tree is temporary — `is_temp`).
2125 let canonical = format!("linux-git-{short_hash}");
2126 ensure_source_operation_not_interrupted("before promoting GitHub snapshot")?;
2127 let promoted = promote_single_kernel_tree_transaction(&staging, dest_dir, &canonical)?;
2128 if let Err(err) = ensure_source_operation_not_interrupted("after promoting GitHub snapshot") {
2129 return Err(promoted.rollback(err));
2130 }
2131 let source_dir = promoted.path.clone();
2132 ensure_source_operation_not_interrupted("before reading GitHub snapshot metadata")?;
2133 let version = read_makefile_version(&source_dir);
2134 if let Err(err) =
2135 ensure_source_operation_not_interrupted("after reading GitHub snapshot metadata")
2136 {
2137 return Err(promoted.rollback(err));
2138 }
2139
2140 let acquired = AcquiredSource {
2141 source_dir,
2142 cache_key: git_cache_key(crate::kernel_path::GitRefKind::Sha, git_ref, commit_hash),
2143 version,
2144 kernel_source: crate::cache::KernelSource::git(short_hash, git_ref),
2145 is_temp: true,
2146 is_dirty: false,
2147 is_git: true,
2148 };
2149 let _source_dir = promoted.commit();
2150 snapshot_progress.finish();
2151 Ok(acquired)
2152}
2153
2154/// Download a kernel tarball (stable or RC) and extract it.
2155///
2156/// `cli_label` prefixes diagnostic status output (e.g. `"ktstr"` or
2157/// `"cargo ktstr"`).
2158///
2159/// `skip_sha256` propagates to `download_stable_tarball` only —
2160/// stable tarballs publish a `sha256sums.asc` manifest the flag
2161/// bypasses. RC tarballs (`download_rc_tarball`) have no published
2162/// manifest so verification is impossible regardless of the flag;
2163/// the RC path always runs unverified and emits its own warning,
2164/// so `skip_sha256` is a no-op on the RC arm. `--kernel <path>` and
2165/// `--kernel git+…` sources do not reach this function at all.
2166///
2167/// `mp` is the progress group the determinate download bar is added
2168/// to; `None` disables the bar (the single-shot `kernel build` paths
2169/// and unit tests pass `None`).
2170pub fn download_tarball(
2171 client: &Client,
2172 version: &str,
2173 dest_dir: &Path,
2174 cli_label: &str,
2175 skip_sha256: bool,
2176 mp: Option<&crate::cli::FetchProgress>,
2177) -> Result<AcquiredSource> {
2178 let (arch, _) = arch_info();
2179 let source_dir = if is_rc(version) {
2180 download_rc_tarball(client, version, dest_dir, cli_label, mp)?
2181 } else {
2182 match download_stable_tarball(client, version, dest_dir, cli_label, skip_sha256, mp) {
2183 Ok(dir) => dir,
2184 // Pruned tarball (EOL series or superseded point release):
2185 // cdn.kernel.org keeps only each maintained series' latest
2186 // .tar.xz. Recover the source from the stable tree's
2187 // `v{version}` tag via a shallow (depth-1) clone. The kernel
2188 // built from this source is cached by the caller under the
2189 // SAME `{version}-tarball-...` key returned below, so a
2190 // re-run hits that cache and never re-clones.
2191 Err(e) if e.downcast_ref::<TarballNotFound>().is_some() => {
2192 let tag = format!("v{version}");
2193 // A 404 says the tarball is gone, not why. cdn.kernel.org
2194 // keeps only the latest tarball per series, but the gregkh
2195 // GitHub mirror carries every `vX.Y.Z` release tag and
2196 // codeload serves the tag's snapshot as a tar.gz — no
2197 // clone, and a commit-pinned snapshot. Resolve the tag to
2198 // its commit first (kind-directed, so a tag never aliases
2199 // a same-named branch); a tag absent there means the
2200 // version simply does not exist — surface the friendly
2201 // "not found" suggestion (with the latest in-series patch)
2202 // instead of a cryptic fetch failure.
2203 let Some(commit_id) = resolve_ref_commit(
2204 STABLE_MIRROR_URL,
2205 &tag,
2206 crate::kernel_path::GitRefKind::Tag,
2207 cli_label,
2208 mp,
2209 )?
2210 else {
2211 anyhow::bail!("{}", version_not_found_msg(client, version));
2212 };
2213 let commit_hash = format!("{commit_id}");
2214 let archive_url = github_archive_url(STABLE_MIRROR_URL, &commit_hash)
2215 .expect("STABLE_MIRROR_URL is a github.com URL");
2216 let msg = format!(
2217 "{cli_label}: {version} not on cdn.kernel.org (pruned/EOL); \
2218 using the GitHub source-snapshot fallback for gregkh mirror tag {tag}"
2219 );
2220 match mp {
2221 Some(fp) => fp.println(&msg),
2222 None => crate::cli::print_status_line(&msg),
2223 }
2224 download_github_archive(
2225 client,
2226 &archive_url,
2227 &tag,
2228 &commit_hash,
2229 dest_dir,
2230 cli_label,
2231 mp,
2232 )?
2233 .source_dir
2234 }
2235 Err(e) => return Err(e),
2236 }
2237 };
2238
2239 Ok(AcquiredSource {
2240 source_dir,
2241 cache_key: format!("{version}-tarball-{arch}-kc{}", crate::cache_key_suffix()),
2242 version: Some(version.to_string()),
2243 kernel_source: crate::cache::KernelSource::Tarball,
2244 is_temp: true,
2245 is_dirty: false,
2246 is_git: true,
2247 })
2248}
2249
2250/// Download a single file from `url` to `dest`, streaming through the
2251/// `DownloadStream` watchdog + progress bar and verifying the
2252/// resulting SHA-256 against `expected_sha256` (hex, case-insensitive).
2253///
2254/// Used by prebuilt-distro-kernel acquisition
2255/// (`crate::distro::acquire`) to fetch each `.rpm`/`.deb` package and
2256/// its (up to ~1 GiB) debuginfo. Rides the same transient-retry seam
2257/// (`TRANSIENT_HTTP_RETRY`) and no-progress watchdog
2258/// (`DOWNLOAD_NO_PROGRESS_TIMEOUT`, which tracks body progress rather
2259/// than total time so a large but progressing download is never
2260/// aborted) as the kernel-source tarball path. Unlike the tarball
2261/// downloaders it does not extract — the bytes land verbatim at `dest`
2262/// for the caller to unpack. `label` names the package on its progress
2263/// bar; `cli_label` prefixes the size status line.
2264pub fn download_verified_file(
2265 url: &str,
2266 dest: &Path,
2267 expected_sha256: &str,
2268 label: &str,
2269 cli_label: &str,
2270 mp: Option<&crate::cli::FetchProgress>,
2271) -> Result<()> {
2272 download_verified_file_with_checksum(
2273 url,
2274 dest,
2275 DownloadChecksum::Sha256(expected_sha256),
2276 label,
2277 cli_label,
2278 mp,
2279 )
2280}
2281
2282/// Download a single file with the shared retry/watchdog/progress
2283/// pipeline and verify its Google Cloud Storage `md5Hash` (standard
2284/// base64). This is the GKE artifact counterpart to
2285/// [`download_verified_file`]; the transport and progress behavior are
2286/// deliberately identical.
2287pub(crate) fn download_verified_md5_file(
2288 url: &str,
2289 dest: &Path,
2290 expected_md5_base64: &str,
2291 label: &str,
2292 cli_label: &str,
2293 mp: Option<&crate::cli::FetchProgress>,
2294) -> Result<()> {
2295 download_verified_file_with_checksum(
2296 url,
2297 dest,
2298 DownloadChecksum::Md5Base64(expected_md5_base64),
2299 label,
2300 cli_label,
2301 mp,
2302 )
2303}
2304
2305#[derive(Clone, Copy)]
2306enum DownloadChecksum<'a> {
2307 Sha256(&'a str),
2308 Md5Base64(&'a str),
2309}
2310
2311fn download_verified_file_with_checksum(
2312 url: &str,
2313 dest: &Path,
2314 checksum: DownloadChecksum<'_>,
2315 label: &str,
2316 cli_label: &str,
2317 mp: Option<&crate::cli::FetchProgress>,
2318) -> Result<()> {
2319 let client = shared_client();
2320 tracing::info!(%url, "downloading verified kernel artifact (requires network)");
2321 let response = get_with_transient_retry(
2322 client,
2323 url,
2324 Some(DOWNLOAD_REQUEST_READ_TIMEOUT),
2325 "download",
2326 &TRANSIENT_HTTP_RETRY,
2327 )?;
2328 if response.status() == reqwest::StatusCode::NOT_FOUND {
2329 anyhow::bail!("package not found: {url}");
2330 }
2331 if !response.status().is_success() {
2332 anyhow::bail!("download {url}: HTTP {}", response.status());
2333 }
2334 reject_html_response(&response, url)?;
2335 print_download_size(&response, url, cli_label, mp);
2336 let total = response.content_length();
2337 let download_bar = mp.map(|fp| fp.download_bar(label, total));
2338 let parent = dest
2339 .parent()
2340 .ok_or_else(|| anyhow!("download destination has no parent: {}", dest.display()))?;
2341 let staging = tempfile::TempDir::new_in(parent)
2342 .with_context(|| format!("create download staging directory in {}", parent.display()))?;
2343 let staged_file = staging.path().join("artifact");
2344 let staged_result = (|| -> Result<()> {
2345 let mut stream =
2346 DownloadStream::with_progress(response, download_bar.as_ref().map(|b| b.bar()));
2347 let mut file = std::fs::File::create(&staged_file)
2348 .with_context(|| format!("create {}", staged_file.display()))?;
2349 std::io::copy(&mut stream, &mut file)
2350 .with_context(|| format!("stream {url} to {}", staged_file.display()))?;
2351 ensure_source_operation_not_interrupted("after writing downloaded artifact")?;
2352 if let Some(bar) = &download_bar {
2353 bar.finish();
2354 }
2355 match checksum {
2356 DownloadChecksum::Sha256(expected) => {
2357 let (actual, _bytes) = stream.finalize();
2358 verify_sha256(&actual, expected, url)?;
2359 }
2360 DownloadChecksum::Md5Base64(expected) => {
2361 let (actual, _bytes) = stream.finalize_md5_base64();
2362 verify_md5_base64(&actual, expected, url)?;
2363 }
2364 }
2365 drop(file);
2366 Ok(())
2367 })();
2368 if let Err(err) = staged_result {
2369 return Err(close_private_staging(staging, err));
2370 }
2371 let promoted = match promote_path_noreplace(&staged_file, dest, PublishedPathKind::File) {
2372 Ok(promoted) => promoted,
2373 Err(err) => return Err(close_private_staging(staging, err)),
2374 };
2375 drop(staging);
2376 promoted.commit();
2377 Ok(())
2378}
2379
2380fn verify_md5_base64(actual: &str, expected: &str, url: &str) -> Result<()> {
2381 if actual == expected {
2382 Ok(())
2383 } else {
2384 anyhow::bail!(
2385 "md5 mismatch for {url}: expected {expected}, got {actual}. \
2386 The object generation may have changed; retry so ktstr can \
2387 resolve fresh Google Cloud Storage metadata."
2388 );
2389 }
2390}
2391
2392/// Parse the patch level from a kernel version string.
2393/// "6.12.8" → Some(8), "7.0" → Some(0), "abc" → None.
2394fn patch_level(version: &str) -> Option<u32> {
2395 let parts: Vec<&str> = version.split('.').collect();
2396 match parts.len() {
2397 2 => Some(0), // "7.0" has patch level 0
2398 3 => parts[2].parse().ok(),
2399 _ => None,
2400 }
2401}
2402
2403/// Production URL for `releases.json`. Transport tests may pass a local URL
2404/// directly to [`fetch_releases`].
2405pub(crate) const RELEASES_URL: &str = "https://www.kernel.org/releases.json";
2406
2407/// Fetch `releases.json` from `url` and return a vector of
2408/// [`Release`] records. Issues an HTTP GET unconditionally — no
2409/// cache consultation.
2410///
2411/// Production callers reach this function via
2412/// [`cached_releases_with`] (or [`cached_releases`]) which pass
2413/// [`RELEASES_URL`]; the cache helper only invokes
2414/// `fetch_releases` on a cache miss for the singleton path or on
2415/// the bypass branch for non-singleton clients. Tests that need
2416/// to exercise the underlying GET directly — without the cache
2417/// layer — call this function with a locally-constructed `Client`
2418/// and a localhost URL pointed at a TcpListener-backed mock that
2419/// returns canned `releases.json` content.
2420pub(crate) fn fetch_releases(client: &Client, url: &str) -> Result<Vec<Release>> {
2421 tracing::info!(%url, "fetching kernel.org releases index (requires network)");
2422 let response = get_with_transient_retry(
2423 client,
2424 url,
2425 Some(SMALL_RESPONSE_REQUEST_TIMEOUT),
2426 "fetch",
2427 &TRANSIENT_HTTP_RETRY,
2428 )?;
2429 if !response.status().is_success() {
2430 anyhow::bail!("fetch {url}: HTTP {}", response.status());
2431 }
2432 let body = String::from_utf8(read_response_bytes(response, url)?)
2433 .with_context(|| "decode response body as UTF-8")?;
2434 parse_releases_body(&body)
2435}
2436
2437fn parse_releases_body(body: &str) -> Result<Vec<Release>> {
2438 let json: serde_json::Value =
2439 serde_json::from_str(body).with_context(|| "parse releases.json")?;
2440 let releases = json
2441 .get("releases")
2442 .and_then(|r| r.as_array())
2443 .ok_or_else(|| anyhow!("releases.json: missing releases array"))?;
2444 let input_rows = releases.len();
2445 let parsed: Vec<Release> = releases
2446 .iter()
2447 .filter_map(|r| {
2448 let moniker = r.get("moniker")?.as_str()?;
2449 let version = r.get("version")?.as_str()?;
2450 Some(Release {
2451 moniker: moniker.to_string(),
2452 version: version.to_string(),
2453 })
2454 })
2455 .collect();
2456 // Per-row tolerance: a corrupt row is silently dropped via the
2457 // filter_map `?` chain so a single bad entry does not abort the
2458 // whole fetch (see `fetch_releases_row_missing_moniker_drops_row`
2459 // and siblings). The drop is also a hazard: the truncated vector
2460 // gets cached in [`RELEASES_CACHE`] for the rest of the process
2461 // lifetime via the singleton path, so a transient malformed row
2462 // at fetch time persists as a partial snapshot for every later
2463 // cache-hit caller. Surface the drop count so an operator
2464 // tailing logs sees that releases.json arrived partial — without
2465 // this, the symptom (a missing version on resolve) is invisible
2466 // until it propagates as "version not found" elsewhere.
2467 let dropped = input_rows - parsed.len();
2468 if dropped > 0 {
2469 tracing::warn!(
2470 input_rows,
2471 parsed_rows = parsed.len(),
2472 dropped,
2473 "releases.json: dropped {dropped} of {input_rows} row(s) \
2474 missing moniker/version (or non-string values); cached \
2475 snapshot will reflect this for the process lifetime"
2476 );
2477 }
2478 Ok(parsed)
2479}
2480
2481/// Fetch the latest stable kernel version from kernel.org.
2482///
2483/// Selects from the `releases` array (moniker "stable" or "longterm"),
2484/// requiring patch version >= 8 to avoid brand-new major versions
2485/// that may have build issues on CI runners.
2486///
2487/// When `client` is the process-wide [`shared_client`] singleton,
2488/// routes through `RELEASES_CACHE`; other clients bypass the
2489/// cache via pointer-equality and exercise `fetch_releases`
2490/// directly — see `cached_releases_with` for details.
2491///
2492/// `cli_label` prefixes diagnostic status output (e.g. `"ktstr"` or
2493/// `"cargo ktstr"`).
2494pub fn fetch_latest_stable_version(client: &Client, cli_label: &str) -> Result<String> {
2495 crate::ktstr_status!("{cli_label}: fetching latest kernel version");
2496 let releases = cached_releases_with(client)?;
2497
2498 let mut best: Option<&str> = None;
2499 for r in &releases {
2500 if r.moniker != "stable" && r.moniker != "longterm" {
2501 continue;
2502 }
2503 if patch_level(&r.version).unwrap_or(0) < 8 {
2504 continue;
2505 }
2506 // Pick the first matching release — releases.json is ordered
2507 // newest first, so the first stable with patch >= 8 is the best.
2508 best = Some(r.version.as_str());
2509 break;
2510 }
2511
2512 let version =
2513 best.ok_or_else(|| anyhow!("no stable kernel with patch >= 8 found in releases.json"))?;
2514 crate::ktstr_status!("{cli_label}: latest stable kernel: {version}");
2515 Ok(version.to_string())
2516}
2517
2518/// Parse a version string into numeric components for comparison.
2519/// "6.14.2" → Some((6, 14, 2)), "6.14" → Some((6, 14, 0)),
2520/// "7.0" → Some((7, 0, 0)). Returns None for unparseable versions.
2521fn version_tuple(version: &str) -> Option<(u32, u32, u32)> {
2522 let parts: Vec<&str> = version.split('.').collect();
2523 match parts.len() {
2524 2 => {
2525 let major = parts[0].parse().ok()?;
2526 let minor = parts[1].parse().ok()?;
2527 Some((major, minor, 0))
2528 }
2529 3 => {
2530 let major = parts[0].parse().ok()?;
2531 let minor = parts[1].parse().ok()?;
2532 let patch = parts[2].parse().ok()?;
2533 Some((major, minor, patch))
2534 }
2535 _ => None,
2536 }
2537}
2538
2539/// Return true when `s` is a kernel major.minor prefix like
2540/// `"6.14"` (as opposed to a full patch version `"6.14.2"` or an rc
2541/// tag `"6.15-rc3"`). Callers use this to decide whether the input
2542/// needs prefix resolution via [`fetch_version_for_prefix`].
2543///
2544/// Accepts any string with fewer than 2 dots and no `-rc` substring,
2545/// so `"7"` (single-segment) and `""` both return true. This matches
2546/// the historical inline check used by kernel-build dispatchers.
2547pub fn is_major_minor_prefix(s: &str) -> bool {
2548 s.matches('.').count() < 2 && !s.contains("-rc")
2549}
2550
2551/// Resolve the highest version matching a prefix.
2552///
2553/// E.g., "6.12" → "6.12.81", "6" → "6.19.12" (highest 6.x.y).
2554///
2555/// Scans all monikers in releases.json except linux-next. On no active
2556/// match (an EOL or unreleased series, absent from releases.json),
2557/// resolves the highest `vX.Y.z` stable patch from the gregkh mirror's
2558/// git tags; if the series has NO stable point release yet (only a base
2559/// tag), falls back to the bare `{prefix}` mainline base — see
2560/// `latest_patch_from_git_tags`.
2561///
2562/// When `client` is the process-wide [`shared_client`] singleton,
2563/// routes through `RELEASES_CACHE`; other clients bypass the
2564/// cache via pointer-equality and exercise `fetch_releases`
2565/// directly — see `cached_releases_with` for details. Cache
2566/// scope is releases.json only; the EOL-series git-tag fallback in
2567/// `latest_patch_from_git_tags` always hits the network.
2568///
2569/// `cli_label` prefixes diagnostic status output (e.g. `"ktstr"` or
2570/// `"cargo ktstr"`).
2571pub fn fetch_version_for_prefix(client: &Client, prefix: &str, cli_label: &str) -> Result<String> {
2572 crate::ktstr_status!("{cli_label}: fetching latest {prefix}.x kernel version");
2573 let releases = cached_releases_with(client)?;
2574
2575 let mut best: Option<(&str, (u32, u32, u32))> = None;
2576 for r in &releases {
2577 if is_skippable_release_moniker(&r.moniker) {
2578 continue;
2579 }
2580 if !r.version.starts_with(prefix) {
2581 continue;
2582 }
2583 if r.version.len() != prefix.len() && r.version.as_bytes()[prefix.len()] != b'.' {
2584 continue;
2585 }
2586 let Some(tuple) = version_tuple(&r.version) else {
2587 continue;
2588 };
2589 if best.is_none() || tuple > best.unwrap().1 {
2590 best = Some((r.version.as_str(), tuple));
2591 }
2592 }
2593
2594 if let Some((version, _)) = best {
2595 crate::ktstr_status!("{cli_label}: latest {prefix}.x kernel: {version}");
2596 return Ok(version.to_string());
2597 }
2598
2599 crate::ktstr_status!(
2600 "{cli_label}: {prefix}.x not in releases.json (EOL or unreleased series); \
2601 resolving latest patch via the gregkh mirror tags"
2602 );
2603 match latest_patch_from_git_tags(STABLE_MIRROR_URL, prefix, cli_label)? {
2604 Some(version) => Ok(version),
2605 None => {
2606 // No stable point release for this series — fall back to the
2607 // mainline base (the `{prefix}` release itself, e.g. a series
2608 // just cut with no `.1` yet, per the "only if there is no
2609 // X.Y.z stable use X.Y mainline" rule). The base tarball is
2610 // fetched by the normal download path (cdn.kernel.org,
2611 // falling back to the gregkh mirror snapshot); torvalds is
2612 // the mainline authority the gregkh mirror tracks.
2613 crate::ktstr_status!(
2614 "{cli_label}: no {prefix}.x stable point release; using {prefix} mainline base"
2615 );
2616 Ok(prefix.to_string())
2617 }
2618 }
2619}
2620
2621/// Resolve a series' latest stable patch by ls-remote-ing the gregkh
2622/// GitHub mirror's `refs/tags/v{prefix}.{patch}` tags and taking the
2623/// highest patch. Returns `Ok(None)` when the series has NO stable
2624/// point release (no `v{prefix}.N` tag) — the caller then falls back to
2625/// the mainline base.
2626///
2627/// The gregkh mirror is the RELIABLE EOL-resolution source: it carries
2628/// every `vX.Y.Z` release tag (back to v2.6) and its codeload CDN
2629/// serves each tag's tarball, so resolution and the pruned-tarball
2630/// download (see [`download_tarball`]'s fallback) share ONE
2631/// comprehensive mirror. cdn.kernel.org cannot be used here: its
2632/// `v{major}.x/` directory index 404s, and its `sha256sums.asc` is
2633/// served inconsistently per CDN edge (200 from some nodes, 404 from
2634/// others — the 404 nodes break CI runners while the tarball fetch on
2635/// those same nodes still succeeds).
2636fn latest_patch_from_git_tags(url: &str, prefix: &str, cli_label: &str) -> Result<Option<String>> {
2637 crate::ktstr_status!("{cli_label}: resolving {prefix}.x release tags via {url}");
2638 let refs = ls_remote_refs(url, &format!("{cli_label}: {prefix}.x tags"))
2639 .with_context(|| format!("ls-remote {url} for {prefix}.x release tags"))?;
2640 match max_tag_patch(refs.iter().map(ref_full_name), prefix) {
2641 Some(patch) => {
2642 let version = format!("{prefix}.{patch}");
2643 crate::ktstr_status!(
2644 "{cli_label}: latest {prefix}.x kernel (from git tags): {version}"
2645 );
2646 Ok(Some(version))
2647 }
2648 None => Ok(None),
2649 }
2650}
2651
2652/// The advertised full ref name (`refs/...`), as raw bytes, of a
2653/// protocol handshake ref.
2654fn ref_full_name(r: &gix::protocol::handshake::Ref) -> &[u8] {
2655 use gix::protocol::handshake::Ref::{Direct, Peeled, Symbolic, Unborn};
2656 match r {
2657 Peeled { full_ref_name, .. }
2658 | Direct { full_ref_name, .. }
2659 | Symbolic { full_ref_name, .. }
2660 | Unborn { full_ref_name, .. } => full_ref_name.as_ref(),
2661 }
2662}
2663
2664/// Highest `{patch}` among `refs/tags/v{prefix}.{patch}` ref names.
2665///
2666/// gix folds an annotated tag's peeled entry into a single
2667/// `Ref::Peeled` whose `full_ref_name` is the BASE name — no `^{}`
2668/// suffix — and a lightweight tag arrives as a `Ref::Direct` with the
2669/// base name too, so every tag advertises its base
2670/// `refs/tags/v{prefix}.{patch}` name for the needle to match. The
2671/// `^{}` strip below is therefore a defensive no-op on real gix output
2672/// (it only affects a raw wire ref name gix never emits; the base
2673/// entry supplies the patch regardless). Pure (no network) so it is
2674/// unit-testable with synthetic ref names.
2675///
2676/// The trailing `.` in the `refs/tags/v{prefix}.` needle keeps a
2677/// `6.14` prefix from matching a `6.140` series, and the numeric-only
2678/// patch tail rejects `-rc` and other non-release tags.
2679fn max_tag_patch<'a>(ref_names: impl Iterator<Item = &'a [u8]>, prefix: &str) -> Option<u32> {
2680 let needle = format!("refs/tags/v{prefix}.");
2681 let mut best: Option<u32> = None;
2682 for name in ref_names {
2683 let Some(rest) = name.strip_prefix(needle.as_bytes()) else {
2684 continue;
2685 };
2686 let rest = rest.strip_suffix(b"^{}").unwrap_or(rest);
2687 if let Ok(s) = std::str::from_utf8(rest)
2688 && let Ok(patch) = s.parse::<u32>()
2689 {
2690 best = Some(best.map_or(patch, |b| b.max(patch)));
2691 }
2692 }
2693 best
2694}
2695
2696/// ls-remote the gregkh stable mirror ([`STABLE_MIRROR_URL`]) once and
2697/// cache the release version strings (`X.Y.Z`) parsed from its
2698/// `refs/tags/vX.Y.Z` advertisement, for `--include-eol` range
2699/// expansion. Returns EVERY release-tag version verbatim (including
2700/// `-rc*` and old series); the caller
2701/// (`crate::cli::select_series_latest_in_range`) does the
2702/// range / rc / per-series filtering. `None` on ls-remote failure —
2703/// not cached, so the next caller retries. gregkh/linux mirrors
2704/// linux-stable comprehensively (tags back to v2.6), so this surfaces
2705/// EOL series that `releases.json` has dropped.
2706pub(crate) fn cached_stable_tags() -> Option<&'static [String]> {
2707 if let Some(tags) = STABLE_TAGS_CACHE.get() {
2708 return Some(tags.as_slice());
2709 }
2710 let refs = ls_remote_refs(STABLE_MIRROR_URL, "ktstr: stable release tags").ok()?;
2711 let tags: Vec<String> = refs
2712 .iter()
2713 .filter_map(|r| {
2714 // Base tag name only: gix folds an annotated tag's peeled
2715 // entry into one `Ref::Peeled` carrying the base name, and a
2716 // lightweight tag is a `Ref::Direct` with the base name, so
2717 // `^{}` never appears on real gix output — the strip is a
2718 // defensive no-op. Non-`refs/tags/v*` refs are skipped.
2719 let name = ref_full_name(r);
2720 let v = name.strip_prefix(b"refs/tags/v")?;
2721 let v = v.strip_suffix(b"^{}").unwrap_or(v);
2722 std::str::from_utf8(v).ok().map(|s| s.to_string())
2723 })
2724 .collect();
2725 // Loser of a concurrent race discards its clone (both fetched the
2726 // same advertisement, so the cached content is equivalent).
2727 let _ = STABLE_TAGS_CACHE.set(tags);
2728 STABLE_TAGS_CACHE.get().map(|v| v.as_slice())
2729}
2730
2731/// Cache key for a git-acquired kernel: the explicit ref kind, a
2732/// fixed-seed ahash of the length-delimited raw ref bytes, the resolved
2733/// commit's full hash, the target arch, and the kconfig-fragment suffix.
2734///
2735/// The kind is part of the identity because `#tag=next`,
2736/// `#branch=next`, and `#sha=...` are distinct acquisition contracts
2737/// even when they currently resolve to the same commit. Hashing the raw
2738/// bytes avoids both filesystem-hostile ref spellings and collisions
2739/// introduced by sanitizing (`a/b` versus `a_b`) while keeping the
2740/// cache key well below `NAME_MAX`. The zero-seeded ahash matches
2741/// ktstr's other fast content-addressed caches; this is an identity
2742/// accelerator, not a cryptographic integrity boundary.
2743///
2744/// The FULL 40-hex commit hash keys the entry (not a 7-hex prefix): a
2745/// branch/tag tip moves over time, so the `{git_ref}` segment alone
2746/// cannot distinguish successive commits — only the hash does. A 7-hex
2747/// (28-bit) prefix would let a moved tip whose new commit shares the
2748/// first 7 hex with the cached old commit hit the stale entry and serve
2749/// the wrong kernel build under the new ref. The full id removes that
2750/// collision class; the probe and clone both render full lowercase hex
2751/// before any truncation, so keying on it is drift-free.
2752pub(crate) fn git_cache_key(
2753 ref_kind: crate::kernel_path::GitRefKind,
2754 git_ref: &str,
2755 commit_hash: &str,
2756) -> String {
2757 use crate::kernel_path::GitRefKind;
2758
2759 let (arch, _) = arch_info();
2760 let (kind_name, kind_byte) = match ref_kind {
2761 GitRefKind::Tag => ("tag", 0u8),
2762 GitRefKind::Branch => ("branch", 1u8),
2763 GitRefKind::Sha => ("sha", 2u8),
2764 GitRefKind::Unknown => ("unknown", 3u8),
2765 };
2766 let canonical_ref = (ref_kind == GitRefKind::Sha).then(|| git_ref.to_ascii_lowercase());
2767 let ref_bytes = canonical_ref.as_deref().unwrap_or(git_ref).as_bytes();
2768 let mut hasher = ahash::RandomState::with_seeds(0, 0, 0, 0).build_hasher();
2769 hasher.write(&[kind_byte]);
2770 hasher.write(&(ref_bytes.len() as u64).to_le_bytes());
2771 hasher.write(ref_bytes);
2772 let ref_hash = hasher.finish();
2773 format!(
2774 "git-{kind_name}-{ref_hash:016x}-{}-{arch}-kc{}",
2775 commit_hash.to_ascii_lowercase(),
2776 crate::cache_key_suffix()
2777 )
2778}
2779
2780/// If `url` is a GitHub remote, build the codeload archive URL for the
2781/// resolved `commit_hash`: `github.com/OWNER/REPO/archive/<commit>.tar.gz`
2782/// (302 → codeload.github.com, its CDN) serves a gzip source snapshot
2783/// for any commit — verified empirically. `resolve_git_kernel` invokes
2784/// this only for an explicit immutable SHA; branches and tags use the
2785/// exact shallow gix path even on GitHub. `None` for a non-GitHub URL
2786/// (self-hosted / GitLab / …), where commit-SHA acquisition is not
2787/// supported.
2788///
2789/// The caller supplies an immutable `commit_hash` (an explicit SHA, or
2790/// the strict stable-tag fallback's resolved commit), so the download
2791/// fetches the exact commit the cache/source record names.
2792/// `commit_hash` is lowercased to align with `git_cache_key`.
2793///
2794/// Accepts the https/http/ssh/git and scp-style GitHub remotes, each
2795/// with an optional trailing `/` and `.git`; the host is matched
2796/// case-insensitively (DNS hostnames are case-insensitive).
2797pub(crate) fn github_archive_url(url: &str, commit_hash: &str) -> Option<String> {
2798 // Match the github.com scheme+host CASE-INSENSITIVELY (DNS
2799 // hostnames are case-insensitive, so `GitHub.com` is a GitHub URL),
2800 // keeping the OWNER/REPO path verbatim. Accept the https/http/ssh/git
2801 // schemes (with an optional `git@` userinfo) and the scp-style
2802 // git@github.com:OWNER/REPO, each with an optional trailing `.git`.
2803 let mut path = None;
2804 for prefix in [
2805 "https://github.com/",
2806 "http://github.com/",
2807 "ssh://git@github.com/",
2808 "ssh://github.com/",
2809 "git://github.com/",
2810 "git@github.com:",
2811 ] {
2812 if url
2813 .get(..prefix.len())
2814 .is_some_and(|head| head.eq_ignore_ascii_case(prefix))
2815 {
2816 path = Some(&url[prefix.len()..]);
2817 break;
2818 }
2819 }
2820 let path = path?;
2821 // Trim trailing slashes (a common copy-paste artifact) before the
2822 // `.git` strip so `OWNER/REPO/` and `OWNER/REPO.git/` still resolve
2823 // to codeload rather than misrouting to the clone path.
2824 let path = path.trim_end_matches('/');
2825 let path = path.strip_suffix(".git").unwrap_or(path);
2826 // Exactly OWNER/REPO — reject deeper paths (a stray extra segment
2827 // is not a repo root, so fall through to the clone path).
2828 let mut segs = path.split('/');
2829 let owner = segs.next().filter(|s| !s.is_empty())?;
2830 let repo = segs.next().filter(|s| !s.is_empty())?;
2831 if segs.next().is_some() {
2832 return None;
2833 }
2834 // Always the resolved COMMIT (lowercased) — never a ref-name
2835 // snapshot — so the extracted tree matches git_cache_key's commit
2836 // exactly regardless of a concurrent branch-tip move. codeload
2837 // serves any commit case-insensitively.
2838 Some(format!(
2839 "https://github.com/{owner}/{repo}/archive/{}.tar.gz",
2840 commit_hash.to_ascii_lowercase()
2841 ))
2842}
2843
2844/// The object id the advertised ref named exactly `target` points at,
2845/// or `None` if no ref matches. For an annotated tag (`Ref::Peeled`)
2846/// this is the PEELED commit (`object`), never the tag object;
2847/// `Ref::Unborn` carries no commit and never matches. Used by the
2848/// kind-directed [`resolve_ref_commit`] so tag-peeling and
2849/// unborn-skipping stay consistent.
2850fn pick_ref_object(
2851 refs: &[gix::protocol::handshake::Ref],
2852 target: &str,
2853) -> Option<gix::hash::ObjectId> {
2854 refs.iter().find_map(|r| {
2855 let (name, object) = match r {
2856 gix::protocol::handshake::Ref::Peeled {
2857 full_ref_name,
2858 object,
2859 ..
2860 }
2861 | gix::protocol::handshake::Ref::Direct {
2862 full_ref_name,
2863 object,
2864 }
2865 | gix::protocol::handshake::Ref::Symbolic {
2866 full_ref_name,
2867 object,
2868 ..
2869 } => (full_ref_name, object),
2870 gix::protocol::handshake::Ref::Unborn { .. } => return None,
2871 };
2872 (*name == target).then_some(*object)
2873 })
2874}
2875
2876/// Resolve `git_ref` to its full commit hash under `ref_kind`, via a
2877/// kind-directed exact-ref discovery. This standalone helper serves
2878/// stable-tag fallback and metadata callers that genuinely need
2879/// ls-remote semantics. Normal branch/tag acquisition instead reads the
2880/// peeled id from its single receive `Prepare`, avoiding a second
2881/// handshake. An explicit SHA resolves offline.
2882///
2883/// A `Sha` ref IS the commit (lowercased to match `git_clone`'s
2884/// rendering) and resolves offline — no handshake. `Tag`/`Branch`
2885/// match ONLY the fully-qualified `refs/tags/{ref}` / `refs/heads/{ref}`
2886/// so a tag never aliases a same-named branch (a bare-name DWIM lookup
2887/// would resolve either). `Ok(None)` means the exact ref was not
2888/// advertised or the kind is `Unknown`. Transport, authentication,
2889/// protocol, and cancellation failures propagate to the caller.
2890pub(crate) fn resolve_ref_commit(
2891 url: &str,
2892 git_ref: &str,
2893 ref_kind: crate::kernel_path::GitRefKind,
2894 cli_label: &str,
2895 mp: Option<&crate::cli::FetchProgress>,
2896) -> Result<Option<gix::ObjectId>> {
2897 use crate::kernel_path::GitRefKind;
2898 let target = match ref_kind {
2899 GitRefKind::Sha => {
2900 let canonical = git_ref.to_ascii_lowercase();
2901 return gix::ObjectId::from_hex(canonical.as_bytes())
2902 .with_context(|| format!("parse explicit git commit {git_ref}"))
2903 .map(Some);
2904 }
2905 GitRefKind::Tag => format!("refs/tags/{git_ref}"),
2906 GitRefKind::Branch => format!("refs/heads/{git_ref}"),
2907 GitRefKind::Unknown => return Ok(None),
2908 };
2909 match gix_policy::classify_source(url).map_err(anyhow::Error::msg)? {
2910 gix_policy::InProcessSource::Local(path) => {
2911 return resolve_local_ref_commit(&path, &target, &SOURCE_OPERATION_INTERRUPTED);
2912 }
2913 gix_policy::InProcessSource::Http | gix_policy::InProcessSource::Https => {}
2914 }
2915 let progress_label = format!("{cli_label}: {git_ref}");
2916 let ref_progress = match mp {
2917 Some(fp) => fp.operation_progress(&progress_label, "resolving git ref", "ref discovery"),
2918 None => crate::cli::progress::CloneProgress::standalone_operation(
2919 &progress_label,
2920 "resolving git ref",
2921 "ref discovery",
2922 ),
2923 };
2924 let refs = discover_remote_refs(
2925 url,
2926 Some(&target),
2927 &ref_progress,
2928 &SOURCE_OPERATION_INTERRUPTED,
2929 )?;
2930 let object = pick_ref_object(&refs, &target);
2931 ref_progress.finish();
2932 Ok(object)
2933}
2934
2935fn resolve_local_ref_commit(
2936 repository_path: &Path,
2937 target: &str,
2938 interrupt: &std::sync::atomic::AtomicBool,
2939) -> Result<Option<gix::ObjectId>> {
2940 ensure_git_operation_not_interrupted(interrupt, "before opening local repository")?;
2941 let repo = gix::open_opts(repository_path, anon_open_opts())
2942 .with_context(|| format!("open local source repository {}", repository_path.display()))?;
2943 ensure_git_operation_not_interrupted(interrupt, "after opening local repository")?;
2944 let Some(mut reference) = repo
2945 .try_find_reference(target)
2946 .with_context(|| format!("find local source ref {target}"))?
2947 else {
2948 return Ok(None);
2949 };
2950 let commit_id = reference
2951 .peel_to_commit()
2952 .with_context(|| format!("peel local source ref {target} to a commit"))?
2953 .id;
2954 #[cfg(test)]
2955 if TEST_INTERRUPT_AFTER_REF_DISCOVERY.swap(false, Ordering::AcqRel) {
2956 interrupt.store(true, Ordering::Release);
2957 }
2958 ensure_git_operation_not_interrupted(interrupt, "after reading local ref")?;
2959 Ok(Some(commit_id))
2960}
2961
2962/// ls-remote `url` and return EVERY advertised ref WITHOUT fetching a
2963/// pack. Callers decide explicitly whether failure is fatal:
2964/// correctness-sensitive resolution propagates it, while optional
2965/// historical-tag enrichment degrades with `.ok()`.
2966///
2967/// The ad-hoc repo (`init_opts` on a tempdir, with repo-local git config
2968/// only — see `anon_open_opts`) carries no working tree and fetches no
2969/// pack. Remote-side ref-prefix filtering is
2970/// DISABLED: gix's default (`prefix_from_spec_as_filter_on_remote =
2971/// true`) derives protocol-v2 `ls-refs` `ref-prefix` filters from the
2972/// remote's fetch refspecs; an anonymous `remote_at` has none, and
2973/// `fetch_tags = Included` injects only `refs/tags/*`, so the server
2974/// would return TAGS ONLY and `refs/heads/*` would never arrive.
2975/// Disabling the filter returns all refs, so a branch, tag, or HEAD
2976/// all resolve.
2977fn ls_remote_refs(url: &str, progress_label: &str) -> Result<Vec<gix::protocol::handshake::Ref>> {
2978 let ref_progress = crate::cli::progress::CloneProgress::standalone_operation(
2979 progress_label,
2980 "resolving git refs",
2981 "ref discovery",
2982 );
2983 let refs = discover_remote_refs(url, None, &ref_progress, &SOURCE_OPERATION_INTERRUPTED)?;
2984 ref_progress.finish();
2985 Ok(refs)
2986}
2987
2988/// Discover either one exact fully-qualified ref or the remote's full
2989/// advertisement without fetching a pack.
2990///
2991/// `exact_ref = Some` installs one exact fetch refspec and leaves
2992/// protocol-v2 prefix filtering enabled, so a normal branch/tag cache
2993/// probe asks the server only for that namespace-qualified ref. The
2994/// `None` form deliberately disables filtering for stable-tag
2995/// enumeration. Both forms create their progress phase before
2996/// `connect`: the first blocking network wait is therefore visible and
2997/// receives CloneProgress's ten-second heartbeat.
2998fn discover_remote_refs(
2999 url: &str,
3000 exact_ref: Option<&str>,
3001 progress: &crate::cli::progress::CloneProgress,
3002 interrupt: &std::sync::atomic::AtomicBool,
3003) -> Result<Vec<gix::protocol::handshake::Ref>> {
3004 ensure_git_operation_not_interrupted(interrupt, "before preparing ref discovery")?;
3005 match gix_policy::classify_source(url).map_err(anyhow::Error::msg)? {
3006 gix_policy::InProcessSource::Http | gix_policy::InProcessSource::Https => {}
3007 gix_policy::InProcessSource::Local(path) => anyhow::bail!(
3008 "local ref discovery must open {} directly instead of constructing a remote transport",
3009 path.display()
3010 ),
3011 }
3012 let tmp = tempfile::TempDir::new().with_context(|| "create ref discovery staging dir")?;
3013 ensure_git_operation_not_interrupted(interrupt, "after preparing ref discovery")?;
3014 let repo = gix::ThreadSafeRepository::init_opts(
3015 tmp.path(),
3016 gix::create::Kind::WithWorktree,
3017 gix::create::Options::default(),
3018 anon_open_opts(),
3019 )
3020 .with_context(|| "prepare ref discovery repository")?
3021 .to_thread_local();
3022 ensure_git_operation_not_interrupted(interrupt, "after initializing ref discovery")?;
3023 let mut remote = repo
3024 .remote_at_without_url_rewrite(url)
3025 .with_context(|| format!("prepare ref discovery remote {url}"))?;
3026 ensure_git_operation_not_interrupted(interrupt, "after preparing ref discovery remote")?;
3027
3028 if let Some(source_ref) = exact_ref {
3029 // A private throwaway destination is enough to derive the exact
3030 // protocol-v2 ref-prefix; no ref is written because this is only
3031 // a ref-map operation.
3032 const DISCOVERY_REF: &str = "refs/ktstr/discovery";
3033 let refspec = format!("+{source_ref}:{DISCOVERY_REF}");
3034 remote = remote.with_fetch_tags(gix::remote::fetch::Tags::None);
3035 remote
3036 .replace_refspecs([refspec.as_str()], gix::remote::Direction::Fetch)
3037 .with_context(|| format!("set exact discovery refspec for {source_ref}"))?;
3038 ensure_git_operation_not_interrupted(interrupt, "after setting discovery refspec")?;
3039 }
3040
3041 // This phase exists before connect(), whose handshake has no
3042 // progress or interrupt parameter. The shared process interrupt is
3043 // checked on both sides of that non-interruptible boundary.
3044 let discovery_progress = progress.item_named("discovering refs");
3045 ensure_git_operation_not_interrupted(interrupt, "before connecting for ref discovery")?;
3046 let mut connection = remote
3047 .connect(gix::remote::Direction::Fetch)
3048 .with_context(|| format!("connect to {url} for ref discovery"))?;
3049 connection.set_credentials(gix_policy::reject_credentials);
3050 ensure_git_operation_not_interrupted(interrupt, "after connecting for ref discovery")?;
3051 let options = gix::remote::ref_map::Options {
3052 prefix_from_spec_as_filter_on_remote: exact_ref.is_some(),
3053 ..Default::default()
3054 };
3055 ensure_git_operation_not_interrupted(interrupt, "before reading remote refs")?;
3056 let (refmap, _handshake) = connection
3057 .ref_map(discovery_progress, options)
3058 .with_context(|| format!("read remote refs from {url}"))?;
3059 #[cfg(test)]
3060 if TEST_INTERRUPT_AFTER_REF_DISCOVERY.swap(false, Ordering::AcqRel) {
3061 interrupt.store(true, Ordering::Release);
3062 }
3063 ensure_git_operation_not_interrupted(interrupt, "after reading remote refs")?;
3064 Ok(refmap.remote_refs)
3065}
3066
3067/// Shared hermetic options for runtime ref discovery, remote acquisition,
3068/// and directly opened local repositories.
3069fn anon_open_opts() -> gix::open::Options {
3070 gix_policy::open_options()
3071}
3072
3073/// Shallow-clone a git repository at a BRANCH ref.
3074///
3075/// `cli_label` prefixes diagnostic status output (e.g. `"ktstr"` or
3076/// `"cargo ktstr"`).
3077///
3078/// `mp` is the progress group a determinate clone bar is added to.
3079/// Without a group, the clone still drives a hidden CloneProgress tree
3080/// and emits escape-free ten-second heartbeats, so single-shot
3081/// `kernel build` cannot disappear during a slow handshake or pack
3082/// index. The bar shows real object/file counts + ETA during the
3083/// receiving / resolving / checkout phases that gix reports a bounded
3084/// total for; see the `crate::cli::progress` module.
3085///
3086/// For a TAG ref use the test-visible `git_clone_tag` wrapper. Both entry points share the
3087/// same lower-level exact-ref fetch; the wrapper selects the native
3088/// `refs/heads/*` or `refs/tags/*` source namespace.
3089pub fn git_clone(
3090 url: &str,
3091 git_ref: &str,
3092 dest_dir: &Path,
3093 cli_label: &str,
3094 mp: Option<&crate::cli::FetchProgress>,
3095) -> Result<AcquiredSource> {
3096 git_clone_inner(
3097 url,
3098 git_ref,
3099 crate::kernel_path::GitRefKind::Branch,
3100 dest_dir,
3101 cli_label,
3102 mp,
3103 )
3104}
3105
3106/// Shallow-clone a git repository at a TAG ref (e.g. `v6.14.11`).
3107///
3108/// The exact fetch maps `refs/tags/{tag}` to ktstr's private fetched
3109/// ref, peels annotated tags to their commit for checkout, and retains
3110/// the original tag object under its native name for `git describe` /
3111/// `setlocalversion`. The `#tag=` source (via
3112/// [`git_clone_kinded`]) uses this path for every remote, including
3113/// GitHub. The pruned/EOL tarball recovery is a separately reported
3114/// archive fallback in [`download_tarball`].
3115#[cfg(test)]
3116pub(crate) fn git_clone_tag(
3117 url: &str,
3118 tag: &str,
3119 dest_dir: &Path,
3120 cli_label: &str,
3121 mp: Option<&crate::cli::FetchProgress>,
3122) -> Result<AcquiredSource> {
3123 git_clone_inner(
3124 url,
3125 tag,
3126 crate::kernel_path::GitRefKind::Tag,
3127 dest_dir,
3128 cli_label,
3129 mp,
3130 )
3131}
3132
3133/// Clone a git source at `git_ref`, dispatching on `ref_kind` to the
3134/// correct clone path. Branches and tags use this exact shallow gix
3135/// path uniformly, including well-formed GitHub remotes. Only an
3136/// explicit immutable GitHub `Sha` is routed to
3137/// [`download_github_archive`] by [`crate::cli::resolve_git_kernel`].
3138///
3139/// - `Tag` → [`git_clone_tag`] (exact `refs/tags/*` source).
3140/// - `Branch` → [`git_clone`] (exact `refs/heads/*` source).
3141/// - `Sha` → a hard error: gix cannot fetch a bare commit, and a
3142/// self-hosted server generally lacks allow-sha-in-want. The
3143/// actionable message points at GitHub (codeload serves any sha) or a
3144/// tag/branch.
3145/// - `Unknown` → a hard error; [`crate::kernel_path::KernelId::validate`]
3146/// rejects it upstream, so this is a defensive backstop.
3147#[cfg(test)]
3148pub(crate) fn git_clone_kinded(
3149 url: &str,
3150 git_ref: &str,
3151 ref_kind: crate::kernel_path::GitRefKind,
3152 dest_dir: &Path,
3153 cli_label: &str,
3154 mp: Option<&crate::cli::FetchProgress>,
3155) -> Result<AcquiredSource> {
3156 use crate::kernel_path::GitRefKind;
3157 match ref_kind {
3158 GitRefKind::Tag => git_clone_tag(url, git_ref, dest_dir, cli_label, mp),
3159 GitRefKind::Branch => git_clone(url, git_ref, dest_dir, cli_label, mp),
3160 GitRefKind::Sha => anyhow::bail!(
3161 "git+{url}#sha={git_ref}: fetching this source by commit sha is \
3162 not supported — gix cannot fetch a bare commit and the remote \
3163 lacks allow-sha-in-want. Use a github.com/OWNER/REPO URL \
3164 (codeload serves any commit) or pin a #tag= / #branch= instead."
3165 ),
3166 GitRefKind::Unknown => anyhow::bail!(
3167 "git+{url}: ref kind could not be determined; use #tag=NAME, \
3168 #branch=NAME, or #sha=<40-hex>"
3169 ),
3170 }
3171}
3172
3173/// Shared shallow-clone implementation for [`git_clone`] (branch) and
3174/// `git_clone_tag` (tag).
3175///
3176/// This deliberately uses gix's lower-level fetch API rather than
3177/// `clone::PrepareFetch`. The clone helper adds the remote's `HEAD` to
3178/// every fetch and, for its ordinary shallow branch path, changes tag
3179/// handling to `Tags::All`. On a kernel repository that turns a
3180/// depth-one branch checkout into a fetch and pack-index of every
3181/// release tag. An exact fully-qualified refspec keeps negotiation to
3182/// the one branch or tag the caller requested. Branches and tags use
3183/// the same fetch and checkout path; only their source namespace
3184/// differs.
3185fn git_clone_inner(
3186 url: &str,
3187 git_ref: &str,
3188 ref_kind: crate::kernel_path::GitRefKind,
3189 dest_dir: &Path,
3190 cli_label: &str,
3191 mp: Option<&crate::cli::FetchProgress>,
3192) -> Result<AcquiredSource> {
3193 match git_clone_inner_gated(url, git_ref, ref_kind, dest_dir, cli_label, mp, |_| {
3194 Ok(ExactRefGate::Fetch(()))
3195 })? {
3196 GitCloneOutcome::Fetched { acquired, .. } => Ok(acquired),
3197 GitCloneOutcome::Skipped { .. } => unreachable!("unconditional clone cannot skip"),
3198 }
3199}
3200
3201pub(crate) enum GitCloneOutcome<T> {
3202 Fetched { acquired: AcquiredSource, token: T },
3203 Skipped { commit_hash: String, token: T },
3204}
3205
3206/// Exact branch/tag acquisition with a cache/election callback executed
3207/// after the single gix ref-map and before receive. Used by
3208/// `resolve_git_kernel` to avoid a separate discovery handshake.
3209pub(crate) fn git_clone_kinded_gated<T>(
3210 url: &str,
3211 git_ref: &str,
3212 ref_kind: crate::kernel_path::GitRefKind,
3213 dest_dir: &Path,
3214 cli_label: &str,
3215 mp: Option<&crate::cli::FetchProgress>,
3216 gate: impl FnOnce(gix::ObjectId) -> Result<ExactRefGate<T>>,
3217) -> Result<GitCloneOutcome<T>> {
3218 use crate::kernel_path::GitRefKind;
3219 match ref_kind {
3220 GitRefKind::Branch | GitRefKind::Tag => {
3221 git_clone_inner_gated(url, git_ref, ref_kind, dest_dir, cli_label, mp, gate)
3222 }
3223 GitRefKind::Sha => anyhow::bail!(
3224 "git+{url}#sha={git_ref}: fetching this source by commit sha is \
3225 not supported — use a github.com/OWNER/REPO URL or pin a \
3226 #tag= / #branch= instead"
3227 ),
3228 GitRefKind::Unknown => anyhow::bail!(
3229 "git+{url}: ref kind could not be determined; use #tag=NAME, \
3230 #branch=NAME, or #sha=<40-hex>"
3231 ),
3232 }
3233}
3234
3235fn close_private_staging(staging: tempfile::TempDir, primary: anyhow::Error) -> anyhow::Error {
3236 match staging.close() {
3237 Ok(()) => primary,
3238 Err(cleanup_err) => primary.context(format!(
3239 "also failed to remove private source-acquisition staging directory: {cleanup_err}"
3240 )),
3241 }
3242}
3243
3244fn git_clone_inner_gated<T>(
3245 url: &str,
3246 git_ref: &str,
3247 ref_kind: crate::kernel_path::GitRefKind,
3248 dest_dir: &Path,
3249 cli_label: &str,
3250 mp: Option<&crate::cli::FetchProgress>,
3251 gate: impl FnOnce(gix::ObjectId) -> Result<ExactRefGate<T>>,
3252) -> Result<GitCloneOutcome<T>> {
3253 let cloning = format!("{cli_label}: cloning {url} (ref: {git_ref}, depth: 1)");
3254 match mp {
3255 Some(fp) => fp.println(&cloning),
3256 None => crate::cli::print_status_line(&cloning),
3257 }
3258
3259 let clone_dir = dest_dir.join("linux");
3260 let staging = tempfile::TempDir::new_in(dest_dir)
3261 .with_context(|| format!("create private clone staging dir in {}", dest_dir.display()))?;
3262
3263 // Drive a determinate clone bar from gix's progress tree (see
3264 // [`crate::cli::progress::CloneProgress`]). A shared group renders
3265 // coordinated bars; a command without one still gets the same
3266 // plain-text ten-second heartbeat through a standalone reporter.
3267 // Every phase polls the process-level signal bridge set by
3268 // cargo-ktstr's interrupt handler.
3269 let progress_label = format!("{cli_label}: {git_ref}");
3270 let clone_progress = match mp {
3271 Some(fp) => fp.clone_progress(&progress_label),
3272 None => crate::cli::progress::CloneProgress::standalone_operation(
3273 &progress_label,
3274 "cloning",
3275 "clone",
3276 ),
3277 };
3278 let fetched = match fetch_exact_ref_and_checkout_gated(
3279 url,
3280 git_ref,
3281 ref_kind,
3282 staging.path(),
3283 Some(&clone_progress),
3284 &SOURCE_OPERATION_INTERRUPTED,
3285 gate,
3286 ) {
3287 Ok(outcome) => outcome,
3288 Err(err) => {
3289 return Err(close_private_staging(staging, err));
3290 }
3291 };
3292 let (commit_id, token) = match fetched {
3293 ExactRefFetch::Skipped { commit_id, token } => {
3294 let commit_hash = format!("{commit_id}");
3295 staging.close().with_context(
3296 || "remove private clone staging directory after cache/election skip",
3297 )?;
3298 return Ok(GitCloneOutcome::Skipped { commit_hash, token });
3299 }
3300 ExactRefFetch::Fetched { commit_id, token } => (commit_id, token),
3301 };
3302
3303 let acquired = (|| -> Result<AcquiredSource> {
3304 // FULL commit hash keys the cache (see `git_cache_key` — a 7-hex
3305 // prefix risks a moved-tip collision serving a stale build); the
3306 // 7-hex `short_hash` is kept only for the human-facing source
3307 // record.
3308 let commit_hash = format!("{commit_id}");
3309 let short_hash = commit_hash.chars().take(7).collect::<String>();
3310 let cache_key = git_cache_key(ref_kind, git_ref, &commit_hash);
3311
3312 // Record the kernel version from the checked-out source
3313 // Makefile, as local_source does. The filesystem read is the
3314 // final non-interruptible boundary after index publication, so
3315 // observe the shared epoch on both sides before making this
3316 // source visible to the cache/build pipeline.
3317 ensure_source_operation_not_interrupted("before reading cloned source metadata")?;
3318 let version = read_makefile_version(staging.path());
3319 ensure_source_operation_not_interrupted("after reading cloned source metadata")?;
3320
3321 Ok(AcquiredSource {
3322 source_dir: clone_dir.clone(),
3323 cache_key,
3324 version,
3325 kernel_source: crate::cache::KernelSource::git(short_hash, git_ref),
3326 is_temp: true,
3327 is_dirty: false,
3328 is_git: true,
3329 })
3330 })();
3331 match acquired {
3332 Ok(acquired) => {
3333 let promoted = match promote_path_noreplace(
3334 staging.path(),
3335 &clone_dir,
3336 PublishedPathKind::Directory,
3337 ) {
3338 Ok(promoted) => promoted,
3339 Err(err) => return Err(close_private_staging(staging, err)),
3340 };
3341 #[cfg(test)]
3342 if TEST_INTERRUPT_AFTER_CLONE_PROMOTE.swap(false, Ordering::AcqRel) {
3343 SOURCE_OPERATION_INTERRUPTED.store(true, Ordering::Release);
3344 }
3345 if let Err(err) = ensure_source_operation_not_interrupted("after promoting exact clone")
3346 {
3347 drop(staging);
3348 return Err(promoted.rollback(err));
3349 }
3350 drop(staging);
3351 let source_dir = promoted.commit();
3352 debug_assert_eq!(source_dir, acquired.source_dir);
3353 clone_progress.finish();
3354 Ok(GitCloneOutcome::Fetched { acquired, token })
3355 }
3356 Err(err) => Err(close_private_staging(staging, err)),
3357 }
3358}
3359
3360/// Measured ceiling for one gix pack-index or checkout operation.
3361///
3362/// A live sched_ext `for-next` depth-one fetch on the 64-CPU
3363/// development host took 55.0s / 705MB with all CPUs, 57.6s / 454MB
3364/// with eight, and 58.0s / 591MB with sixteen. Eight therefore keeps
3365/// essentially all transfer throughput while avoiding hundreds of MB
3366/// of per-clone worker state. Smaller hosts use their available CPU
3367/// count instead.
3368const GIX_WORKERS_PER_OPERATION: usize = 8;
3369
3370/// Machine-wide, non-blocking lease for checkout's optional extra
3371/// workers. Every checkout always has its calling thread. It then tries
3372/// to flock up to seven numbered tokens in `KTSTR_LOCK_DIR` (or `/tmp`)
3373/// and uses one worker per acquired token. Contending processes simply
3374/// get fewer extras; no source acquisition is ever queued behind this
3375/// performance optimization.
3376struct GixCheckoutWorkerLease {
3377 extra_workers: Vec<std::os::fd::OwnedFd>,
3378}
3379
3380impl GixCheckoutWorkerLease {
3381 fn acquire() -> Self {
3382 let wanted = std::thread::available_parallelism()
3383 .map(usize::from)
3384 .unwrap_or(1)
3385 .min(GIX_WORKERS_PER_OPERATION)
3386 .saturating_sub(1);
3387 let lock_dir = crate::cache::resolve_lock_dir();
3388 Self::acquire_in(&lock_dir, wanted)
3389 }
3390
3391 fn acquire_in(lock_dir: &Path, wanted: usize) -> Self {
3392 let wanted = wanted.min(GIX_WORKERS_PER_OPERATION.saturating_sub(1));
3393 if let Err(err) = std::fs::create_dir_all(lock_dir) {
3394 tracing::warn!(
3395 %err,
3396 path = %lock_dir.display(),
3397 "could not create gix worker-token directory; checkout will use its caller thread",
3398 );
3399 return Self {
3400 extra_workers: Vec::new(),
3401 };
3402 }
3403 let mut extra_workers = Vec::with_capacity(wanted);
3404 // `wanted` is also the size of the machine-wide namespace.
3405 // Scanning all seven ceiling slots on a two-CPU host would let
3406 // seven processes each lease a different extra despite the
3407 // host-wide budget being one.
3408 for slot in 0..wanted {
3409 if extra_workers.len() == wanted {
3410 break;
3411 }
3412 let path = lock_dir.join(format!("ktstr-gix-checkout-worker-{slot}.lock"));
3413 match crate::flock::try_flock(&path, crate::flock::FlockMode::Exclusive) {
3414 Ok(Some(fd)) => extra_workers.push(fd),
3415 Ok(None) => {}
3416 Err(err) => {
3417 tracing::warn!(
3418 %err,
3419 path = %path.display(),
3420 "could not lease gix checkout worker token; continuing with fewer workers",
3421 );
3422 }
3423 }
3424 }
3425 Self { extra_workers }
3426 }
3427
3428 fn workers(&self) -> usize {
3429 1 + self.extra_workers.len()
3430 }
3431}
3432
3433/// Fetch one fully-qualified branch or tag at depth one and materialize
3434/// its peeled commit into `clone_dir`.
3435pub(crate) enum ExactRefGate<T> {
3436 Fetch(T),
3437 Skip(T),
3438}
3439
3440enum ExactRefFetch<T> {
3441 Fetched { commit_id: gix::ObjectId, token: T },
3442 Skipped { commit_id: gix::ObjectId, token: T },
3443}
3444
3445#[cfg(test)]
3446fn fetch_exact_ref_and_checkout(
3447 url: &str,
3448 git_ref: &str,
3449 ref_kind: crate::kernel_path::GitRefKind,
3450 clone_dir: &Path,
3451 clone_progress: Option<&crate::cli::progress::CloneProgress>,
3452 interrupt: &std::sync::atomic::AtomicBool,
3453) -> Result<gix::ObjectId> {
3454 match fetch_exact_ref_and_checkout_gated(
3455 url,
3456 git_ref,
3457 ref_kind,
3458 clone_dir,
3459 clone_progress,
3460 interrupt,
3461 |_| Ok(ExactRefGate::Fetch(())),
3462 )? {
3463 ExactRefFetch::Fetched { commit_id, .. } => Ok(commit_id),
3464 ExactRefFetch::Skipped { .. } => unreachable!("unconditional exact fetch cannot skip"),
3465 }
3466}
3467
3468/// Prepare one exact gix fetch, expose its advertised peeled commit to
3469/// the caller before receiving a pack, and reuse that same `Prepare`
3470/// when the caller wins the content-key builder election.
3471fn fetch_exact_ref_and_checkout_gated<T>(
3472 url: &str,
3473 git_ref: &str,
3474 ref_kind: crate::kernel_path::GitRefKind,
3475 clone_dir: &Path,
3476 clone_progress: Option<&crate::cli::progress::CloneProgress>,
3477 interrupt: &std::sync::atomic::AtomicBool,
3478 gate: impl FnOnce(gix::ObjectId) -> Result<ExactRefGate<T>>,
3479) -> Result<ExactRefFetch<T>> {
3480 use crate::kernel_path::GitRefKind;
3481
3482 let source_ref = match ref_kind {
3483 GitRefKind::Branch => format!("refs/heads/{git_ref}"),
3484 GitRefKind::Tag => format!("refs/tags/{git_ref}"),
3485 GitRefKind::Sha | GitRefKind::Unknown => {
3486 anyhow::bail!("exact shallow fetch requires a branch or tag")
3487 }
3488 };
3489 if let gix_policy::InProcessSource::Local(repository_path) =
3490 gix_policy::classify_source(url).map_err(anyhow::Error::msg)?
3491 {
3492 return materialize_local_exact_ref_gated(
3493 &repository_path,
3494 &source_ref,
3495 clone_dir,
3496 clone_progress,
3497 interrupt,
3498 gate,
3499 );
3500 }
3501 // A fixed private destination means arbitrary valid branch/tag names
3502 // never have to be re-encoded as a local branch name.
3503 const FETCHED_REF: &str = "refs/ktstr/source";
3504 let refspec = format!("+{source_ref}:{FETCHED_REF}");
3505
3506 ensure_git_operation_not_interrupted(interrupt, "before preparing clone")?;
3507 let mut repo = gix::ThreadSafeRepository::init_opts(
3508 clone_dir,
3509 gix::create::Kind::WithWorktree,
3510 gix::create::Options::default(),
3511 anon_open_opts(),
3512 )
3513 .with_context(|| "prepare clone")?
3514 .to_thread_local();
3515 ensure_git_operation_not_interrupted(interrupt, "after preparing clone")?;
3516
3517 // Pack indexing runs on the caller thread. Checkout separately
3518 // leases machine-wide optional workers immediately before it starts;
3519 // no worker token is held across connect or ref mapping.
3520 ensure_git_operation_not_interrupted(interrupt, "before configuring pack workers")?;
3521 repo.config_snapshot_mut()
3522 .set_value(&gix::config::tree::Pack::THREADS, "1")
3523 .with_context(|| "configure single-threaded gix pack indexing")?;
3524 ensure_git_operation_not_interrupted(interrupt, "after configuring pack workers")?;
3525
3526 ensure_git_operation_not_interrupted(interrupt, "before preparing clone remote")?;
3527 let mut remote = repo
3528 .remote_at_without_url_rewrite(url)
3529 .with_context(|| "prepare clone remote")?
3530 .with_fetch_tags(gix::remote::fetch::Tags::None);
3531 ensure_git_operation_not_interrupted(interrupt, "after preparing clone remote")?;
3532 ensure_git_operation_not_interrupted(interrupt, "before setting exact fetch refspec")?;
3533 remote
3534 .replace_refspecs([refspec.as_str()], gix::remote::Direction::Fetch)
3535 .with_context(|| format!("set exact fetch refspec for {source_ref}"))?;
3536 ensure_git_operation_not_interrupted(interrupt, "after setting exact fetch refspec")?;
3537
3538 // prepare_fetch owns this item later, but create it before connect:
3539 // connect performs the first network handshake and accepts no
3540 // progress/interrupt handle of its own.
3541 let discovery_progress =
3542 clone_progress.map(|progress| progress.item_named("discovering exact ref"));
3543 ensure_git_operation_not_interrupted(interrupt, "before connecting")?;
3544 let connection = remote
3545 .connect(gix::remote::Direction::Fetch)
3546 .with_context(|| format!("connect to {url}"))?;
3547 let mut connection = connection;
3548 connection.set_credentials(gix_policy::reject_credentials);
3549 ensure_git_operation_not_interrupted(interrupt, "after connecting")?;
3550 ensure_git_operation_not_interrupted(interrupt, "before mapping exact remote ref")?;
3551 let prepare = match discovery_progress {
3552 Some(progress) => connection
3553 .prepare_fetch(progress, gix::remote::ref_map::Options::default())
3554 .with_context(|| format!("map exact remote ref {source_ref}"))?,
3555 None => connection
3556 .prepare_fetch(
3557 gix::progress::Discard,
3558 gix::remote::ref_map::Options::default(),
3559 )
3560 .with_context(|| format!("map exact remote ref {source_ref}"))?,
3561 };
3562 #[cfg(test)]
3563 TEST_EXACT_PREPARE_COUNT.fetch_add(1, Ordering::AcqRel);
3564 ensure_git_operation_not_interrupted(interrupt, "after mapping exact remote ref")?;
3565 let prepare = prepare.with_shallow(gix::remote::fetch::Shallow::DepthAtRemote(
3566 NonZeroU32::new(1).expect("1 is nonzero"),
3567 ));
3568 ensure_git_operation_not_interrupted(interrupt, "after configuring shallow fetch")?;
3569
3570 // `Prepare` already owns the one connect + ref-map handshake. Read
3571 // the exact mapping it will receive and elect/cache-probe before any
3572 // pack bytes move; the winner continues with this same value.
3573 let advertised_commit = prepare
3574 .ref_map()
3575 .mappings
3576 .iter()
3577 .find_map(|mapping| mapping.remote.peeled_id().map(ToOwned::to_owned))
3578 .with_context(|| format!("remote did not advertise exact ref {source_ref}"))?;
3579 ensure_git_operation_not_interrupted(interrupt, "before exact-ref cache election")?;
3580 let token = match gate(advertised_commit)? {
3581 ExactRefGate::Fetch(token) => token,
3582 ExactRefGate::Skip(token) => {
3583 return Ok(ExactRefFetch::Skipped {
3584 commit_id: advertised_commit,
3585 token,
3586 });
3587 }
3588 };
3589 ensure_git_operation_not_interrupted(interrupt, "after exact-ref cache election")?;
3590
3591 ensure_git_operation_not_interrupted(interrupt, "before receiving the pack")?;
3592 match clone_progress {
3593 Some(progress) => prepare
3594 .receive(
3595 progress.item_named("receiving and indexing exact ref"),
3596 interrupt,
3597 )
3598 .with_context(|| format!("fetch exact remote ref {source_ref}"))?,
3599 None => prepare
3600 .receive(gix::progress::Discard, interrupt)
3601 .with_context(|| format!("fetch exact remote ref {source_ref}"))?,
3602 };
3603 ensure_git_operation_not_interrupted(interrupt, "after receiving the pack")?;
3604 ensure_git_operation_not_interrupted(interrupt, "before reading fetched ref")?;
3605 let mut fetched_ref = repo
3606 .find_reference(FETCHED_REF)
3607 .with_context(|| format!("find fetched ref {source_ref}"))?;
3608 ensure_git_operation_not_interrupted(interrupt, "after reading fetched ref")?;
3609 let unpeeled_id = fetched_ref.id().detach();
3610 // Preserve exactly the requested ref in its native namespace. In
3611 // particular, retaining the selected tag keeps setlocalversion and
3612 // git-describe semantics without importing any unrelated tag.
3613 ensure_git_operation_not_interrupted(interrupt, "before preserving exact source ref")?;
3614 repo.reference(
3615 source_ref.as_str(),
3616 unpeeled_id,
3617 gix::refs::transaction::PreviousValue::Any,
3618 "clone: preserve exact source ref",
3619 )
3620 .with_context(|| format!("preserve fetched ref {source_ref}"))?;
3621 ensure_git_operation_not_interrupted(interrupt, "after preserving exact source ref")?;
3622 ensure_git_operation_not_interrupted(interrupt, "before peeling exact source ref")?;
3623 let commit_id = fetched_ref
3624 .peel_to_commit()
3625 .with_context(|| format!("peel fetched ref {source_ref} to a commit"))?
3626 .id;
3627 ensure_git_operation_not_interrupted(interrupt, "after peeling exact source ref")?;
3628 if commit_id != advertised_commit {
3629 anyhow::bail!(
3630 "exact ref {source_ref} changed during its single fetch handshake \
3631 (advertised {advertised_commit}, received {commit_id})"
3632 );
3633 }
3634
3635 // Both branch and tag clones are snapshots for ktstr, so a detached
3636 // HEAD is the uniform representation of the exact peeled commit.
3637 ensure_git_operation_not_interrupted(interrupt, "before publishing exact HEAD")?;
3638 repo.reference(
3639 "HEAD",
3640 commit_id,
3641 gix::refs::transaction::PreviousValue::Any,
3642 "clone: checkout exact ktstr source",
3643 )
3644 .with_context(|| "set cloned repository HEAD")?;
3645 ensure_git_operation_not_interrupted(interrupt, "after publishing exact HEAD")?;
3646
3647 match clone_progress {
3648 Some(progress) => {
3649 checkout_exact_commit(
3650 &repo,
3651 commit_id,
3652 progress.item_named("checking out exact ref"),
3653 interrupt,
3654 )?;
3655 }
3656 None => {
3657 checkout_exact_commit(&repo, commit_id, gix::progress::Discard, interrupt)?;
3658 }
3659 }
3660 Ok(ExactRefFetch::Fetched { commit_id, token })
3661}
3662
3663fn materialize_local_exact_ref_gated<T>(
3664 repository_path: &Path,
3665 source_ref: &str,
3666 clone_dir: &Path,
3667 clone_progress: Option<&crate::cli::progress::CloneProgress>,
3668 interrupt: &std::sync::atomic::AtomicBool,
3669 gate: impl FnOnce(gix::ObjectId) -> Result<ExactRefGate<T>>,
3670) -> Result<ExactRefFetch<T>> {
3671 const FETCHED_REF: &str = "refs/ktstr/source";
3672
3673 ensure_git_operation_not_interrupted(interrupt, "before opening local exact source")?;
3674 let source = gix::open_opts(repository_path, anon_open_opts())
3675 .with_context(|| format!("open local source repository {}", repository_path.display()))?;
3676 ensure_git_operation_not_interrupted(interrupt, "after opening local exact source")?;
3677 let mut reference = source
3678 .find_reference(source_ref)
3679 .with_context(|| format!("find local source ref {source_ref}"))?;
3680 let unpeeled_id = reference.id().detach();
3681 let commit_id = reference
3682 .peel_to_commit()
3683 .with_context(|| format!("peel local source ref {source_ref} to a commit"))?
3684 .id;
3685 ensure_git_operation_not_interrupted(interrupt, "before exact-ref cache election")?;
3686 let token = match gate(commit_id)? {
3687 ExactRefGate::Fetch(token) => token,
3688 ExactRefGate::Skip(token) => {
3689 return Ok(ExactRefFetch::Skipped { commit_id, token });
3690 }
3691 };
3692 ensure_git_operation_not_interrupted(interrupt, "after exact-ref cache election")?;
3693
3694 let _copy_progress =
3695 clone_progress.map(|progress| progress.item_named("copying local exact ref"));
3696 ensure_git_operation_not_interrupted(interrupt, "before preparing local clone")?;
3697 let repo = gix::ThreadSafeRepository::init_opts(
3698 clone_dir,
3699 gix::create::Kind::WithWorktree,
3700 gix::create::Options::default(),
3701 anon_open_opts(),
3702 )
3703 .with_context(|| "prepare local clone")?
3704 .to_thread_local();
3705 ensure_git_operation_not_interrupted(interrupt, "before copying local exact objects")?;
3706 copy_local_exact_ref_objects(&source, &repo, unpeeled_id, commit_id, interrupt)?;
3707 ensure_git_operation_not_interrupted(interrupt, "after copying local exact objects")?;
3708 drop(_copy_progress);
3709
3710 repo.reference(
3711 FETCHED_REF,
3712 unpeeled_id,
3713 gix::refs::transaction::PreviousValue::Any,
3714 "clone: stage exact local source ref",
3715 )
3716 .with_context(|| "stage exact local source ref")?;
3717 repo.reference(
3718 source_ref,
3719 unpeeled_id,
3720 gix::refs::transaction::PreviousValue::Any,
3721 "clone: preserve exact local source ref",
3722 )
3723 .with_context(|| format!("preserve local source ref {source_ref}"))?;
3724 repo.reference(
3725 "HEAD",
3726 commit_id,
3727 gix::refs::transaction::PreviousValue::Any,
3728 "clone: checkout exact local source",
3729 )
3730 .with_context(|| "set local clone HEAD")?;
3731 std::fs::write(repo.git_dir().join("shallow"), format!("{commit_id}\n"))
3732 .with_context(|| "write local clone shallow boundary")?;
3733 ensure_git_operation_not_interrupted(interrupt, "after publishing local exact refs")?;
3734
3735 match clone_progress {
3736 Some(progress) => checkout_exact_commit(
3737 &repo,
3738 commit_id,
3739 progress.item_named("checking out local exact ref"),
3740 interrupt,
3741 )?,
3742 None => checkout_exact_commit(&repo, commit_id, gix::progress::Discard, interrupt)?,
3743 }
3744 Ok(ExactRefFetch::Fetched { commit_id, token })
3745}
3746
3747fn copy_local_exact_ref_objects(
3748 source: &gix::Repository,
3749 destination: &gix::Repository,
3750 unpeeled_id: gix::ObjectId,
3751 commit_id: gix::ObjectId,
3752 interrupt: &std::sync::atomic::AtomicBool,
3753) -> Result<()> {
3754 let mut copied = std::collections::HashSet::new();
3755 let mut current = unpeeled_id;
3756 loop {
3757 ensure_git_operation_not_interrupted(interrupt, "while copying local exact ref")?;
3758 if !copied.insert(current) {
3759 anyhow::bail!("local exact ref contains a tag cycle at {current}");
3760 }
3761 let object = source
3762 .find_object(current)
3763 .with_context(|| format!("read local exact-ref object {current}"))?;
3764 write_local_object(destination, &object)?;
3765 match object.kind {
3766 gix::objs::Kind::Tag => {
3767 current = object
3768 .try_into_tag()
3769 .with_context(|| format!("decode local tag object {current}"))?
3770 .target_id()
3771 .with_context(|| format!("read target of local tag object {current}"))?
3772 .detach();
3773 }
3774 gix::objs::Kind::Commit => {
3775 if current != commit_id {
3776 anyhow::bail!(
3777 "local exact ref peeled inconsistently: expected {commit_id}, found {current}"
3778 );
3779 }
3780 let tree_id = object
3781 .try_into_commit()
3782 .with_context(|| format!("decode local commit {commit_id}"))?
3783 .tree_id()
3784 .with_context(|| format!("read tree of local commit {commit_id}"))?
3785 .detach();
3786 copy_local_tree_objects(source, destination, tree_id, &mut copied, interrupt)?;
3787 return Ok(());
3788 }
3789 kind => anyhow::bail!("local exact ref resolves to {kind}, expected a tag or commit"),
3790 }
3791 }
3792}
3793
3794fn copy_local_tree_objects(
3795 source: &gix::Repository,
3796 destination: &gix::Repository,
3797 root_tree: gix::ObjectId,
3798 copied: &mut std::collections::HashSet<gix::ObjectId>,
3799 interrupt: &std::sync::atomic::AtomicBool,
3800) -> Result<()> {
3801 let mut pending = vec![root_tree];
3802 while let Some(object_id) = pending.pop() {
3803 #[cfg(test)]
3804 if TEST_INTERRUPT_DURING_LOCAL_COPY.swap(false, Ordering::AcqRel) {
3805 interrupt.store(true, Ordering::Release);
3806 }
3807 ensure_git_operation_not_interrupted(interrupt, "while copying local checkout tree")?;
3808 if !copied.insert(object_id) {
3809 continue;
3810 }
3811 let object = source
3812 .find_object(object_id)
3813 .with_context(|| format!("read local checkout object {object_id}"))?;
3814 write_local_object(destination, &object)?;
3815 match object.kind {
3816 gix::objs::Kind::Tree => {
3817 let tree = object
3818 .try_into_tree()
3819 .with_context(|| format!("decode local tree {object_id}"))?;
3820 for entry in tree.iter() {
3821 let entry =
3822 entry.with_context(|| format!("decode entry in local tree {object_id}"))?;
3823 if entry.kind() != gix::objs::tree::EntryKind::Commit {
3824 pending.push(entry.object_id());
3825 }
3826 }
3827 }
3828 gix::objs::Kind::Blob => {}
3829 kind => anyhow::bail!(
3830 "local tree {root_tree} references unexpected {kind} object {object_id}"
3831 ),
3832 }
3833 }
3834 Ok(())
3835}
3836
3837fn write_local_object(destination: &gix::Repository, object: &gix::Object<'_>) -> Result<()> {
3838 let written = gix::objs::Write::write_buf(destination, object.kind, &object.data)
3839 .map_err(anyhow::Error::from_boxed)
3840 .with_context(|| format!("write local source object {}", object.id))?;
3841 if written != object.id {
3842 anyhow::bail!(
3843 "local source object {} changed identity while materializing (wrote {written})",
3844 object.id
3845 );
3846 }
3847 Ok(())
3848}
3849
3850fn ensure_git_operation_not_interrupted(
3851 interrupt: &std::sync::atomic::AtomicBool,
3852 phase: &str,
3853) -> Result<()> {
3854 if interrupt.load(Ordering::Acquire) {
3855 anyhow::bail!("git operation interrupted {phase}");
3856 }
3857 Ok(())
3858}
3859
3860#[cfg(test)]
3861static TEST_INTERRUPT_AFTER_CHECKOUT: std::sync::atomic::AtomicBool =
3862 std::sync::atomic::AtomicBool::new(false);
3863
3864#[cfg(test)]
3865static TEST_INTERRUPT_AFTER_INDEX_WRITE: std::sync::atomic::AtomicBool =
3866 std::sync::atomic::AtomicBool::new(false);
3867
3868#[cfg(test)]
3869static TEST_INTERRUPT_AFTER_REF_DISCOVERY: std::sync::atomic::AtomicBool =
3870 std::sync::atomic::AtomicBool::new(false);
3871
3872#[cfg(test)]
3873static TEST_INTERRUPT_DURING_LOCAL_COPY: std::sync::atomic::AtomicBool =
3874 std::sync::atomic::AtomicBool::new(false);
3875
3876#[cfg(test)]
3877static TEST_EXACT_PREPARE_COUNT: std::sync::atomic::AtomicUsize =
3878 std::sync::atomic::AtomicUsize::new(0);
3879
3880#[cfg(test)]
3881static TEST_INTERRUPT_AFTER_CLONE_PROMOTE: std::sync::atomic::AtomicBool =
3882 std::sync::atomic::AtomicBool::new(false);
3883
3884/// Materialize `commit_id` into the main worktree and persist its index.
3885fn checkout_exact_commit(
3886 repo: &gix::Repository,
3887 commit_id: gix::ObjectId,
3888 mut progress: impl gix::Progress,
3889 interrupt: &std::sync::atomic::AtomicBool,
3890) -> Result<()> {
3891 ensure_git_operation_not_interrupted(interrupt, "before locating exact clone worktree")?;
3892 let workdir = repo
3893 .workdir()
3894 .with_context(|| "exact clone repository has no worktree")?;
3895 ensure_git_operation_not_interrupted(interrupt, "after locating exact clone worktree")?;
3896 ensure_git_operation_not_interrupted(interrupt, "before resolving fetched commit tree")?;
3897 let tree_id = repo
3898 .find_object(commit_id)
3899 .with_context(|| format!("find fetched commit {commit_id}"))?
3900 .peel_to_tree()
3901 .with_context(|| format!("peel fetched commit {commit_id} to its tree"))?
3902 .id;
3903 ensure_git_operation_not_interrupted(interrupt, "after resolving fetched commit tree")?;
3904 ensure_git_operation_not_interrupted(interrupt, "before building exact clone index")?;
3905 let mut index = repo
3906 .index_from_tree(&tree_id)
3907 .with_context(|| format!("build index from fetched tree {tree_id}"))?;
3908 ensure_git_operation_not_interrupted(interrupt, "after building exact clone index")?;
3909 ensure_git_operation_not_interrupted(interrupt, "before building checkout options")?;
3910 let mut opts = repo
3911 .checkout_options(gix::worktree::stack::state::attributes::Source::IdMapping)
3912 .with_context(|| "build exact clone checkout options")?;
3913 ensure_git_operation_not_interrupted(interrupt, "after building checkout options")?;
3914 opts.destination_is_initially_empty = true;
3915 opts.keep_going = false;
3916 let checkout_workers = GixCheckoutWorkerLease::acquire();
3917 opts.thread_limit = Some(checkout_workers.workers());
3918
3919 progress.init(Some(index.entries().len()), gix::progress::count("files"));
3920 ensure_git_operation_not_interrupted(interrupt, "before building checkout object handle")?;
3921 let objects = repo
3922 .objects
3923 .clone()
3924 .into_arc()
3925 .with_context(|| "build thread-safe object handle for exact clone checkout")?;
3926 ensure_git_operation_not_interrupted(interrupt, "after building checkout object handle")?;
3927 ensure_git_operation_not_interrupted(interrupt, "before checkout")?;
3928 gix::worktree::state::checkout(
3929 &mut index,
3930 workdir,
3931 objects,
3932 &progress,
3933 &gix::progress::Discard,
3934 interrupt,
3935 opts,
3936 )
3937 .with_context(|| format!("check out fetched commit {commit_id}"))?;
3938 #[cfg(test)]
3939 if TEST_INTERRUPT_AFTER_CHECKOUT.swap(false, Ordering::AcqRel) {
3940 interrupt.store(true, Ordering::Release);
3941 }
3942 // gix checkout returns Ok with a partial worktree when its
3943 // interrupt flag becomes set. Never persist that partial index or
3944 // report success; the caller removes the entire clone directory.
3945 ensure_git_operation_not_interrupted(interrupt, "during checkout")?;
3946 drop(checkout_workers);
3947
3948 ensure_git_operation_not_interrupted(interrupt, "before writing exact clone index")?;
3949 index
3950 .write(Default::default())
3951 .with_context(|| "write exact clone index")?;
3952 #[cfg(test)]
3953 if TEST_INTERRUPT_AFTER_INDEX_WRITE.swap(false, Ordering::AcqRel) {
3954 interrupt.store(true, Ordering::Release);
3955 }
3956 // Publication is not complete until the persisted index has been
3957 // followed by one final observation of the shared signal epoch.
3958 // Otherwise a signal landing inside index.write() could return an
3959 // apparently successful clone whose caller proceeds to cache it.
3960 ensure_git_operation_not_interrupted(interrupt, "after writing exact clone index")?;
3961 Ok(())
3962}
3963
3964/// Use a local kernel source tree.
3965///
3966/// Dirty detection uses gix `tree_index_status` (HEAD-vs-index) and
3967/// `status().into_index_worktree_iter()` (index-vs-worktree) to check
3968/// for modifications to tracked files. Submodule checks are skipped
3969/// entirely. Untracked files do not affect the dirty flag.
3970///
3971/// When the tree is dirty, the HEAD commit does not describe the
3972/// source actually being built, so `git_hash` is dropped — no
3973/// commit identifies a dirty worktree. `is_dirty=true` carries that
3974/// fact forward; callers (see [`crate::cli`]) use it to bypass the
3975/// kernel cache entirely.
3976///
3977/// No diagnostic output: all operator-visible messaging for a
3978/// local source is routed through `kernel_build_pipeline`'s
3979/// cache-skip hint (`DIRTY_TREE_CACHE_SKIP_HINT` /
3980/// `NON_GIT_TREE_CACHE_SKIP_HINT`), which has the full context
3981/// to emit a single informational line rather than two redundant
3982/// warnings. Sibling entries (`download_tarball`, `git_clone`)
3983/// still take a `cli_label` because they genuinely print
3984/// progress lines — `local_source` does not.
3985pub fn local_source(source_path: &Path) -> Result<AcquiredSource> {
3986 let (arch, _) = arch_info();
3987
3988 if !source_path.is_dir() {
3989 anyhow::bail!("{}: not a directory", source_path.display());
3990 }
3991
3992 let canonical = source_path
3993 .canonicalize()
3994 .with_context(|| format!("canonicalize {}", source_path.display()))?;
3995
3996 let LocalSourceState {
3997 short_hash,
3998 is_dirty,
3999 is_git,
4000 } = inspect_local_source_state(&canonical)?;
4001
4002 // User .config is folded into the cache key so two builds of the
4003 // same HEAD with different `.config` files do NOT collide on the
4004 // same key — see [`config_hash_for_key`] for the encoding.
4005 // Read at `local_source` time (rather than at the post-build
4006 // store site) so cache LOOKUP and cache STORE see the same key.
4007 let user_config_hash = config_hash_for_key(&canonical);
4008
4009 let cache_key =
4010 compose_local_cache_key(arch, &short_hash, &canonical, user_config_hash.as_deref());
4011
4012 // Record the kernel version from the source-tree Makefile so the
4013 // tmpfs-fraction gate (TmpfsFraction::for_kernel_version, via the
4014 // cache metadata.json sidecar) recognizes a locally-built honoring
4015 // kernel — symmetric with the tarball path. None when the Makefile
4016 // is unreadable/unparsable, which keeps the conservative 50% default.
4017 let version = read_makefile_version(&canonical);
4018
4019 Ok(AcquiredSource {
4020 source_dir: canonical.clone(),
4021 cache_key,
4022 version,
4023 kernel_source: crate::cache::KernelSource::Local {
4024 source_tree_path: Some(canonical),
4025 git_hash: short_hash,
4026 },
4027 is_temp: false,
4028 is_dirty,
4029 is_git,
4030 })
4031}
4032
4033/// Parse the kernel `MAJOR.MINOR.PATCH` version from a source tree's
4034/// top-level `Makefile` (`VERSION` / `PATCHLEVEL` / `SUBLEVEL`) — the
4035/// authoritative version of a locally-built kernel, mirroring the
4036/// version a tarball acquisition records. Returns `None` if the
4037/// `Makefile` is unreadable or any of the three fields is absent or
4038/// non-numeric, so the caller records no version and the rootfs tmpfs
4039/// fraction conservatively defaults to 50% (the honoring gate
4040/// `TmpfsFraction::for_kernel_version` keys on a positively-known
4041/// version). `EXTRAVERSION` (e.g. `-rc7`) is intentionally ignored: the
4042/// gate keys on `MAJOR.MINOR.PATCH` only.
4043fn read_makefile_version(source_dir: &Path) -> Option<String> {
4044 let text = std::fs::read_to_string(source_dir.join("Makefile")).ok()?;
4045 // Each field is a top-of-file `NAME = N` assignment; take the first
4046 // matching line and require a bare integer (a trailing comment or
4047 // non-numeric value yields None for that field, hence overall None).
4048 let field = |name: &str| -> Option<u16> {
4049 text.lines().find_map(|line| {
4050 line.trim()
4051 .strip_prefix(name)?
4052 .trim_start()
4053 .strip_prefix('=')?
4054 .trim()
4055 .parse::<u16>()
4056 .ok()
4057 })
4058 };
4059 Some(format!(
4060 "{}.{}.{}",
4061 field("VERSION")?,
4062 field("PATCHLEVEL")?,
4063 field("SUBLEVEL")?
4064 ))
4065}
4066
4067/// Result of [`inspect_local_source_state`] — git hash and dirty/git
4068/// classification of a canonical source-tree path. Pulled out of
4069/// [`local_source`] so the post-build dirty re-check (a second call
4070/// from [`crate::cli::kernel_build_pipeline`]) reuses the exact same
4071/// gix path.
4072#[derive(Debug, Clone)]
4073pub struct LocalSourceState {
4074 /// HEAD short hash (7 chars). `None` when the tree is dirty
4075 /// (HEAD doesn't describe the actual source) or non-git (no
4076 /// HEAD at all). Mirrors the `git_hash` field on
4077 /// [`AcquiredSource::kernel_source`] for [`crate::cache::KernelSource::Local`].
4078 pub short_hash: Option<String>,
4079 /// Tracked-file dirt: HEAD-vs-index disagreement OR
4080 /// index-vs-worktree disagreement. Always `true` for non-git
4081 /// trees (dirty detection is impossible without git, so the
4082 /// pessimistic stance is dirty).
4083 pub is_dirty: bool,
4084 /// `true` when `gix::discover` succeeded (the tree is a git
4085 /// repo); `false` otherwise. Lets the cache-skip hint branch
4086 /// on whether `commit` / `stash` is actionable.
4087 pub is_git: bool,
4088}
4089
4090/// Inspect a canonical source-tree path for git hash + dirty state.
4091///
4092/// Submodule checks are skipped (false positives on kernel trees
4093/// with uninitialized submodules). The non-git arm returns
4094/// `(None, true, false)` so the caller's cache-skip hint can
4095/// distinguish "dirty git repo" from "not a git repo at all".
4096///
4097/// Called twice per build by [`crate::cli::kernel_build_pipeline`]:
4098/// once at acquire time (via [`local_source`]) and again after
4099/// `make` returns to detect mid-build worktree edits, branch flips,
4100/// or commits that would otherwise let a racing-write build land in
4101/// the cache under a stale identity. Both calls share the same gix
4102/// path so the post-build comparison is apples-to-apples.
4103///
4104/// Non-atomic against concurrent git operations: the probe runs
4105/// six sequential gix calls (`discover` → `head_id` → `head_tree`
4106/// → `index_or_empty` → `tree_index_status` → `status`), each a
4107/// separate filesystem read with no transactional bracket. A
4108/// concurrent `git commit`, `git add`, or worktree write between
4109/// any two calls can produce internally-inconsistent results —
4110/// e.g. `head_id` reads commit C0, a peer commit lands C1, then
4111/// `head_tree` reads C1's root tree and the diff against the
4112/// post-add index reports unexpected dirt. Git itself serializes
4113/// its own writes via per-resource lockfiles under `.git/`
4114/// (`index.lock` for staging operations, `HEAD.lock` and
4115/// `refs/heads/<branch>.lock` for ref updates), so peer `git`
4116/// processes wait on whichever lockfile their operation touches;
4117/// the genuinely-unsynchronized class is worktree-only writes
4118/// (autoformatter, IDE-on-save) which the index-worktree status
4119/// step catches regardless of timing.
4120///
4121/// The disposition is intentionally pessimistic so inconsistency is
4122/// safe: any `Err` propagates to the caller, which treats it as a
4123/// rebuild signal (`MidWaitState::ProbeFailed` in the mid-wait
4124/// caller); any spurious dirty signal falls into DirtyEdit /
4125/// HashAdvanced, both forcing a rebuild. The cost of a false-
4126/// positive rebuild is one extra `make`; the cost of a false-
4127/// negative would be a cache slot keyed on a HEAD that no longer
4128/// describes the source — the asymmetry is the reason for the
4129/// pessimistic disposition. Callers should treat the returned
4130/// state as a best-effort approximation of probe-time, not an
4131/// instantaneous snapshot.
4132pub fn inspect_local_source_state(canonical: &Path) -> Result<LocalSourceState> {
4133 let (short_hash, is_dirty, is_git) = match gix::discover(canonical) {
4134 Ok(repo) => {
4135 let head = repo.head_id().with_context(|| "read HEAD")?;
4136 let short_hash = format!("{}", head).chars().take(7).collect::<String>();
4137
4138 // tree_index_status compares a TREE id against the index;
4139 // the HEAD commit id is not itself a tree, so peel HEAD
4140 // to its root tree before diffing or the diff silently
4141 // returns an error and index dirt goes undetected.
4142 let head_tree = repo.head_tree().with_context(|| "read HEAD tree")?;
4143 let head_tree_id = head_tree.id;
4144
4145 // Check HEAD-vs-index for tracked file changes.
4146 let mut index_dirty = false;
4147 let index = repo.index_or_empty().with_context(|| "open index")?;
4148 let _ = repo.tree_index_status(
4149 &head_tree_id,
4150 &index,
4151 None,
4152 gix::status::tree_index::TrackRenames::Disabled,
4153 |_, _, _| {
4154 index_dirty = true;
4155 Ok::<_, std::convert::Infallible>(std::ops::ControlFlow::Break(()))
4156 },
4157 );
4158
4159 // Check index-vs-worktree for modified tracked files,
4160 // skipping submodules entirely (Ignore::All).
4161 let worktree_dirty = if !index_dirty {
4162 repo.status(gix::progress::Discard)
4163 .with_context(|| "status")?
4164 .index_worktree_rewrites(None)
4165 .index_worktree_submodules(gix::status::Submodule::Given {
4166 ignore: gix::submodule::config::Ignore::All,
4167 check_dirty: false,
4168 })
4169 .index_worktree_options_mut(|opts| {
4170 opts.dirwalk_options = None;
4171 crate::git_status::configure_index_worktree_parallelism(opts);
4172 })
4173 .into_index_worktree_iter(Vec::new())
4174 .map(crate::git_status::consume_has_any)
4175 .unwrap_or(false)
4176 } else {
4177 false
4178 };
4179
4180 let is_dirty = index_dirty || worktree_dirty;
4181 // Drop the HEAD hash when dirty — the commit does not
4182 // describe the actual source being built, so publishing
4183 // it via git_hash / cache_key would misidentify the
4184 // build input.
4185 let hash = if is_dirty { None } else { Some(short_hash) };
4186 (hash, is_dirty, true)
4187 }
4188 Err(_) => {
4189 // The downstream kernel_build_pipeline (cli::kernel_build_pipeline)
4190 // emits `NON_GIT_TREE_CACHE_SKIP_HINT` — a single
4191 // informational line that names both the cause and the
4192 // remediation paths — once the is_dirty=true branch
4193 // decides to skip the cache. Emitting a second
4194 // "not a git repository" warning here duplicated that
4195 // content for every non-git `--kernel <path>` run. The
4196 // `(None, true, false)` tuple silently communicates
4197 // the non-git state to the cache-skip decision site;
4198 // no separate stderr line is needed on this path.
4199 (None, true, false)
4200 }
4201 };
4202 Ok(LocalSourceState {
4203 short_hash,
4204 is_dirty,
4205 is_git,
4206 })
4207}
4208
4209/// Compose the cache key for a local source given its arch, optional
4210/// HEAD short hash, canonical source path, and optional user
4211/// `.config` hash.
4212///
4213/// Three shapes:
4214/// - `local-{hash7}-{arch}-kc{suffix}` — clean git tree, no user
4215/// `.config` (plain `make defconfig` path or no config file yet)
4216/// - `local-{hash7}-{arch}-cfg{user_config}-kc{suffix}` — clean git
4217/// tree with a user `.config` whose hash differs from `defconfig`
4218/// - `local-unknown-{path_hash}-{arch}-kc{suffix}` — dirty / non-git
4219/// tree (HEAD does not describe the source; the path-derived
4220/// crc32 salt keeps two distinct dirty trees from colliding on the
4221/// same `local-unknown-...` slot)
4222///
4223/// `path_hash` is the full 8-char (32-bit) lowercase-hex CRC32 of
4224/// the canonical source-path bytes. CRC32 keeps the per-path
4225/// disambiguator stable across runs without pulling in a
4226/// crypto-grade hash for what is fundamentally a slot disambiguator.
4227///
4228/// `user_config_hash` is `None` whenever the source tree has no
4229/// `.config` file yet (the build will run `make defconfig` and
4230/// produce one). This collapses the user-config branch back into the
4231/// hash-only key so a fresh checkout's first build still hits a
4232/// later cache lookup keyed without the cfg segment.
4233pub fn compose_local_cache_key(
4234 arch: &str,
4235 short_hash: &Option<String>,
4236 canonical: &Path,
4237 user_config_hash: Option<&str>,
4238) -> String {
4239 let suffix = crate::cache_key_suffix();
4240 match short_hash {
4241 Some(hash) => match user_config_hash {
4242 Some(cfg) => format!("local-{hash}-{arch}-cfg{cfg}-kc{suffix}"),
4243 None => format!("local-{hash}-{arch}-kc{suffix}"),
4244 },
4245 None => {
4246 let path_hash = canonical_path_hash(canonical);
4247 format!("local-unknown-{path_hash}-{arch}-kc{suffix}")
4248 }
4249 }
4250}
4251
4252/// CRC32 of the canonical source-path bytes, lowercase hex
4253/// (full 8-char width — the entire 32-bit value). Disambiguates
4254/// `local-unknown-...` cache keys and per-source-tree lockfile
4255/// names across distinct dirty / non-git source trees so two
4256/// parallel `cargo ktstr test --kernel ./linux-a` and
4257/// `--kernel ./linux-b` runs can't write each other's vmlinux into
4258/// the same cache slot or share a single source-tree flock.
4259///
4260/// Full 32 bits (8 hex chars) of CRC32 keep collision risk
4261/// negligible against the practical population (handful of source
4262/// trees per host) while staying human-readable. The earlier
4263/// 6-char (24-bit) form left ~6× the collision surface for the
4264/// same key shape; truncation served no purpose other than visual
4265/// brevity. Path bytes are taken via `OsStr::as_encoded_bytes` so
4266/// a non-UTF-8 component (rare on Linux but possible) doesn't lose
4267/// entropy through a UTF-8 lossy conversion.
4268pub(crate) fn canonical_path_hash(canonical: &Path) -> String {
4269 let bytes = canonical.as_os_str().as_encoded_bytes();
4270 format!("{:08x}", crc32fast::hash(bytes))
4271}
4272
4273/// Read `<canonical>/.config` and return its CRC32 as a lowercase
4274/// hex string suitable for embedding in the cache key. Returns
4275/// `None` when no `.config` exists (a fresh tree before the build
4276/// runs `make defconfig`).
4277///
4278/// Distinct from the `config_hash` written into [`crate::cache::KernelMetadata`]
4279/// at store time — that records the FINAL `.config` after
4280/// configuration runs, for diagnostic display in `kernel list`.
4281/// This helper records the PRE-BUILD `.config` so the cache key
4282/// reflects what the operator's tree currently has on disk; the
4283/// same `.config` content always maps to the same key, even if the
4284/// downstream `make olddefconfig` step elaborates additional
4285/// defaults.
4286fn config_hash_for_key(canonical: &Path) -> Option<String> {
4287 let config_path = canonical.join(".config");
4288 let data = std::fs::read(&config_path).ok()?;
4289 Some(format!("{:08x}", crc32fast::hash(&data)))
4290}
4291
4292#[cfg(test)]
4293#[path = "fetch_tests.rs"]
4294mod tests;