ktstr/probe/
process.rs

1//! Probe skeleton lifecycle: load, attach, run, collect.
2//!
3//! When the pipeline is split into two phases (see [`PhaseBInput`]):
4//! - **Phase A** attaches kprobes + the trigger + kernel fexit before
5//!   the scheduler starts; it runs under the initial skeleton load.
6//! - **Phase B** attaches fentry/fexit to the scheduler's BPF
7//!   struct_ops callbacks after the scheduler is up and BPF programs
8//!   are discoverable.
9//!
10//! In the single-phase path, all probes attach after the scheduler is
11//! up.
12//!
13//! ## Two-phase sync mechanism
14//!
15//! Phase A runs on a probe worker thread. Caller and worker
16//! synchronize via two `Latch`es, one status slot, and one mpsc channel:
17//!
18//! 1. Caller spawns the probe worker, which loads the skeleton and
19//!    attaches kprobes + trigger + kernel fexit, publishes Phase A's
20//!    success/failure status, then signals the `probes_ready` latch
21//!    (see `publish_phase_a` below). On success the worker enters the
22//!    ringbuf poll loop.
23//! 2. Caller waits on `probes_ready` and reads the published status.
24//!    Only after success does it start the scheduler — the scheduler
25//!    launches with Phase A probes already attached, so the trigger
26//!    and any kprobes that fire during scheduler init are observed.
27//! 3. After the scheduler is up, the caller discovers BPF programs
28//!    by scheduler pid (see `discover_bpf_symbols` /
29//!    `expand_bpf_to_kernel_callers`) and sends a [`PhaseBInput`]
30//!    over the channel. The Phase B input includes BPF program FDs
31//!    held open while the scheduler is alive.
32//! 4. The probe worker's poll loop calls `try_recv` on the channel
33//!    every 100 ms; on receipt it attaches BPF fentry/fexit + extra
34//!    kprobes for kernel callers, then signals the `done` latch on
35//!    the [`PhaseBInput`].
36//! 5. Caller waits on `done` and proceeds to the test scenario with
37//!    full instrumentation in place.
38
39use std::collections::HashSet;
40use std::sync::Arc;
41use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
42use std::time::Duration;
43
44use super::btf::{BtfFunc, RenderHint, STRUCT_FIELDS};
45use super::stack::StackFunction;
46
47use crate::bpf_skel::types;
48use crate::sync::{Latch, RwLockExt};
49
50/// Cross-thread mirror of the BPF probe's `ktstr_err_exit_detected`
51/// BSS latch. The probe-poll thread reads the BSS slot via volatile
52/// load each iteration and `store`s the observed value here; scenario
53/// code calls [`sched_exit_kind`] from any thread to read the current
54/// classification synchronously without owning the skeleton.
55///
56/// Encoded as `u32` for `AtomicU32`-friendliness:
57/// - `0` (initial / probe-not-yet-armed / scheduler still alive):
58///   [`SchedExitKind::Unknown`].
59/// - `1` ([`SchedExitKind::Clean`]) — probe armed, BSS latch read as
60///   `0` after at least one poll iteration. Means the scheduler is
61///   either still alive (latch never set) or exited via the clean
62///   `SCX_EXIT_NONE` path.
63/// - `2` ([`SchedExitKind::Crashed`]) — probe armed, BSS latch
64///   observed at `!= 0`. Set by the selected BPF exit handler
65///   after any successfully claimed non-clean kernel exit
66///   (`SCX_EXIT_ERROR`, `SCX_EXIT_ERROR_STALL`, watchdog kick,
67///   BPF-side error).
68///
69/// 0 → 1/2 transitions are monotonic for a single test run; once the
70/// accepted exit path returns, the latch never clears. Cross-thread
71/// observation uses Release / Acquire so the scenario-side load sees
72/// every prior probe-side store.
73pub(crate) static PROBE_SCHED_EXIT_STATE: AtomicU32 = AtomicU32::new(0);
74
75const PROBE_EXIT_STATE_CLEAN: u32 = 1;
76const PROBE_EXIT_STATE_CRASHED: u32 = 2;
77
78/// Classification of the scheduler's exit observed by the probe
79/// poll thread. Mirrors the BSS latch in
80/// [`PROBE_SCHED_EXIT_STATE`].
81#[derive(Debug, Clone, Copy, PartialEq, Eq)]
82pub enum SchedExitKind {
83    /// Probe has not observed the scheduler's exit yet — either the
84    /// scheduler is still alive, the probe never armed for this run,
85    /// or the poll thread has not completed a single iteration since
86    /// the prior reset. Callers gating clean-vs-crash classification
87    /// on a known-dead scheduler treat `Unknown` as a probe-coverage
88    /// gap, not as evidence of either outcome.
89    Unknown,
90    /// Probe armed AND completed at least one observation with the
91    /// BSS latch unset. Scheduler is either still alive or exited
92    /// cleanly via the `SCX_EXIT_NONE` path that does not latch the
93    /// error flag.
94    Clean,
95    /// Probe armed AND observed the BSS latch set. The probe observed a
96    /// successful non-clean scheduler exit (error, stall,
97    /// watchdog kick, BPF-side error) before this read.
98    Crashed,
99}
100
101/// Read the current probe-observed scheduler exit classification.
102/// Synchronous, cross-thread-safe (Acquire load mirrors the probe's
103/// Release stores). Returns [`SchedExitKind::Unknown`] when the
104/// probe has not yet completed its first poll iteration.
105pub fn sched_exit_kind() -> SchedExitKind {
106    match PROBE_SCHED_EXIT_STATE.load(Ordering::Acquire) {
107        PROBE_EXIT_STATE_CLEAN => SchedExitKind::Clean,
108        PROBE_EXIT_STATE_CRASHED => SchedExitKind::Crashed,
109        _ => SchedExitKind::Unknown,
110    }
111}
112
113/// Render the useful tail of libbpf's verifier/load log for an early Phase-A
114/// failure.
115///
116/// The complete bounded log remains in `ProbeDiagnostics`; guest init needs a
117/// concise reason before it reboots, otherwise a missing typed-BTF target is
118/// reduced to the opaque outer `ESRCH`. Preserve the final six non-empty
119/// records (where libbpf places the program and target diagnostics), collapse
120/// whitespace, and cap the result so this control error cannot flood stdout.
121fn bounded_libbpf_tail(log: &str) -> String {
122    const MAX_LINES: usize = 6;
123    const MAX_CHARS: usize = 1_000;
124
125    let mut lines: Vec<String> = log
126        .lines()
127        .map(|line| line.split_whitespace().collect::<Vec<_>>().join(" "))
128        .filter(|line| !line.is_empty())
129        .rev()
130        .take(MAX_LINES)
131        .collect();
132    lines.reverse();
133    let condensed = lines.join(" | ");
134    if condensed.chars().count() > MAX_CHARS {
135        format!(
136            "…{}",
137            condensed
138                .chars()
139                .rev()
140                .take(MAX_CHARS - 1)
141                .collect::<String>()
142                .chars()
143                .rev()
144                .collect::<String>()
145        )
146    } else {
147        condensed
148    }
149}
150
151/// Kernel-generation-specific hook used to observe sched_ext exits.
152///
153/// All programs live in the same BPF object. Userspace selects exactly one
154/// before load so a missing target cannot reject the whole object.
155#[derive(Debug, Clone, Copy, PartialEq, Eq)]
156enum SchedulerExitTrigger {
157    /// Per-scheduler typed tracepoint emitted after a successful claim.
158    SchedExtExitTracepoint,
159    /// Raw entry/return pair around the preceding generation's exit path.
160    ScxVexitKprobePair,
161    /// Global-era error dump path used by 6.14.
162    ScxDumpState,
163}
164
165impl SchedulerExitTrigger {
166    fn diagnostic_type(self) -> &'static str {
167        match self {
168            Self::SchedExtExitTracepoint => "tp_btf",
169            Self::ScxVexitKprobePair => "kprobe+kretprobe",
170            Self::ScxDumpState => "fentry-global",
171        }
172    }
173
174    fn section(self) -> &'static str {
175        match self {
176            Self::SchedExtExitTracepoint => "tp_btf/sched_ext_exit",
177            Self::ScxVexitKprobePair => "kprobe+kretprobe/scx_vexit",
178            Self::ScxDumpState => "fentry/scx_dump_state",
179        }
180    }
181}
182
183#[derive(Debug, Clone, Copy, PartialEq, Eq)]
184struct SchedulerExitProgramSelection {
185    tracepoint: bool,
186    vexit_enter: bool,
187    vexit_return: bool,
188    dump_fentry: bool,
189}
190
191/// Return the complete pre-load autoload selection for a scheduler-exit
192/// trigger. Keeping the raw scx_vexit entry and return halves in one
193/// projection prevents a future selector change from loading only half of the
194/// correlation pair.
195fn scheduler_exit_program_selection(
196    trigger: SchedulerExitTrigger,
197) -> SchedulerExitProgramSelection {
198    SchedulerExitProgramSelection {
199        tracepoint: trigger == SchedulerExitTrigger::SchedExtExitTracepoint,
200        vexit_enter: trigger == SchedulerExitTrigger::ScxVexitKprobePair,
201        vexit_return: trigger == SchedulerExitTrigger::ScxVexitKprobePair,
202        dump_fentry: trigger == SchedulerExitTrigger::ScxDumpState,
203    }
204}
205
206/// Prefer the exact post-claim tracepoint whenever it exists. It has a stable
207/// two-argument ABI while `scx_vexit`'s trailing by-value `va_list` has an
208/// architecture- and compiler-dependent BTF representation. The raw
209/// `scx_vexit` kprobe/kretprobe pair is therefore selected by function
210/// presence when the tracepoint is absent, and the two-argument global-era
211/// dump hook is the final fallback.
212fn select_scheduler_exit_trigger(
213    has_sched_ext_exit_tracepoint: bool,
214    has_scx_vexit: bool,
215    has_scx_dump_state_2: bool,
216) -> Option<SchedulerExitTrigger> {
217    if has_sched_ext_exit_tracepoint {
218        Some(SchedulerExitTrigger::SchedExtExitTracepoint)
219    } else if has_scx_vexit {
220        Some(SchedulerExitTrigger::ScxVexitKprobePair)
221    } else if has_scx_dump_state_2 {
222        Some(SchedulerExitTrigger::ScxDumpState)
223    } else {
224        None
225    }
226}
227
228fn btf_has_function(btf: &btf_rs::Btf, name: &str) -> bool {
229    btf.resolve_types_by_name(name)
230        .is_ok_and(|types| types.iter().any(|ty| matches!(ty, btf_rs::Type::Func(_))))
231}
232
233fn btf_has_function_arity(btf: &btf_rs::Btf, name: &str, arity: usize) -> bool {
234    btf.resolve_types_by_name(name).is_ok_and(|types| {
235        types.iter().any(|ty| {
236            let btf_rs::Type::Func(func) = ty else {
237                return false;
238            };
239            matches!(
240                btf.resolve_chained_type(func),
241                Ok(btf_rs::Type::FuncProto(proto)) if proto.parameters.len() == arity
242            )
243        })
244    })
245}
246
247/// `tp_btf/<name>` attachment is keyed by the kernel-generated
248/// `btf_trace_<name>` typedef, not the ordinary tracefs event record.
249///
250/// A typed tracepoint typedef chains to a function pointer. Its prototype has
251/// one hidden tracepoint context parameter followed by the explicit event
252/// arguments; `sched_ext_exit(sch, kind)` therefore has arity three in BTF.
253/// Validate the whole chain so a same-named, non-attachable typedef cannot
254/// select a program that libbpf will reject at load time.
255fn btf_has_typed_tracepoint_arity(btf: &btf_rs::Btf, name: &str, arity: usize) -> bool {
256    let target = format!("btf_trace_{name}");
257    btf.resolve_types_by_name(&target).is_ok_and(|types| {
258        types.iter().any(|ty| {
259            let btf_rs::Type::Typedef(typedef) = ty else {
260                return false;
261            };
262            let Ok(btf_rs::Type::Ptr(ptr)) = btf.resolve_chained_type(typedef) else {
263                return false;
264            };
265            matches!(
266                btf.resolve_chained_type(&ptr),
267                Ok(btf_rs::Type::FuncProto(proto)) if proto.parameters.len() == arity
268            )
269        })
270    })
271}
272
273/// Override the cross-thread exit-state mirror. The probe poll thread
274/// is the production writer (it mirrors the BSS `ktstr_err_exit_detected`
275/// latch into [`PROBE_SCHED_EXIT_STATE`]); this lets host-side tests,
276/// where no BPF probe runs, simulate an observed crash latch. Pair a
277/// `Crashed` set with a reset to [`SchedExitKind::Unknown`] so neighbor
278/// tests start from a clean mirror.
279#[cfg(test)]
280pub(crate) fn set_probe_sched_exit_state(kind: SchedExitKind) {
281    let encoded = match kind {
282        SchedExitKind::Unknown => 0,
283        SchedExitKind::Clean => PROBE_EXIT_STATE_CLEAN,
284        SchedExitKind::Crashed => PROBE_EXIT_STATE_CRASHED,
285    };
286    PROBE_SCHED_EXIT_STATE.store(encoded, Ordering::Release);
287}
288
289/// Input for Phase B probe attachment (BPF fentry/fexit).
290///
291/// Sent via mpsc channel after the scheduler starts and BPF programs
292/// are discoverable. Phase A (kprobes + trigger + kernel fexit) runs
293/// before the scheduler; Phase B attaches fentry/fexit to the
294/// scheduler's BPF struct_ops callbacks.
295pub struct PhaseBInput {
296    /// BPF functions and kernel callers discovered from the running
297    /// scheduler. Includes both BPF callbacks (for fentry) and their
298    /// kernel-side callers from `expand_bpf_to_kernel_callers` (for
299    /// additional kprobes).
300    pub functions: Vec<StackFunction>,
301    /// Pre-opened BPF program FDs keyed by prog_id.
302    pub bpf_prog_fds: std::collections::HashMap<u32, i32>,
303    /// BTF-resolved function signatures for BPF callbacks and kernel callers.
304    pub btf_funcs: Vec<BtfFunc>,
305    /// Signaled by the probe worker thread once Phase B attachment
306    /// completes so the dispatch path can proceed past its wait.
307    pub done: Arc<Latch>,
308    /// Starting func_idx for Phase B functions. Must equal the number
309    /// of functions in Phase A to avoid index collisions in the shared
310    /// `func_meta_map` and `probe_data` maps.
311    pub func_idx_offset: u32,
312}
313
314/// Ring buffer event type for the trigger (matches `EVENT_TRIGGER`
315/// in `intf.h`). Currently the only record type emitted on the
316/// `ktstr_events` ringbuf — `EVENT_SCX_EVENT` was removed alongside
317/// the `tp_btf/sched_ext_event` BPF handler.
318const EVENT_TRIGGER: u32 = 2;
319
320/// Maximum string length carried in a probe_event entry (matches
321/// `MAX_STR_LEN` in `intf.h`). Used to size the `RbEvent.str_val`
322/// field for byte-level wire compatibility with `struct probe_event`
323/// in `intf.h`; the Rust dispatch path leaves it zeroed because
324/// the only producer that populated it (the removed
325/// `tp_btf/sched_ext_event` handler) is gone.
326const MAX_STR_LEN: usize = 64;
327
328/// Pipeline diagnostics from a probe run.
329///
330/// Tracks how many functions/events survived each stage so users can
331/// see WHERE data is being lost (filter, attach, capture, stitch).
332#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
333pub struct ProbeDiagnostics {
334    /// Kernel functions resolved to IPs.
335    pub kprobe_resolved: u32,
336    /// Kernel functions that failed IP resolution.
337    pub kprobe_resolve_failed: Vec<String>,
338    /// Kprobes successfully attached.
339    pub kprobe_attached: u32,
340    /// Kprobes that failed to attach (name, error).
341    pub kprobe_attach_failed: Vec<(String, String)>,
342    /// BPF functions with valid prog IDs for fentry.
343    pub fentry_candidates: u32,
344    /// Fentry probes successfully attached.
345    pub fentry_attached: u32,
346    /// Fentry probes that failed (name, error).
347    pub fentry_attach_failed: Vec<(String, String)>,
348    /// Total keys in probe_data map at readout.
349    pub probe_data_keys: u32,
350    /// Keys with unmatched IPs (no func_meta entry).
351    pub probe_data_unmatched_ips: u32,
352    /// Events read from probe_data before stitching.
353    pub events_before_stitch: u32,
354    /// Events surviving tptr+time stitching.
355    pub events_after_stitch: u32,
356    /// Whether the trigger fired.
357    pub trigger_fired: bool,
358    /// Which trigger mechanism attached ("tp_btf", "fexit", or
359    /// "fentry-global").
360    pub trigger_type: String,
361    /// Error from the selected scheduler-exit hook.
362    #[serde(default, skip_serializing_if = "Option::is_none")]
363    pub trigger_attach_error: Option<String>,
364    /// Panic payload from the guest-side probe-collection thread
365    /// when its `JoinHandle::join()` returned `Err`. `None` on a
366    /// clean run (thread exited normally — events may still be
367    /// empty if the trigger never fired). `Some(payload)`
368    /// distinguishes "the probe thread crashed before producing
369    /// events" from "the probe thread ran cleanly and observed no
370    /// trigger" — the COM2 payload's `events: []` is otherwise
371    /// indistinguishable between those two cases. Any consumer of
372    /// the payload (host harness, render layer, downstream test
373    /// verdict) MUST treat `Some(_)` as a failure even when
374    /// `trigger.fired == false` and `events` is empty.
375    #[serde(default, skip_serializing_if = "Option::is_none")]
376    pub host_thread_panic: Option<String>,
377    /// BPF-side kprobe fire count (cross-CPU sum of the
378    /// `KTSTR_PCPU_PROBE_COUNT` slot in `ktstr_pcpu_counters`).
379    pub bpf_kprobe_fires: u64,
380    /// BPF-side kprobe commit count (cross-CPU sum of the
381    /// `KTSTR_PCPU_KPROBE_RETURNS` slot).
382    /// `bpf_kprobe_fires - bpf_kprobe_returns` is the number of
383    /// kprobe fires that bailed before pushing into `probe_data`
384    /// (meta-map miss or scratch-slot miss).
385    pub bpf_kprobe_returns: u64,
386    /// BPF-side trigger fire count (cross-CPU sum of the
387    /// `KTSTR_PCPU_TRIGGER_COUNT` slot).
388    pub bpf_trigger_fires: u64,
389    /// BPF-side func_meta_map misses (cross-CPU sum of the
390    /// `KTSTR_PCPU_META_MISS` slot — IP not found in map).
391    pub bpf_meta_misses: u64,
392    /// IPs that missed func_meta_map lookup (from BSS ktstr_miss_log).
393    pub bpf_miss_ips: Vec<u64>,
394    /// BPF-side `bpf_ringbuf_reserve` failures inside the trigger
395    /// handler (cross-CPU sum of `KTSTR_PCPU_RINGBUF_DROPS`).
396    /// Non-zero means the userspace consumer fell behind on the
397    /// events ringbuf, so auto-repro will see a missing trigger
398    /// event even though the scheduler did fire.
399    pub bpf_ringbuf_drops: u64,
400    /// Nanosecond timestamp captured by the BPF trigger handler on
401    /// the first accepted error-class scheduler exit (from BSS
402    /// ktstr_last_trigger_ts). 0 when no error-class exit fired.
403    pub bpf_first_trigger_ns: u64,
404    /// `kind` argument captured by the BPF trigger handler on the
405    /// first accepted error-class scheduler exit (from BSS
406    /// `ktstr_exit_kind_snap`). 0 when no error-class exit fired,
407    /// otherwise one of the [`SCX_EXIT_*`](super::scx_defs) values.
408    /// Used by the host renderer to disambiguate "trigger fired with
409    /// kind=STALL/ERROR (no causal task; events suppressed)" from
410    /// "trigger never fired" when the post-stitch event count is 0.
411    pub bpf_exit_kind_snap: u32,
412    /// `true` when the readout phase reached the no-causal-tptr
413    /// branch and emitted events grouped by frequency rather than
414    /// stitched against a real trigger task pointer. Set in
415    /// [`run_probe_skeleton`] when the trigger fired with
416    /// `args[0] == 0` (kind=STALL or generic ERROR) or never fired
417    /// at all but at least one captured kprobe event had a non-zero
418    /// task pointer. Surfaced by the host renderer as
419    /// `events: ... — trigger absent, grouped by frequency` so the
420    /// operator does not misread the candidate chain as a verified
421    /// stitch.
422    pub stitch_fallback_used: bool,
423    /// Cumulative count of `tp_btf/sched_switch +
424    /// sched_migrate_task + sched_wakeup` records committed into
425    /// the dedicated `timeline_events` ringbuf by the timeline
426    /// handlers (cross-CPU sum of the
427    /// `KTSTR_PCPU_TIMELINE_COUNT` slot). Zero before any of
428    /// those tracepoints fire; otherwise grows continuously
429    /// while the probe runs. Combined with `bpf_timeline_drops`
430    /// it lets an operator tell whether a failure-time drain
431    /// saw the full window or only the tail.
432    pub bpf_timeline_count: u64,
433    /// `bpf_ringbuf_reserve` failures across the three timeline
434    /// tracepoint handlers (sched_switch / sched_migrate_task /
435    /// sched_wakeup), aggregated as cross-CPU sum of the
436    /// `KTSTR_PCPU_TIMELINE_DROPS` slot. Each drop is one new
437    /// event lost — the ring's existing contents are NOT evicted
438    /// on overflow, so the drain on failure recovers the OLDEST
439    /// captured events first.
440    pub bpf_timeline_drops: u64,
441}
442
443/// Structured probe event captured by the BPF skeleton.
444///
445/// One per (function, task_ptr) combination. `fields` contains BTF-resolved
446/// struct field values keyed as `"param:struct.field"` (from
447/// [`build_field_keys`]). Events are sorted by `ts` and stitched by
448/// `task_struct` pointer before output.
449#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
450pub struct ProbeEvent {
451    /// Index into the run's function list (matches the slot used in
452    /// the shared `func_meta_map`).
453    pub func_idx: u32,
454    /// `task_struct` pointer for the thread that triggered the probe;
455    /// used to stitch entry/exit pairs.
456    pub task_ptr: u64,
457    /// Nanosecond timestamp captured at entry.
458    pub ts: u64,
459    /// First six callee arguments captured verbatim from the call
460    /// site (after convention lowering).
461    pub args: [u64; 6],
462    /// BTF-resolved struct field values decoded post-hoc by the
463    /// caller; each entry is `(field_key, raw_value)`.
464    pub fields: Vec<(String, u64)>,
465    /// Kernel stack trace captured at entry, most recent frame first.
466    pub kstack: Vec<u64>,
467    /// Optional UTF-8 string associated with the event (e.g. a
468    /// scheduler-provided exit reason).
469    #[serde(default, skip_serializing_if = "Option::is_none")]
470    pub str_val: Option<String>,
471    /// Post-mutation field values captured by fexit.
472    /// Same field keys as `fields`, paired by index.
473    #[serde(default, skip_serializing_if = "Vec::is_empty")]
474    pub exit_fields: Vec<(String, u64)>,
475    /// Timestamp when fexit fired.
476    #[serde(default, skip_serializing_if = "Option::is_none")]
477    pub exit_ts: Option<u64>,
478}
479
480/// Parse `/proc/kallsyms` into a `name -> address` map. Returns `None`
481/// when the file is unreadable (expected outside a privileged context)
482/// OR when every parsed entry has a zero address (see
483/// [`accept_kallsyms_map`] for the kptr_restrict rationale).
484fn load_kallsyms() -> Option<std::collections::HashMap<String, u64>> {
485    let raw = std::fs::read_to_string("/proc/kallsyms").ok()?;
486    accept_kallsyms_map(parse_kallsyms(&raw))
487}
488
489/// Return `Some(map)` when at least one entry has a non-zero address,
490/// otherwise `None`. The all-zero case is what the kernel emits under
491/// `kernel.kptr_restrict=2` for non-CAP_SYSLOG callers — the file is
492/// readable, all symbol names are present, but every line carries
493/// `0000000000000000` for its address. Caching such a map would
494/// poison every later [`resolve_func_ip`] lookup with `Some(0)`,
495/// masking the unprivileged state from the retry-after-sudo path;
496/// treating it as a load failure instead lets the next caller (after
497/// [`RETRY_MIN_INTERVAL`]) try again under the new privilege level.
498fn accept_kallsyms_map(
499    map: std::collections::HashMap<String, u64>,
500) -> Option<std::collections::HashMap<String, u64>> {
501    if !map.values().any(|&a| a != 0) {
502        tracing::warn!(
503            entries = map.len(),
504            "/proc/kallsyms parsed with zero addresses only — kptr_restrict \
505             likely active; declining to cache",
506        );
507        return None;
508    }
509    Some(map)
510}
511
512/// Parse kallsyms-format text (one `HEX TYPE NAME ...` line per
513/// symbol) into a `name -> address` map. Extracted from
514/// [`load_kallsyms`] so unit tests can exercise the parser without
515/// touching `/proc/kallsyms`, which is usually unreadable in the
516/// unprivileged contexts the crate runs under.
517///
518/// Skipped lines (silently, without affecting other symbols):
519/// - lines with fewer than 3 whitespace-separated tokens (addr,
520///   type, name — all three are required; the type column is
521///   accepted but ignored)
522/// - lines whose first token is not a hex-parseable `u64`
523///
524/// A permanently-empty map is a valid return value — callers treat
525/// it as "no symbols found" rather than an error.
526fn parse_kallsyms(raw: &str) -> std::collections::HashMap<String, u64> {
527    // No pre-scan — HashMap grows from empty in a single pass over
528    // `raw`.
529    let mut map = std::collections::HashMap::new();
530    for line in raw.lines() {
531        let mut parts = line.split_whitespace();
532        let Some(addr) = parts.next() else { continue };
533        let _ty = parts.next();
534        let Some(sym) = parts.next() else { continue };
535        let Ok(addr) = u64::from_str_radix(addr, 16) else {
536            continue;
537        };
538        map.insert(sym.to_string(), addr);
539    }
540    map
541}
542
543/// Resolve a kernel function name to its address via /proc/kallsyms.
544///
545/// The parsed `name -> address` map is cached on first successful load
546/// so later lookups avoid re-reading and re-splitting ~200k lines.
547/// Callers that resolve many functions in a batch (auto-probe attach,
548/// probe-stack load) drop from O(N\*M) line scans to O(N) hash lookups.
549///
550/// A failed load (unreadable `/proc/kallsyms` — typical for
551/// unprivileged processes where the file is either missing or
552/// returns zeroed addresses) is rate-limited to one retry per
553/// `RETRY_MIN_INTERVAL` (1 s); calls within that window return
554/// `None` immediately. This matters both for performance — a caller
555/// resolving N symbols under a permanently unreadable
556/// `/proc/kallsyms` pays one load attempt, not N — and for
557/// privilege-escalation correctness, where a test harness that
558/// re-execs under `sudo` after the first miss still sees a retry
559/// within seconds.
560pub fn resolve_func_ip(name: &str) -> Option<u64> {
561    use std::sync::{OnceLock, RwLock};
562    use std::time::Instant;
563    static CACHE: OnceLock<RwLock<Option<std::collections::HashMap<String, u64>>>> =
564        OnceLock::new();
565    // Rate-limit the load retry on chronic failure. Without a
566    // floor, a caller that resolves N symbols under a permanently
567    // unreadable /proc/kallsyms triggers N * (read + parse) of a
568    // ~10 MB file. The floor turns that into one retry per window
569    // and returns `None` fast in between.
570    static LAST_LOAD_ATTEMPT: OnceLock<RwLock<Option<Instant>>> = OnceLock::new();
571
572    let slot = CACHE.get_or_init(|| RwLock::new(None));
573    // Fast path: take the read lock when the cache is populated.
574    // Post-load the lookup is read-only and batches resolving many
575    // symbols contend only on the shared read lock.
576    {
577        let read = slot.read_unpoisoned();
578        if let Some(map) = read.as_ref() {
579            return map.get(name).copied();
580        }
581    }
582    // Optional fast-decline: when the retry clock rules the caller
583    // out, avoid the write-lock acquire entirely. This is a
584    // performance hint only — correctness is enforced by the
585    // re-check below under the write lock, so a concurrent racer
586    // that slips past this gate still gets serialized.
587    let last_slot = LAST_LOAD_ATTEMPT.get_or_init(|| RwLock::new(None));
588    {
589        let last = last_slot.read_unpoisoned();
590        if let Some(t) = *last
591            && t.elapsed() < RETRY_MIN_INTERVAL
592        {
593            return None;
594        }
595    }
596    // Slow path: escalate to write lock to populate. Re-check both
597    // the cache and the retry clock under the write lock so N
598    // concurrent first-callers don't stampede into N serialized
599    // loads: only the winner gets past the timestamp gate, everyone
600    // else observes `*last = Some(now)` and bails.
601    let mut write = slot.write_unpoisoned();
602    if write.is_none() {
603        let mut last = last_slot.write_unpoisoned();
604        if last.is_none_or(|t| t.elapsed() >= RETRY_MIN_INTERVAL) {
605            *write = load_kallsyms();
606            *last = Some(Instant::now());
607        }
608    }
609    write.as_ref()?.get(name).copied()
610}
611
612/// Minimum interval between retry attempts when `/proc/kallsyms` is
613/// unreadable; see [`resolve_func_ip`] for the rationale.
614const RETRY_MIN_INTERVAL: std::time::Duration = std::time::Duration::from_secs(1);
615
616/// Build a `func_idx -> task_struct_param_idx` map for stitching by
617/// task pointer. Resolves each function's task-struct argument
618/// position from [`super::stack::BPF_OP_CALLERS`] first, falling back
619/// to the BTF param list (Phase A `btf_funcs` chained with Phase B
620/// `phase_b_btf`).
621///
622/// Entries with `pidx >= 6` are dropped with a warn rather than
623/// stored: the stitch code reads `ProbeEvent::args[pidx]` against a
624/// fixed-size `[u64; 6]` (matching the BPF-side capture limit), so
625/// any larger index would panic. A function with its task_struct
626/// past arg-6 simply cannot be stitched here — the BPF probe never
627/// captured that arg.
628fn build_task_param_idx(
629    func_ips: &[(u32, u64, String)],
630    btf_funcs: &[BtfFunc],
631    phase_b_btf: &[BtfFunc],
632) -> std::collections::HashMap<u32, usize> {
633    func_ips
634        .iter()
635        .filter_map(|(idx, _, name)| {
636            // BPF_OP_CALLERS: (op_fragment, kernel_caller, task_arg_idx)
637            let pidx = if let Some((_, _, tidx)) = super::stack::BPF_OP_CALLERS
638                .iter()
639                .find(|(_, caller, _)| *caller == name.as_str())
640            {
641                *tidx as usize
642            } else {
643                // Fallback: BTF params with task_struct
644                let btf = btf_funcs
645                    .iter()
646                    .chain(phase_b_btf.iter())
647                    .find(|f| f.name == *name)?;
648                btf.params
649                    .iter()
650                    .position(|p| p.struct_name.as_deref() == Some("task_struct"))?
651            };
652            if pidx >= 6 {
653                tracing::warn!(
654                    func = %name,
655                    pidx,
656                    "task_struct param index out of args[6] bounds — \
657                     skipping stitch entry",
658                );
659                return None;
660            }
661            Some((*idx, pidx))
662        })
663        .collect()
664}
665
666/// Populate a `func_meta` with field specs from BTF-resolved offsets.
667/// Shared between kprobe and fentry paths.
668///
669/// Invariant: `meta.nr_field_specs = max(field_idx) + 1`, NOT the count
670/// of specs. The BPF side writes `entry.fields[field_idx]` positionally,
671/// and the Rust side reads `entry.fields[..nr_field_specs]` positionally
672/// against [`build_field_keys`] (which includes skipped fields). A
673/// reorder of either loop that turns this into a count would silently
674/// mismatch keys to values.
675fn populate_field_specs(meta: &mut types::func_meta, field_specs: &[super::btf::FieldSpec]) {
676    let n = field_specs.len().min(16);
677    let max_fidx = field_specs
678        .iter()
679        .take(n)
680        .map(|fs| fs.field_idx)
681        .max()
682        .map(|m| m + 1)
683        .unwrap_or(0);
684    meta.nr_field_specs = max_fidx.min(16);
685    for fs in field_specs.iter().take(n) {
686        let slot = fs.field_idx as usize;
687        if slot < 16 {
688            meta.specs[slot] = types::field_spec {
689                param_idx: fs.param_idx,
690                offset: fs.offset,
691                size: fs.size,
692                field_idx: fs.field_idx,
693                ptr_offset: fs.ptr_offset,
694            };
695        }
696    }
697}
698
699/// Build field key names for a function based on its BTF info.
700///
701/// Returns a vec mapping `field_idx` to an output key name. Format:
702/// - Known struct param: `"p:task_struct.pid"`
703/// - Auto-discovered BPF struct: `"ctx:task_ctx.field_a"`
704/// - Scalar param: `"flags:val.flags"`
705///
706/// Processes at most 6 params (fentry/kprobe register limit) and
707/// at most 16 fields total (matching `MAX_FIELDS` in intf.h).
708///
709/// Invariant: keys are emitted in the same order [`populate_field_specs`]
710/// consumes `field_specs`, so `entry.fields[i]` maps to the i-th key.
711fn build_field_keys(btf_func: &BtfFunc) -> Vec<(String, RenderHint)> {
712    let mut keys = Vec::new();
713    let mut field_idx: u32 = 0;
714
715    let max_params = btf_func.params.len().min(6);
716    for param in &btf_func.params[..max_params] {
717        if let Some(ref sname) = param.struct_name {
718            if let Some((_, fields)) = STRUCT_FIELDS.iter().find(|(s, _)| *s == sname) {
719                for (_, key) in *fields {
720                    // Known struct fields use dedicated decoders in
721                    // decode.rs — hint is irrelevant (Default/Hex).
722                    keys.push((format!("{}:{}.{}", param.name, sname, key), RenderHint::Hex));
723                    field_idx += 1;
724                    if field_idx >= 16 {
725                        break;
726                    }
727                }
728            }
729        } else if !param.auto_fields.is_empty() {
730            let tname = param.type_name.as_deref().unwrap_or("void");
731            for (fname, _, hint) in &param.auto_fields {
732                keys.push((format!("{}:{}.{}", param.name, tname, fname), *hint));
733                field_idx += 1;
734                if field_idx >= 16 {
735                    break;
736                }
737            }
738        } else if !param.is_ptr {
739            keys.push((
740                format!("{}:val.{}", param.name, param.name),
741                RenderHint::Hex,
742            ));
743            field_idx += 1;
744        }
745    }
746
747    keys
748}
749
750/// Detect which param (if any) is a char * string.
751/// Uses BTF type detection first, then name heuristic as fallback.
752/// Returns 0xff if none found.
753fn detect_str_param(btf_func: &BtfFunc) -> u8 {
754    let max = btf_func.params.len().min(6);
755    // BTF-based: check is_string_ptr flag set by parse_btf_functions.
756    for (i, p) in btf_func.params[..max].iter().enumerate() {
757        if p.is_string_ptr {
758            return i as u8;
759        }
760    }
761    // Name heuristic fallback.
762    for (i, p) in btf_func.params[..max].iter().enumerate() {
763        if !p.is_ptr || p.struct_name.is_some() {
764            continue;
765        }
766        let n = p.name.as_str();
767        if matches!(n, "fmt" | "msg" | "str" | "reason" | "buf" | "s")
768            || n.contains("str")
769            || n.contains("msg")
770            || n.contains("fmt")
771        {
772            return i as u8;
773        }
774    }
775    0xff
776}
777
778/// Pre-open BPF program FDs while the scheduler is alive.
779///
780/// Returns a map from `bpf_prog_id` to owned fd. Holding these FDs
781/// keeps the BPF programs alive via kernel refcounting even after the
782/// scheduler exits. Must be called before the test function runs
783/// (which may crash the scheduler).
784pub fn open_bpf_prog_fds(functions: &[StackFunction]) -> std::collections::HashMap<u32, i32> {
785    let mut fds = std::collections::HashMap::new();
786    for f in functions {
787        if let Some(prog_id) = f.bpf_prog_id {
788            let fd = unsafe { libbpf_rs::libbpf_sys::bpf_prog_get_fd_by_id(prog_id) };
789            if fd >= 0 {
790                fds.insert(prog_id, fd);
791            }
792        }
793    }
794    fds
795}
796
797/// `&ProgramMut<'_>` newtype that asserts thread-shared access is
798/// safe for [`libbpf_rs::ProgramMut::attach_kprobe`].
799///
800/// `ProgramMut` holds a `NonNull<bpf_program>` which keeps it
801/// `!Sync` at the Rust type-system level. Concurrent attach is
802/// nonetheless sound:
803/// - `bpf_program__attach_kprobe_opts` (the libbpf C path
804///   `attach_kprobe` calls) takes `const struct bpf_program *prog` —
805///   it does not mutate the program through that pointer.
806/// - Each attach call creates an independent `perf_event_open` fd
807///   plus an independent `BPF_LINK_CREATE` syscall; those resources
808///   are not shared with the program object.
809/// - The only program-adjacent state the path touches is the global
810///   feature-detection cache via `kernel_supports` →
811///   `feat_supported`, which uses `READ_ONCE` / `WRITE_ONCE` and is
812///   idempotent across racing readers (see
813///   `libbpf-sys/libbpf/src/features.c::feat_supported`).
814///   All threads call only `attach_kprobe` on the inner reference; no
815///   mutating method is invoked here.
816///
817/// `Send` is paired with `Sync` because `std::thread::scope` requires
818/// the captured reference's referent to be `Sync` for `&_` to be
819/// `Send`.
820struct AttachableProgRef<'a, 'obj> {
821    inner: &'a libbpf_rs::ProgramMut<'obj>,
822}
823// SAFETY: see the type-level doc.
824unsafe impl Send for AttachableProgRef<'_, '_> {}
825// SAFETY: see the type-level doc.
826unsafe impl Sync for AttachableProgRef<'_, '_> {}
827
828/// Attach `prog` as a kprobe to every name in `func_names`,
829/// parallelising across worker threads.
830///
831/// Each attach is an independent `perf_event_open` + `BPF_LINK_CREATE`
832/// syscall pair. Sequentialised, the per-attach syscall round-trip
833/// dominates the auto-repro setup phase when the crash backtrace
834/// names many kernel functions. Spawning a small worker pool lets
835/// the kernel run the attaches concurrently and shrinks the total
836/// wall-clock cost of the loop to roughly `total / parallelism`
837/// (bounded by kernel-side serialisation inside `perf_event_open` for
838/// kprobe registration).
839///
840/// Worker count is `min(func_names.len(), 8)` — a fixed cap that
841/// matches the typical small ktstr backtrace width while avoiding
842/// thread-spawn overhead when there are only a handful of probes.
843/// Returns one `(name, Result<Link, libbpf_rs::Error>)` entry per
844/// input in input order, so the caller can populate
845/// [`ProbeDiagnostics::kprobe_attach_failed`] / `links` with the
846/// same shape as the prior sequential loop.
847fn parallel_attach_kprobes<'a, 'obj>(
848    prog: &'a libbpf_rs::ProgramMut<'obj>,
849    func_names: &[String],
850) -> Vec<(String, libbpf_rs::Result<libbpf_rs::Link>)> {
851    if func_names.is_empty() {
852        return Vec::new();
853    }
854
855    let prog_ref = AttachableProgRef { inner: prog };
856    // `min(N, 8)` balances syscall parallelism against thread-spawn
857    // overhead. Empirically the kernel kprobe registration path
858    // serialises on `kprobe_mutex` so going wider yields diminishing
859    // returns; the cap keeps us comfortably under that wall.
860    const MAX_WORKERS: usize = 8;
861    let workers = func_names.len().min(MAX_WORKERS);
862
863    // Slot the function names into round-robin per-worker buckets
864    // so each thread owns a disjoint subset. Index-tagged so the
865    // output preserves input order regardless of which worker
866    // finishes first.
867    let mut buckets: Vec<Vec<(usize, String)>> = (0..workers).map(|_| Vec::new()).collect();
868    for (i, name) in func_names.iter().enumerate() {
869        buckets[i % workers].push((i, name.clone()));
870    }
871
872    let mut results: Vec<Option<(String, libbpf_rs::Result<libbpf_rs::Link>)>> =
873        (0..func_names.len()).map(|_| None).collect();
874
875    std::thread::scope(|s| {
876        let mut handles = Vec::with_capacity(workers);
877        for bucket in buckets {
878            if bucket.is_empty() {
879                continue;
880            }
881            let prog_ref = &prog_ref;
882            handles.push(s.spawn(move || {
883                let mut out: Vec<(usize, String, libbpf_rs::Result<libbpf_rs::Link>)> =
884                    Vec::with_capacity(bucket.len());
885                for (i, name) in bucket {
886                    let r = prog_ref.inner.attach_kprobe(false, &name);
887                    out.push((i, name, r));
888                }
889                out
890            }));
891        }
892        for h in handles {
893            // Worker panics propagate as the join Err — re-panic on
894            // the main thread to surface bugs rather than silently
895            // dropping attach results.
896            let out = h.join().expect("kprobe attach worker panicked");
897            for (i, name, r) in out {
898                results[i] = Some((name, r));
899            }
900        }
901    });
902
903    results
904        .into_iter()
905        .map(|o| o.expect("every input slot must be filled by exactly one worker"))
906        .collect()
907}
908
909/// Attach the fentry program in slot 0..=3 on the fentry skeleton.
910///
911/// The fentry skeleton exposes four indexed programs
912/// (`ktstr_fentry_0`..`ktstr_fentry_3`) matching the 4-slot batch
913/// model in `src/bpf/fentry_probe.bpf.c`. Call sites previously spelled the
914/// full 4-arm `match slot { ... }` inline; routing through this
915/// family of helpers keeps the dispatch in one place so a future
916/// slot addition is a one-line change per helper instead of
917/// scattered across every batch.
918///
919/// Returns `None` for slot indices outside 0..=3, matching the
920/// existing `continue;` behaviour at call sites.
921fn attach_fentry_by_slot(
922    skel: &crate::bpf_skel::fentry::FentryProbeSkel<'_>,
923    slot: usize,
924) -> Option<libbpf_rs::Result<libbpf_rs::Link>> {
925    Some(match slot {
926        0 => skel.progs.ktstr_fentry_0.attach_trace(),
927        1 => skel.progs.ktstr_fentry_1.attach_trace(),
928        2 => skel.progs.ktstr_fentry_2.attach_trace(),
929        3 => skel.progs.ktstr_fentry_3.attach_trace(),
930        _ => return None,
931    })
932}
933
934/// Attach the fexit program in slot 0..=3 on the fentry skeleton.
935/// Sibling of [`attach_fentry_by_slot`]; see its doc for the routing
936/// rationale.
937fn attach_fexit_by_slot(
938    skel: &crate::bpf_skel::fentry::FentryProbeSkel<'_>,
939    slot: usize,
940) -> Option<libbpf_rs::Result<libbpf_rs::Link>> {
941    Some(match slot {
942        0 => skel.progs.ktstr_fexit_0.attach_trace(),
943        1 => skel.progs.ktstr_fexit_1.attach_trace(),
944        2 => skel.progs.ktstr_fexit_2.attach_trace(),
945        3 => skel.progs.ktstr_fexit_3.attach_trace(),
946        _ => return None,
947    })
948}
949
950/// Borrow the open fentry program in slot 0..=3 for pre-load
951/// configuration (`set_attach_target`, `set_autoload`).
952///
953/// Pre-load sibling of [`attach_fentry_by_slot`]: operates on
954/// [`OpenFentryProbeSkel`] before [`OpenSkel::load`] consumes it.
955/// Returns `None` for slot indices outside 0..=3, matching the
956/// existing `continue;` behaviour at call sites.
957///
958/// [`OpenFentryProbeSkel`]: crate::bpf_skel::fentry::OpenFentryProbeSkel
959/// [`OpenSkel::load`]: libbpf_rs::skel::OpenSkel::load
960fn fentry_prog_mut_by_slot<'a, 'obj>(
961    open_skel: &'a mut crate::bpf_skel::fentry::OpenFentryProbeSkel<'obj>,
962    slot: usize,
963) -> Option<&'a mut libbpf_rs::OpenProgramMut<'obj>> {
964    Some(match slot {
965        0 => &mut open_skel.progs.ktstr_fentry_0,
966        1 => &mut open_skel.progs.ktstr_fentry_1,
967        2 => &mut open_skel.progs.ktstr_fentry_2,
968        3 => &mut open_skel.progs.ktstr_fentry_3,
969        _ => return None,
970    })
971}
972
973/// Borrow the open fexit program in slot 0..=3 for pre-load
974/// configuration. Pre-load sibling of [`attach_fexit_by_slot`];
975/// see [`fentry_prog_mut_by_slot`] for the routing rationale.
976fn fexit_prog_mut_by_slot<'a, 'obj>(
977    open_skel: &'a mut crate::bpf_skel::fentry::OpenFentryProbeSkel<'obj>,
978    slot: usize,
979) -> Option<&'a mut libbpf_rs::OpenProgramMut<'obj>> {
980    Some(match slot {
981        0 => &mut open_skel.progs.ktstr_fexit_0,
982        1 => &mut open_skel.progs.ktstr_fexit_1,
983        2 => &mut open_skel.progs.ktstr_fexit_2,
984        3 => &mut open_skel.progs.ktstr_fexit_3,
985        _ => return None,
986    })
987}
988
989/// Disable autoload on both the fentry and fexit programs for
990/// slot 0..=3 so the verifier skips them at
991/// [`OpenSkel::load`][libbpf_rs::skel::OpenSkel::load]. Used for
992/// unused batch slots and slots whose
993/// [`set_attach_target`][libbpf_rs::OpenProgramMut::set_attach_target]
994/// call failed — either leaves the skeleton with placeholder
995/// targets the verifier would reject.
996///
997/// No-op for slot indices outside 0..=3.
998fn disable_slot_programs(
999    open_skel: &mut crate::bpf_skel::fentry::OpenFentryProbeSkel<'_>,
1000    slot: usize,
1001) {
1002    if let Some(p) = fentry_prog_mut_by_slot(open_skel, slot) {
1003        p.set_autoload(false);
1004    }
1005    if let Some(p) = fexit_prog_mut_by_slot(open_skel, slot) {
1006        p.set_autoload(false);
1007    }
1008}
1009
1010/// Write the per-slot rodata fields (`ktstr_fentry_func_idx_N`,
1011/// `ktstr_fentry_is_kernel_N`) for slot 0..=3. Mirrors the BPF
1012/// side's positional `rodata` layout in `src/bpf/fentry_probe.bpf.c`.
1013///
1014/// No-op for slot indices outside 0..=3.
1015fn set_rodata_slot(
1016    rodata: &mut crate::bpf_skel::fentry::types::rodata,
1017    slot: usize,
1018    idx: u32,
1019    is_kernel: bool,
1020) {
1021    let k = is_kernel as u8;
1022    match slot {
1023        0 => {
1024            rodata.ktstr_fentry_func_idx_0 = idx;
1025            rodata.ktstr_fentry_is_kernel_0 = k;
1026        }
1027        1 => {
1028            rodata.ktstr_fentry_func_idx_1 = idx;
1029            rodata.ktstr_fentry_is_kernel_1 = k;
1030        }
1031        2 => {
1032            rodata.ktstr_fentry_func_idx_2 = idx;
1033            rodata.ktstr_fentry_is_kernel_2 = k;
1034        }
1035        3 => {
1036            rodata.ktstr_fentry_func_idx_3 = idx;
1037            rodata.ktstr_fentry_is_kernel_3 = k;
1038        }
1039        _ => {}
1040    }
1041}
1042
1043/// Bounded retry count for the kernel-function fexit batch load.
1044///
1045/// On a heavily contended host the fexit `BPF_PROG_LOAD` can be
1046/// rejected with a transient error even for a privileged loader whose
1047/// same targets load cleanly on a quiet host — observed as an
1048/// intermittent "Permission denied (os error 13)" on the busiest
1049/// arm64 CI lanes, which dropped the exit-side counters and left the
1050/// failure dump a placeholder. os error 13 is EACCES (not EPERM):
1051/// the errno `BPF_PROG_LOAD` uses for a verifier rejection, so the
1052/// failure can also be deterministic for a given kernel/arch — on the
1053/// arm64 7.1 CI kernel the `enqueue_task_scx` fexit load fails all
1054/// three attempts in every observed run. Retrying the open+load a few
1055/// times with a short backoff — a cost bounded well inside the
1056/// probe's boot budget — recovers the transient case and costs ~200ms
1057/// in the deterministic one. `set_attach_target` is deterministic, so
1058/// a chunk with no valid target is never retried.
1059const KERNEL_FEXIT_LOAD_ATTEMPTS: u32 = 3;
1060
1061// More than one attempt is the whole point: it turns a transient load
1062// EACCES into a recovered load rather than an immediate placeholder dump.
1063const _: () = assert!(
1064    KERNEL_FEXIT_LOAD_ATTEMPTS >= 2,
1065    "the kernel-fexit batch load must retry at least once"
1066);
1067
1068/// Base backoff between kernel-fexit load attempts, scaled by the
1069/// zero-based attempt number (see [`kernel_fexit_load_backoff`]).
1070const KERNEL_FEXIT_LOAD_BACKOFF: std::time::Duration = std::time::Duration::from_millis(100);
1071
1072/// Backoff before load `attempt` (zero-based): no wait before the
1073/// first try, then a linear ramp (0, base, 2·base, …). Factored out so
1074/// the schedule is unit-testable without loading BPF.
1075fn kernel_fexit_load_backoff(attempt: u32) -> std::time::Duration {
1076    KERNEL_FEXIT_LOAD_BACKOFF * attempt
1077}
1078
1079/// Batch size for one fentry/fexit skeleton load — the skeleton's slot
1080/// count. Both attach phases chunk their targets by it.
1081const FENTRY_BATCH: usize = 4;
1082
1083/// One BPF-callback attach target within a batch. `ok` records whether
1084/// `set_attach_target` succeeded, so the load can disable the unused slots.
1085struct FentryTarget<'a> {
1086    slot: usize,
1087    fd: i32,
1088    idx: u32,
1089    name: &'a str,
1090    ok: bool,
1091}
1092
1093/// Attach fentry+fexit to every BPF callback in `valid_bpf`, in batches of
1094/// [`FENTRY_BATCH`] per skeleton load to keep the verifier passes down.
1095///
1096/// Callbacks are addressed by program FD plus a sentinel IP, unlike the
1097/// kernel half (`load_kernel_fexit_batch`), which targets vmlinux BTF with
1098/// fd 0 and a real IP. Both attach phases run this: Phase A when the
1099/// scheduler is already up, Phase B once it starts. `label` tags the log
1100/// lines with the caller's phase, same convention as
1101/// `load_kernel_fexit_batch`.
1102#[allow(clippy::too_many_arguments)]
1103fn attach_bpf_fentry_batches(
1104    skel: &crate::bpf_skel::ProbeSkel<'_>,
1105    valid_bpf: &[(u32, &StackFunction)],
1106    functions: &[StackFunction],
1107    btf_funcs: &[BtfFunc],
1108    bpf_prog_fds: &std::collections::HashMap<u32, i32>,
1109    cached_btf: Option<&std::sync::Arc<btf_rs::Btf>>,
1110    func_ips: &mut Vec<(u32, u64, String)>,
1111    fentry_links: &mut Vec<libbpf_rs::Link>,
1112    fexit_links: &mut Vec<libbpf_rs::Link>,
1113    diag: &mut ProbeDiagnostics,
1114    label: &str,
1115) {
1116    use libbpf_rs::MapCore;
1117    use libbpf_rs::MapFlags;
1118    use libbpf_rs::skel::{OpenSkel, SkelBuilder};
1119    for chunk in valid_bpf.chunks(FENTRY_BATCH) {
1120        let mut targets: Vec<FentryTarget<'_>> = Vec::new();
1121        for (slot, (idx, func)) in chunk.iter().enumerate() {
1122            let prog_id = func.bpf_prog_id.unwrap();
1123            let fd = if let Some(&pre_fd) = bpf_prog_fds.get(&prog_id) {
1124                let dup_fd = unsafe { libc::dup(pre_fd) };
1125                if dup_fd < 0 {
1126                    tracing::warn!(label, prog_id, func = %func.display_name, "fentry: dup failed");
1127                    diag.fentry_attach_failed.push((
1128                        func.display_name.clone(),
1129                        format!("dup(pre_fd={pre_fd}) failed"),
1130                    ));
1131                    continue;
1132                }
1133                dup_fd
1134            } else {
1135                let fd = unsafe { libbpf_rs::libbpf_sys::bpf_prog_get_fd_by_id(prog_id) };
1136                if fd < 0 {
1137                    tracing::warn!(label, prog_id, func = %func.display_name, "fentry: failed to get fd");
1138                    diag.fentry_attach_failed.push((
1139                        func.display_name.clone(),
1140                        format!("bpf_prog_get_fd_by_id({prog_id}) returned {fd}"),
1141                    ));
1142                    continue;
1143                }
1144                fd
1145            };
1146            targets.push(FentryTarget {
1147                slot,
1148                fd,
1149                idx: *idx,
1150                name: &func.display_name,
1151                ok: false,
1152            });
1153        }
1154        if targets.is_empty() {
1155            continue;
1156        }
1157
1158        use crate::bpf_skel::fentry::*;
1159        let mut fentry_open_obj = std::mem::MaybeUninit::uninit();
1160        let fentry_builder = FentryProbeSkelBuilder::default();
1161        let mut fentry_open = match fentry_builder.open(&mut fentry_open_obj) {
1162            Ok(s) => s,
1163            Err(e) => {
1164                tracing::warn!(label, %e, "fentry skeleton open failed");
1165                for t in &targets {
1166                    unsafe { libc::close(t.fd) };
1167                }
1168                continue;
1169            }
1170        };
1171
1172        // Set rodata: func_idx and is_kernel per slot.
1173        if let Some(rodata) = fentry_open.maps.rodata_data.as_mut() {
1174            rodata.ktstr_enabled = true;
1175            for t in &targets {
1176                // BPF-callback targets are never kernel-BTF targets; the
1177                // kernel half runs through `load_kernel_fexit_batch`.
1178                set_rodata_slot(rodata, t.slot, t.idx, false);
1179            }
1180        }
1181
1182        for t in targets.iter_mut() {
1183            // Set fentry attach target.
1184            let Some(fentry_prog) = fentry_prog_mut_by_slot(&mut fentry_open, t.slot) else {
1185                continue;
1186            };
1187            match fentry_prog.set_attach_target(t.fd, Some(t.name.to_string())) {
1188                Ok(()) => {
1189                    t.ok = true;
1190                    tracing::debug!(
1191                        label,
1192                        slot = t.slot,
1193                        func = t.name,
1194                        "fentry: set_attach_target ok"
1195                    );
1196                }
1197                Err(e) => {
1198                    tracing::warn!(label, slot = t.slot, func = t.name, %e, "fentry: set_attach_target failed");
1199                    diag.fentry_attach_failed
1200                        .push((t.name.to_string(), format!("set_attach_target: {e}")));
1201                    continue;
1202                }
1203            }
1204            // Set fexit attach target on the same function.
1205            let Some(fexit_prog) = fexit_prog_mut_by_slot(&mut fentry_open, t.slot) else {
1206                continue;
1207            };
1208            if let Err(e) = fexit_prog.set_attach_target(t.fd, Some(t.name.to_string())) {
1209                tracing::debug!(label, slot = t.slot, func = t.name, %e, "fexit: set_attach_target failed (entry-only)");
1210                // Disable autoload so the verifier doesn't reject the
1211                // skeleton due to a stale placeholder target.
1212                fexit_prog.set_autoload(false);
1213            }
1214        }
1215
1216        if !targets.iter().any(|t| t.ok) {
1217            for t in &targets {
1218                unsafe { libc::close(t.fd) };
1219            }
1220            continue;
1221        }
1222
1223        // Disable autoload on unused or failed fentry/fexit slots so the
1224        // verifier doesn't reject the placeholder target.
1225        let used_slots: std::collections::HashSet<usize> =
1226            targets.iter().filter(|t| t.ok).map(|t| t.slot).collect();
1227        for slot in 0..FENTRY_BATCH {
1228            if !used_slots.contains(&slot) {
1229                disable_slot_programs(&mut fentry_open, slot);
1230            }
1231        }
1232        tracing::debug!(
1233            label,
1234            active = used_slots.len(),
1235            disabled = FENTRY_BATCH - used_slots.len(),
1236            "fentry: loading batch",
1237        );
1238        // Reuse the main skeleton's maps so fentry events land in the
1239        // same probe_data map that the Rust side reads.
1240        use std::os::unix::io::AsFd;
1241        if let Err(e) = fentry_open
1242            .maps
1243            .probe_data
1244            .reuse_fd(skel.maps.probe_data.as_fd())
1245        {
1246            tracing::warn!(label, %e, "fentry: probe_data reuse_fd failed");
1247        }
1248        if let Err(e) = fentry_open
1249            .maps
1250            .func_meta_map
1251            .reuse_fd(skel.maps.func_meta_map.as_fd())
1252        {
1253            tracing::warn!(label, %e, "fentry: func_meta_map reuse_fd failed");
1254        }
1255
1256        let fentry_skel = match fentry_open.load() {
1257            Ok(s) => {
1258                tracing::debug!(label, "fentry: batch load success");
1259                for t in &targets {
1260                    unsafe { libc::close(t.fd) };
1261                }
1262                s
1263            }
1264            Err(e) => {
1265                tracing::warn!(label, %e, "fentry: batch load failed");
1266                for t in &targets {
1267                    if t.ok {
1268                        diag.fentry_attach_failed
1269                            .push((t.name.to_string(), format!("batch load: {e}")));
1270                    }
1271                    unsafe { libc::close(t.fd) };
1272                }
1273                continue;
1274            }
1275        };
1276
1277        // Populate func_meta and attach each slot.
1278        for t in &targets {
1279            if !t.ok {
1280                continue;
1281            }
1282
1283            let sentinel_ip = (t.idx as u64) | (1u64 << 63);
1284            let mut meta = crate::bpf_skel::types::func_meta {
1285                func_idx: t.idx,
1286                ..Default::default()
1287            };
1288
1289            if let Some(btf_func) = btf_funcs.iter().find(|f| f.name == t.name) {
1290                // Try vmlinux BTF first (for known struct params like
1291                // task_struct and auto-discovered vmlinux fields),
1292                // then BPF program BTF (for BPF-local types like task_ctx).
1293                let mut field_specs = match cached_btf {
1294                    Some(btf) => super::btf::resolve_field_specs_with_btf(btf_func, btf),
1295                    None => Vec::new(),
1296                };
1297                if field_specs.is_empty()
1298                    && let Some(prog_id) = functions
1299                        .iter()
1300                        .find(|f| f.display_name == t.name)
1301                        .and_then(|f| f.bpf_prog_id)
1302                {
1303                    field_specs = super::btf::resolve_bpf_field_specs(btf_func, prog_id);
1304                }
1305                populate_field_specs(&mut meta, &field_specs);
1306                meta.str_param_idx = detect_str_param(btf_func);
1307            }
1308
1309            let Some(result) = attach_fentry_by_slot(&fentry_skel, t.slot) else {
1310                continue;
1311            };
1312            let link = match result {
1313                Ok(link) => {
1314                    tracing::debug!(label, func = t.name, "fentry attached");
1315                    link
1316                }
1317                Err(e) => {
1318                    tracing::warn!(label, %e, func = t.name, "fentry attach failed");
1319                    diag.fentry_attach_failed
1320                        .push((t.name.to_string(), e.to_string()));
1321                    continue;
1322                }
1323            };
1324
1325            // func_meta_map.update + func_ips.push run AFTER the
1326            // fentry attach succeeds. Reversing the order would
1327            // orphan map entries and func_ip tuples on attach
1328            // failure — downstream reporting ("successfully
1329            // probed N funcs") would then show false-positive
1330            // successes for probes that never fired. If the
1331            // map update fails after the attach succeeded, drop
1332            // the Link (which detaches the program) so the
1333            // post-attach state matches what func_ips reports.
1334            let key_bytes = sentinel_ip.to_ne_bytes();
1335            let meta_bytes = unsafe {
1336                std::slice::from_raw_parts(
1337                    &meta as *const _ as *const u8,
1338                    std::mem::size_of::<crate::bpf_skel::types::func_meta>(),
1339                )
1340            };
1341            if let Err(e) = skel
1342                .maps
1343                .func_meta_map
1344                .update(&key_bytes, meta_bytes, MapFlags::ANY)
1345            {
1346                tracing::warn!(label, %e, func = t.name, "fentry: failed to update func_meta_map; dropping attached link");
1347                drop(link);
1348                continue;
1349            }
1350            fentry_links.push(link);
1351            func_ips.push((t.idx, sentinel_ip, t.name.to_string()));
1352            // Attach fexit for exit-side capture.
1353            let Some(fexit_result) = attach_fexit_by_slot(&fentry_skel, t.slot) else {
1354                continue;
1355            };
1356            match fexit_result {
1357                Ok(link) => {
1358                    tracing::debug!(label, func = t.name, "fexit attached");
1359                    fexit_links.push(link);
1360                }
1361                Err(e) => {
1362                    tracing::debug!(label, %e, func = t.name, "fexit attach failed (entry-only)");
1363                }
1364            }
1365        }
1366
1367        drop(fentry_skel);
1368    }
1369}
1370
1371/// Open, configure, load, and attach one batch (`chunk`) of
1372/// kernel-function fexit targets against vmlinux BTF (fd 0), retrying
1373/// the load [`KERNEL_FEXIT_LOAD_ATTEMPTS`] times on a transient
1374/// failure.
1375///
1376/// `chunk` is `(func_idx, func_name)` pairs, at most `batch` long;
1377/// `batch` is the skeleton's fexit slot count. `probe_data_fd` and
1378/// `func_meta_fd` are the main skeleton's maps, reused so the fexit
1379/// programs write into the shared maps. Attached links are pushed onto
1380/// `fexit_links`; `label` tags the log lines with the caller's phase.
1381///
1382/// Returns `Ok(())` once a load succeeds (individual attach failures
1383/// are logged, not fatal) or when the chunk had no valid attach target
1384/// (a deterministic outcome that is not retried). Returns
1385/// `Err((error, targeted_names))` when every attempt failed so the
1386/// caller can surface the error and record the affected functions.
1387fn load_kernel_fexit_batch(
1388    chunk: &[(u32, &str)],
1389    batch: usize,
1390    probe_data_fd: std::os::unix::io::BorrowedFd<'_>,
1391    func_meta_fd: std::os::unix::io::BorrowedFd<'_>,
1392    fexit_links: &mut Vec<libbpf_rs::Link>,
1393    label: &str,
1394) -> Result<(), (libbpf_rs::Error, Vec<String>)> {
1395    use crate::bpf_skel::fentry::*;
1396    use libbpf_rs::skel::{OpenSkel, SkelBuilder};
1397
1398    let mut load_err: Option<libbpf_rs::Error> = None;
1399
1400    for attempt in 0..KERNEL_FEXIT_LOAD_ATTEMPTS {
1401        if attempt > 0 {
1402            std::thread::sleep(kernel_fexit_load_backoff(attempt));
1403        }
1404
1405        // `.load()` consumes the open skeleton, so the whole open must
1406        // be rebuilt each attempt; the obj storage lives for this
1407        // iteration only.
1408        let mut fentry_open_obj = std::mem::MaybeUninit::uninit();
1409        let mut fentry_open = match FentryProbeSkelBuilder::default().open(&mut fentry_open_obj) {
1410            Ok(s) => s,
1411            Err(e) => {
1412                tracing::warn!(%e, label, "kernel fexit skeleton open failed");
1413                load_err = Some(e);
1414                continue;
1415            }
1416        };
1417
1418        if let Some(rodata) = fentry_open.maps.rodata_data.as_mut() {
1419            rodata.ktstr_enabled = true;
1420            for (slot, &(idx, _)) in chunk.iter().enumerate() {
1421                set_rodata_slot(rodata, slot, idx, true);
1422            }
1423        }
1424
1425        // Only fexit programs are needed here (entry capture is the
1426        // kprobe skeleton's job): disable the fentry half and point
1427        // each fexit program at its kernel function.
1428        let mut ok = vec![false; chunk.len()];
1429        for (slot, &(_, name)) in chunk.iter().enumerate() {
1430            let Some(fentry_prog) = fentry_prog_mut_by_slot(&mut fentry_open, slot) else {
1431                continue;
1432            };
1433            fentry_prog.set_autoload(false);
1434
1435            let Some(fexit_prog) = fexit_prog_mut_by_slot(&mut fentry_open, slot) else {
1436                continue;
1437            };
1438            match fexit_prog.set_attach_target(0, Some(name.to_string())) {
1439                Ok(()) => {
1440                    ok[slot] = true;
1441                    tracing::debug!(
1442                        slot,
1443                        func = name,
1444                        label,
1445                        "kernel fexit: set_attach_target ok"
1446                    );
1447                }
1448                Err(e) => {
1449                    tracing::debug!(slot, func = name, %e, label, "kernel fexit: set_attach_target failed");
1450                    fexit_prog.set_autoload(false);
1451                }
1452            }
1453        }
1454
1455        if !ok.iter().any(|&v| v) {
1456            // No valid attach target in this chunk; retrying cannot
1457            // change a deterministic set_attach_target outcome.
1458            return Ok(());
1459        }
1460
1461        // Disable both halves of every unused slot. A generic
1462        // `SEC("fentry")` program without an attach target is not a
1463        // valid load target on all kernels; leaving an unused entry
1464        // half enabled can reject the whole short final batch with
1465        // EINVAL even though every used fexit target is valid.
1466        for slot in 0..batch {
1467            if !ok.get(slot).copied().unwrap_or(false) {
1468                disable_slot_programs(&mut fentry_open, slot);
1469            }
1470        }
1471
1472        if let Err(e) = fentry_open.maps.probe_data.reuse_fd(probe_data_fd) {
1473            tracing::warn!(%e, label, "kernel fexit: probe_data reuse_fd failed");
1474        }
1475        if let Err(e) = fentry_open.maps.func_meta_map.reuse_fd(func_meta_fd) {
1476            tracing::warn!(%e, label, "kernel fexit: func_meta_map reuse_fd failed");
1477        }
1478
1479        let fentry_skel = match fentry_open.load() {
1480            Ok(s) => s,
1481            Err(e) => {
1482                tracing::debug!(%e, attempt, label, "kernel fexit: batch load attempt failed");
1483                load_err = Some(e);
1484                continue;
1485            }
1486        };
1487
1488        for (slot, &(_, name)) in chunk.iter().enumerate() {
1489            if !ok[slot] {
1490                continue;
1491            }
1492            let Some(result) = attach_fexit_by_slot(&fentry_skel, slot) else {
1493                continue;
1494            };
1495            match result {
1496                Ok(link) => {
1497                    tracing::debug!(func = name, label, "kernel fexit attached");
1498                    fexit_links.push(link);
1499                }
1500                Err(e) => {
1501                    tracing::debug!(%e, func = name, label, "kernel fexit attach failed");
1502                }
1503            }
1504        }
1505
1506        drop(fentry_skel);
1507        return Ok(());
1508    }
1509
1510    let targeted: Vec<String> = chunk.iter().map(|&(_, name)| name.to_string()).collect();
1511    Err((
1512        load_err.expect("load_err is Some whenever every load attempt failed"),
1513        targeted,
1514    ))
1515}
1516
1517/// Causal-task filter for the trigger event's `task_ptr` (sourced
1518/// from `args[0]`).
1519///
1520/// The selected BPF scheduler-exit handler sets `args[0]` to
1521/// `bpf_get_current_task()` only for `SCX_EXIT_ERROR_BPF` (where
1522/// `current` IS the causal task); for `SCX_EXIT_ERROR` (kworker /
1523/// sysrq context) it emits `args[0] == 0`. A zero `task_ptr` means
1524/// "no causal task" and must suppress probe stitching, so this
1525/// returns `None` for 0 and `Some(p)` otherwise. `run_probe_skeleton`
1526/// calls this on the last trigger event; the `args0_*` tests pin both
1527/// sides of the contract against this exact predicate.
1528fn causal_tptr(task_ptr: u64) -> Option<u64> {
1529    Some(task_ptr).filter(|&p| p != 0)
1530}
1531
1532/// Run the BPF probe skeleton for auto-repro.
1533///
1534/// Operates in two modes depending on `phase_b_rx`:
1535///
1536/// **Single-phase (`phase_b_rx = None`):** loads the kprobe skeleton
1537/// and fentry/fexit skeleton together, attaches all probes, then
1538/// polls until the trigger fires.
1539///
1540/// **Two-phase (`phase_b_rx = Some`):** Phase A attaches kprobes +
1541/// kernel fexit + the kernel-selected exit trigger before the
1542/// scheduler starts, signals `ready`, then polls the ring buffer
1543/// while waiting for Phase B input via the channel. When Phase B
1544/// input arrives, attaches fentry/fexit to BPF struct_ops callbacks
1545/// and additional kprobes for kernel callers. Signals `done` on the
1546/// `PhaseBInput` when Phase B attachment completes. If the trigger
1547/// fires before Phase B input arrives, fentry is skipped — the
1548/// crash happened before BPF programs could be probed.
1549///
1550/// Newest kernels trigger directly after a successful `scx_claim_exit()` via
1551/// `tp_btf/sched_ext_exit`; the preceding generation triggers from a raw
1552/// entry/return pair around `scx_vexit()`; global-era kernels trigger on their
1553/// filtered two-argument `scx_dump_state()` error entry. If no compatible
1554/// target is available, auto-repro fails closed before object load.
1555///
1556/// Returns accumulated func_names from both phases as the third
1557/// tuple element.
1558pub fn run_probe_skeleton(
1559    functions: &[StackFunction],
1560    btf_funcs: &[BtfFunc],
1561    stop: &AtomicBool,
1562    bpf_prog_fds: &std::collections::HashMap<u32, i32>,
1563    ready: &Latch,
1564    phase_a_status: &std::sync::Mutex<Option<Result<(), String>>>,
1565    phase_b_rx: Option<std::sync::mpsc::Receiver<PhaseBInput>>,
1566) -> (
1567    Option<Vec<ProbeEvent>>,
1568    ProbeDiagnostics,
1569    Vec<(u32, String)>,
1570) {
1571    use crate::bpf_skel::*;
1572    use libbpf_rs::skel::{OpenSkel, SkelBuilder};
1573    use libbpf_rs::{Link, MapCore, MapFlags, RingBufferBuilder};
1574
1575    tracing::debug!(n = functions.len(), "run_probe_skeleton");
1576
1577    let publish_phase_a = |status: Result<(), String>| {
1578        let mut slot = phase_a_status
1579            .lock()
1580            .unwrap_or_else(std::sync::PoisonError::into_inner);
1581        if slot.is_none() {
1582            *slot = Some(status);
1583        }
1584        drop(slot);
1585        ready.set();
1586    };
1587
1588    let mut diag = ProbeDiagnostics::default();
1589
1590    // Capture libbpf's own log (verifier rejection reasons, "load
1591    // failed: -E..." lines) into a buffer so a skeleton-load failure
1592    // surfaces its ROOT CAUSE through `diag.trigger_attach_error`
1593    // rather than the bare errno. The guest's stderr is not forwarded
1594    // to the host, so without this the pipeline reports a one-line
1595    // "Permission denied" for a verifier rejection whose actual reason
1596    // (e.g. a bad program prototype) lives only in the swallowed log.
1597    // `set_print` is process-global; the buffer is cleared at entry so
1598    // each load attempt starts fresh, and only its tail is read back
1599    // on failure below.
1600    static LIBBPF_LOG: std::sync::Mutex<String> = std::sync::Mutex::new(String::new());
1601    fn libbpf_log_capture(_lvl: libbpf_rs::PrintLevel, msg: String) {
1602        if let Ok(mut b) = LIBBPF_LOG.lock() {
1603            b.push_str(&msg);
1604        }
1605    }
1606    libbpf_rs::set_print(Some((libbpf_rs::PrintLevel::Debug, libbpf_log_capture)));
1607    if let Ok(mut b) = LIBBPF_LOG.lock() {
1608        b.clear();
1609    }
1610
1611    // Load vmlinux BTF once and use it for both trigger selection and
1612    // kprobe metadata below. All trigger programs live in probe.o; every
1613    // unselected program must have autoload disabled before libbpf resolves
1614    // its attach target.
1615    let cached_btf = crate::monitor::btf_offsets::cached_vmlinux_btf();
1616    let exit_trigger = cached_btf
1617        .as_deref()
1618        .and_then(|btf| {
1619            select_scheduler_exit_trigger(
1620                btf_has_typed_tracepoint_arity(btf, "sched_ext_exit", 3),
1621                btf_has_function(btf, "scx_vexit"),
1622                btf_has_function_arity(btf, "scx_dump_state", 2),
1623            )
1624        })
1625        .ok_or_else(|| {
1626            "kernel BTF exposes no compatible sched_ext exit target \
1627             (`sched_ext_exit`, `scx_vexit`, or two-argument \
1628             `scx_dump_state`)"
1629                .to_string()
1630        });
1631    let exit_trigger = match exit_trigger {
1632        Ok(trigger) => trigger,
1633        Err(error) => {
1634            diag.trigger_attach_error = Some(error.clone());
1635            publish_phase_a(Err(error));
1636            return (None, diag, Vec::new());
1637        }
1638    };
1639
1640    // Open skeleton. Two MaybeUninit slots: the first backs the
1641    // initial load attempt; the second backs the fallback retry when
1642    // optional programs cause ESRCH. Both must outlive `skel`.
1643    let mut open_object = std::mem::MaybeUninit::uninit();
1644    let mut open_object_fallback = std::mem::MaybeUninit::uninit();
1645    let builder = ProbeSkelBuilder::default();
1646    let mut open_skel = match builder.open(&mut open_object) {
1647        Ok(s) => s,
1648        Err(e) => {
1649            tracing::error!(%e, "probe skeleton open failed");
1650            diag.trigger_attach_error = Some(format!("skeleton open: {e}"));
1651            publish_phase_a(Err(format!("probe skeleton open failed: {e}")));
1652            return (None, diag, Vec::new());
1653        }
1654    };
1655
1656    // Enable probes (must set before load — rodata is immutable after)
1657    if let Some(rodata) = open_skel.maps.rodata_data.as_mut() {
1658        rodata.ktstr_enabled = true;
1659    }
1660    let trigger_programs = scheduler_exit_program_selection(exit_trigger);
1661    open_skel
1662        .progs
1663        .ktstr_trigger_tp
1664        .set_autoload(trigger_programs.tracepoint);
1665    open_skel
1666        .progs
1667        .ktstr_trigger_vexit_enter
1668        .set_autoload(trigger_programs.vexit_enter);
1669    open_skel
1670        .progs
1671        .ktstr_trigger_vexit_return
1672        .set_autoload(trigger_programs.vexit_return);
1673    open_skel
1674        .progs
1675        .ktstr_trigger_dump_fentry
1676        .set_autoload(trigger_programs.dump_fentry);
1677
1678    // Load skeleton. Try with all programs first; if a missing optional
1679    // typed-BTF target causes ESRCH, re-open with optional programs disabled.
1680    // The fallback unconditionally fires on any load error, not
1681    // strictly ESRCH — libbpf doesn't surface a stable errno through
1682    // its Error type, so an exact ESRCH match is brittle. The retry
1683    // is cheap (re-open + load with autoload disabled on the optional
1684    // set), so the broader gate is acceptable. When the retry ALSO
1685    // fails, we surface BOTH errors so the operator sees the original
1686    // root cause alongside the retry failure — a verifier rejection
1687    // on a non-optional program would otherwise be masked by the
1688    // retry's unrelated error.
1689    let (skel, optional_programs_loaded) = match open_skel.load() {
1690        Ok(s) => (s, true),
1691        Err(first_err) => {
1692            tracing::debug!(
1693                %first_err,
1694                "probe skeleton load failed with all programs; \
1695                 retrying with optional programs disabled"
1696            );
1697            let builder2 = ProbeSkelBuilder::default();
1698            let mut open_skel2 = match builder2.open(&mut open_object_fallback) {
1699                Ok(s) => s,
1700                Err(e) => {
1701                    tracing::error!(%e, "probe skeleton re-open failed");
1702                    diag.trigger_attach_error = Some(format!(
1703                        "skeleton open (retry): {e}; original load error: {first_err}"
1704                    ));
1705                    publish_phase_a(Err(format!(
1706                        "probe skeleton re-open failed: {e}; original load error: {first_err}"
1707                    )));
1708                    return (None, diag, Vec::new());
1709                }
1710            };
1711            if let Some(rodata) = open_skel2.maps.rodata_data.as_mut() {
1712                rodata.ktstr_enabled = true;
1713            }
1714            let trigger_programs = scheduler_exit_program_selection(exit_trigger);
1715            open_skel2
1716                .progs
1717                .ktstr_trigger_tp
1718                .set_autoload(trigger_programs.tracepoint);
1719            open_skel2
1720                .progs
1721                .ktstr_trigger_vexit_enter
1722                .set_autoload(trigger_programs.vexit_enter);
1723            open_skel2
1724                .progs
1725                .ktstr_trigger_vexit_return
1726                .set_autoload(trigger_programs.vexit_return);
1727            open_skel2
1728                .progs
1729                .ktstr_trigger_dump_fentry
1730                .set_autoload(trigger_programs.dump_fentry);
1731            open_skel2.progs.ktstr_pi_fentry.set_autoload(false);
1732            open_skel2.progs.ktstr_pi_fexit.set_autoload(false);
1733            open_skel2.progs.ktstr_lock_contend.set_autoload(false);
1734            open_skel2.progs.ktstr_tl_switch.set_autoload(false);
1735            open_skel2.progs.ktstr_tl_migrate.set_autoload(false);
1736            open_skel2.progs.ktstr_tl_wakeup.set_autoload(false);
1737            open_skel2
1738                .progs
1739                .ktstr_preempt_disable_tp
1740                .set_autoload(false);
1741            open_skel2.progs.ktstr_preempt_enable_tp.set_autoload(false);
1742            match open_skel2.load() {
1743                Ok(s) => (s, false),
1744                Err(e) => {
1745                    tracing::error!(
1746                        %e, %first_err,
1747                        "probe skeleton load failed (retry); \
1748                         surfacing both original and retry errors"
1749                    );
1750                    let libbpf_log = LIBBPF_LOG
1751                        .lock()
1752                        .map(|b| {
1753                            // Keep the tail: the verifier's rejection
1754                            // reason and the "load failed: -E..." line
1755                            // land at the end of the log. Slice on bytes
1756                            // and lossily reconstitute so a cut through a
1757                            // multi-byte char cannot panic.
1758                            let bytes = b.as_bytes();
1759                            let start = bytes.len().saturating_sub(8000);
1760                            String::from_utf8_lossy(&bytes[start..]).into_owned()
1761                        })
1762                        .unwrap_or_default();
1763                    diag.trigger_attach_error = Some(format!(
1764                        "skeleton load (retry): {e}; original error before retry: {first_err}; \
1765                         LIBBPF-LOG>>>\n{libbpf_log}\n<<<LIBBPF-LOG"
1766                    ));
1767                    let libbpf_tail = bounded_libbpf_tail(&libbpf_log);
1768                    publish_phase_a(Err(format!(
1769                        "probe skeleton load failed after minimal retry: {e}; \
1770                         original load error: {first_err}; \
1771                         libbpf tail: {libbpf_tail}"
1772                    )));
1773                    return (None, diag, Vec::new());
1774                }
1775            }
1776        }
1777    };
1778
1779    // Populate func_meta_map with function IPs and metadata
1780    let mut func_ips: Vec<(u32, u64, String)> = Vec::new(); // (idx, ip, display_name)
1781    let mut bpf_funcs: Vec<(u32, &StackFunction)> = Vec::new(); // BPF functions for fentry
1782
1783    // Load vmlinux BTF once and reuse across every kprobe meta
1784    // population in the loop below. The previous code called
1785    // `resolve_field_specs(_, None)` per function, which re-parsed
1786    // the multi-MB vmlinux BTF on every iteration (>1 s per kernel
1787    // with thousands of kprobed functions). Loading once turns the
1788    // hot path into pure type lookups against a borrowed handle.
1789    // `None` (load failure) leaves `cached_btf` empty and downstream
1790    // call sites fall back to the no-BTF path — same behaviour as
1791    // the previous per-call `Err(...) -> Vec::new()` branch.
1792    //
1793    // `cached_vmlinux_btf` memoises the first successful parse
1794    // process-wide so repeated auto-repro cycles in the same nextest
1795    // process share one `Arc<Btf>` instead of re-reading and
1796    // re-parsing the multi-MB blob each time.
1797    for (idx, func) in functions.iter().enumerate() {
1798        if func.is_bpf {
1799            bpf_funcs.push((idx as u32, func));
1800            continue;
1801        }
1802        let ip = match resolve_func_ip(&func.raw_name) {
1803            Some(ip) => ip,
1804            None => {
1805                tracing::warn!(func = %func.raw_name, "could not resolve function IP");
1806                diag.kprobe_resolve_failed.push(func.raw_name.clone());
1807                continue;
1808            }
1809        };
1810
1811        let mut meta = types::func_meta {
1812            func_idx: idx as u32,
1813            ..Default::default()
1814        };
1815
1816        // Populate field specs from BTF-resolved offsets.
1817        if let Some(btf_func) = btf_funcs.iter().find(|f| f.name == func.raw_name) {
1818            let field_specs = match cached_btf.as_ref() {
1819                Some(btf) => super::btf::resolve_field_specs_with_btf(btf_func, btf),
1820                None => Vec::new(),
1821            };
1822            populate_field_specs(&mut meta, &field_specs);
1823            // Detect char * params for string capture.
1824            meta.str_param_idx = detect_str_param(btf_func);
1825        }
1826
1827        let key_bytes = ip.to_ne_bytes();
1828        let meta_bytes = unsafe {
1829            std::slice::from_raw_parts(
1830                &meta as *const _ as *const u8,
1831                std::mem::size_of::<types::func_meta>(),
1832            )
1833        };
1834
1835        if let Err(e) = skel
1836            .maps
1837            .func_meta_map
1838            .update(&key_bytes, meta_bytes, MapFlags::ANY)
1839        {
1840            tracing::warn!(%e, func = %func.raw_name, "failed to update func_meta_map");
1841            continue;
1842        }
1843
1844        tracing::debug!(func = %func.raw_name, ip, nr = meta.nr_field_specs, "kprobe meta");
1845        diag.kprobe_resolved += 1;
1846        func_ips.push((idx as u32, ip, func.display_name.clone()));
1847    }
1848
1849    if func_ips.is_empty() && bpf_funcs.is_empty() && phase_b_rx.is_none() {
1850        tracing::warn!("no kprobe IPs resolved and no BPF functions for fentry");
1851        diag.trigger_attach_error =
1852            Some("no functions resolved — kprobes and trigger skipped".to_string());
1853        publish_phase_a(Err("no probe functions resolved during Phase A".to_string()));
1854        return (None, diag, Vec::new());
1855    }
1856    if func_ips.is_empty() && (phase_b_rx.is_some() || !bpf_funcs.is_empty()) {
1857        tracing::debug!("no kernel functions resolved to IPs, proceeding with fentry only");
1858    }
1859
1860    // Attach kprobes to each function for entry capture. Exit capture
1861    // for kernel functions uses fexit via the fentry skeleton (batched
1862    // separately below with fd=0 for vmlinux BTF).
1863    //
1864    // Parallelised via [`parallel_attach_kprobes`]: each attach is an
1865    // independent `perf_event_open` + `BPF_LINK_CREATE` syscall pair,
1866    // and the sequential loop's round-trip cost dominated the auto-
1867    // repro Phase A setup time when the crash backtrace named many
1868    // functions. Worker pool runs them concurrently; results land
1869    // back in the original input order so the post-attach
1870    // `kprobe_attach_failed` / `links` shape matches the prior loop.
1871    //
1872    // At most one kprobe per address, tracked in `probed_ips` and carried
1873    // into Phase B. A second kprobe on an address is pure redundancy: the
1874    // kernel folds both into an aggrprobe whose members run the same
1875    // `ktstr_probe` program and resolve the same IP-keyed `func_meta`, so
1876    // the duplicate yields no extra data. Folding is also what keeps the
1877    // address at a single probepoint, so the cost is a second run of that
1878    // one handler: the argument and field reads and the `probe_data`
1879    // update repeat, and the per-CPU counters behind `bpf_kprobe_fires` /
1880    // `bpf_kprobe_returns` are bumped twice per hit. Phase B derives
1881    // kernel bridge callers from the scheduler's BPF ops and readily
1882    // re-names a function Phase A already probed.
1883    let mut links: Vec<(Link, String)> = Vec::new();
1884    let mut probed_ips: HashSet<u64> = HashSet::new();
1885    let mut attach_input: Vec<String> = Vec::new();
1886    let mut attach_ips: Vec<u64> = Vec::new();
1887    for (idx, ip, _) in &func_ips {
1888        let raw = &functions[*idx as usize].raw_name;
1889        if !probed_ips.insert(*ip) {
1890            tracing::debug!(func = %raw, ip, "kprobe already planned for ip, skipping");
1891            continue;
1892        }
1893        attach_input.push(raw.clone());
1894        attach_ips.push(*ip);
1895    }
1896    for ((raw, result), ip) in parallel_attach_kprobes(&skel.progs.ktstr_probe, &attach_input)
1897        .into_iter()
1898        .zip(attach_ips)
1899    {
1900        match result {
1901            Ok(link) => {
1902                links.push((link, raw));
1903            }
1904            Err(e) => {
1905                // Nothing is planted at this address, so leave it open
1906                // for Phase B to try again.
1907                probed_ips.remove(&ip);
1908                tracing::warn!(%e, func = %raw, "kprobe attach failed");
1909                diag.kprobe_attach_failed.push((raw, e.to_string()));
1910            }
1911        }
1912    }
1913    diag.kprobe_attached = links.len() as u32;
1914    tracing::debug!(attached = links.len(), total = func_ips.len(), "kprobes");
1915
1916    // Attach fentry+fexit for BPF callbacks and kernel functions.
1917    // Batched in groups of FENTRY_BATCH per skeleton load to reduce
1918    // verifier passes. BPF callbacks use prog FD + sentinel IP.
1919    // Kernel functions use fd=0 (vmlinux BTF) + real IP.
1920    let mut fentry_links: Vec<Link> = Vec::new();
1921    let mut fexit_links: Vec<Link> = Vec::new();
1922
1923    // Build combined list of targets: BPF callbacks + kernel functions.
1924    let valid_bpf: Vec<(u32, &StackFunction)> = bpf_funcs
1925        .iter()
1926        .filter(|(_, f)| f.bpf_prog_id.is_some())
1927        .copied()
1928        .collect();
1929    diag.fentry_candidates = valid_bpf.len() as u32;
1930
1931    // Kernel functions that were attached via kprobe also get fentry+fexit
1932    // for exit capture. fd=0 targets vmlinux BTF.
1933    struct KernelFentryTarget {
1934        idx: u32,
1935        name: String,
1936    }
1937    let kernel_fexit_targets: Vec<KernelFentryTarget> = func_ips
1938        .iter()
1939        .map(|(idx, _, name)| KernelFentryTarget {
1940            idx: *idx,
1941            name: name.clone(),
1942        })
1943        .collect();
1944
1945    // Interleave: process BPF targets first, then kernel targets.
1946    // Each gets batched into the fentry skeleton in groups of 4.
1947    //
1948    // When phase_b_rx is Some (Phase A/B split), BPF callback fentry
1949    // attachment is deferred to Phase B after the scheduler starts.
1950    // Only kernel fexit (fd=0) and kprobes run in Phase A.
1951
1952    // --- BPF callback batches (skip in Phase A when split is active) ---
1953    if phase_b_rx.is_none() {
1954        attach_bpf_fentry_batches(
1955            &skel,
1956            &valid_bpf,
1957            functions,
1958            btf_funcs,
1959            bpf_prog_fds,
1960            cached_btf.as_ref(),
1961            &mut func_ips,
1962            &mut fentry_links,
1963            &mut fexit_links,
1964            &mut diag,
1965            "fentry",
1966        );
1967        diag.fentry_attached = fentry_links.len() as u32;
1968        if !valid_bpf.is_empty() {
1969            tracing::debug!(
1970                fentry = fentry_links.len(),
1971                fexit = fexit_links.len(),
1972                total = valid_bpf.len(),
1973                "BPF probes",
1974            );
1975        }
1976    }
1977    // --- Kernel function fexit batches (fd=0 = vmlinux BTF) ---
1978    use std::os::unix::io::AsFd;
1979    for chunk in kernel_fexit_targets.chunks(FENTRY_BATCH) {
1980        let names: Vec<(u32, &str)> = chunk.iter().map(|kt| (kt.idx, kt.name.as_str())).collect();
1981        if let Err((e, targeted)) = load_kernel_fexit_batch(
1982            &names,
1983            FENTRY_BATCH,
1984            skel.maps.probe_data.as_fd(),
1985            skel.maps.func_meta_map.as_fd(),
1986            &mut fexit_links,
1987            "kernel fexit",
1988        ) {
1989            tracing::warn!(
1990                %e,
1991                kind = ?e.kind(),
1992                attempts = KERNEL_FEXIT_LOAD_ATTEMPTS,
1993                targets = ?targeted,
1994                "kernel fexit: batch load failed",
1995            );
1996            for name in targeted {
1997                diag.fentry_attach_failed
1998                    .push((name, format!("batch load: {e}")));
1999            }
2000        }
2001    }
2002    if !kernel_fexit_targets.is_empty() {
2003        tracing::debug!(
2004            fexit = fexit_links.len(),
2005            total = kernel_fexit_targets.len(),
2006            "kernel fexit probes",
2007        );
2008    }
2009
2010    // Attach the selected trigger program(s). The newest tracepoint is emitted
2011    // only for a successful claim, the preceding generation's raw return probe
2012    // rejects losing concurrent claims, and the global-era entry hook rechecks
2013    // scx_exit_info.kind to reject the SysRq-D caller.
2014    let trigger_attach: Result<Vec<(Link, String)>, String> = match exit_trigger {
2015        SchedulerExitTrigger::SchedExtExitTracepoint => skel
2016            .progs
2017            .ktstr_trigger_tp
2018            .attach_trace()
2019            .map(|link| vec![(link, "tp_btf/sched_ext_exit".to_string())])
2020            .map_err(|e| e.to_string()),
2021        SchedulerExitTrigger::ScxVexitKprobePair => (|| {
2022            let enter = skel
2023                .progs
2024                .ktstr_trigger_vexit_enter
2025                .attach_kprobe(false, "scx_vexit")
2026                .map_err(|e| format!("entry kprobe: {e}"))?;
2027            let ret = skel
2028                .progs
2029                .ktstr_trigger_vexit_return
2030                .attach_kprobe(true, "scx_vexit")
2031                .map_err(|e| format!("return kprobe: {e}"))?;
2032            Ok(vec![
2033                (enter, "kprobe/scx_vexit".to_string()),
2034                (ret, "kretprobe/scx_vexit".to_string()),
2035            ])
2036        })(),
2037        SchedulerExitTrigger::ScxDumpState => skel
2038            .progs
2039            .ktstr_trigger_dump_fentry
2040            .attach_trace()
2041            .map(|link| vec![(link, "fentry/scx_dump_state".to_string())])
2042            .map_err(|e| e.to_string()),
2043    };
2044    match trigger_attach {
2045        Ok(trigger_links) => {
2046            tracing::debug!(
2047                trigger = exit_trigger.section(),
2048                "scheduler-exit trigger attached"
2049            );
2050            diag.trigger_type = exit_trigger.diagnostic_type().to_string();
2051            links.extend(trigger_links);
2052        }
2053        Err(e) => {
2054            let msg = format!(
2055                "auto-repro could not attach selected scheduler-exit trigger {}: {e}",
2056                exit_trigger.section()
2057            );
2058            tracing::error!(%msg, "trigger attach failed");
2059            diag.trigger_attach_error = Some(msg.clone());
2060            publish_phase_a(Err(format!(
2061                "probe trigger attach failed during Phase A: {msg}"
2062            )));
2063            return (None, diag, Vec::new());
2064        }
2065    }
2066
2067    // Attach timeline programs (loaded by the skeleton but not
2068    // auto-attached — they need explicit attach_trace calls).
2069    if optional_programs_loaded {
2070        match skel.progs.ktstr_tl_switch.attach_trace() {
2071            Ok(link) => links.push((link, "tp_btf/sched_switch".to_string())),
2072            Err(e) => tracing::warn!(%e, "timeline sched_switch attach failed"),
2073        }
2074        match skel.progs.ktstr_tl_migrate.attach_trace() {
2075            Ok(link) => links.push((link, "tp_btf/sched_migrate_task".to_string())),
2076            Err(e) => tracing::warn!(%e, "timeline sched_migrate_task attach failed"),
2077        }
2078        match skel.progs.ktstr_tl_wakeup.attach_trace() {
2079            Ok(link) => links.push((link, "tp_btf/sched_wakeup".to_string())),
2080            Err(e) => tracing::warn!(%e, "timeline sched_wakeup attach failed"),
2081        }
2082    }
2083
2084    // Attach optional programs. When the first load succeeded (all
2085    // targets present), they were loaded and just need attachment.
2086    // When the fallback path ran, they were not loaded — attach
2087    // returns an error that we silently absorb.
2088    if optional_programs_loaded {
2089        if let Ok(link) = skel.progs.ktstr_pi_fentry.attach_trace() {
2090            links.push((link, "fentry/rt_mutex_setprio".to_string()));
2091        }
2092        if let Ok(link) = skel.progs.ktstr_pi_fexit.attach_trace() {
2093            links.push((link, "fexit/rt_mutex_setprio".to_string()));
2094        }
2095        if let Ok(link) = skel.progs.ktstr_lock_contend.attach_trace() {
2096            links.push((link, "tp_btf/contention_begin".to_string()));
2097        }
2098        if let Ok(link) = skel.progs.ktstr_preempt_disable_tp.attach_trace() {
2099            links.push((link, "tp_btf/preempt_disable".to_string()));
2100        }
2101        if let Ok(link) = skel.progs.ktstr_preempt_enable_tp.attach_trace() {
2102            links.push((link, "tp_btf/preempt_enable".to_string()));
2103        }
2104    }
2105
2106    // Set up ring buffer
2107    let events: std::sync::Arc<std::sync::Mutex<Vec<ProbeEvent>>> =
2108        std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
2109    let events_clone = events.clone();
2110    let triggered = std::sync::Arc::new(AtomicBool::new(false));
2111    let triggered_clone = triggered.clone();
2112
2113    // Ring buffer event layout matching probe_event in intf.h.
2114    // `str_val` + `has_str` + `str_param_idx` are kept in the wire
2115    // layout for ABI symmetry with `struct probe_entry` (the
2116    // kprobe-side hash map shares the field names) — the
2117    // EVENT_TRIGGER producer leaves them zeroed.
2118    #[repr(C)]
2119    struct RbEvent {
2120        type_: u32,
2121        tid: u32,
2122        func_idx: u32,
2123        ts: u64,
2124        args: [u64; 6],
2125        fields: [u64; 16],
2126        nr_fields: u32,
2127        kstack: [u64; 32],
2128        kstack_sz: u32,
2129        str_val: [u8; MAX_STR_LEN],
2130        has_str: u8,
2131        str_param_idx: u8,
2132    }
2133
2134    let mut rb_builder = RingBufferBuilder::new();
2135    if let Err(e) = rb_builder.add(&skel.maps.ktstr_events, move |data: &[u8]| {
2136        if data.len() < std::mem::size_of::<RbEvent>() {
2137            return 0;
2138        }
2139        let raw: &RbEvent = unsafe { &*(data.as_ptr() as *const RbEvent) };
2140
2141        if raw.type_ == EVENT_TRIGGER {
2142            triggered_clone.store(true, Ordering::Release);
2143
2144            let kstack_sz = (raw.kstack_sz as usize).min(32);
2145            let event = ProbeEvent {
2146                func_idx: 0,
2147                task_ptr: raw.args[0],
2148                ts: raw.ts,
2149                args: raw.args,
2150                fields: vec![],
2151                kstack: raw.kstack[..kstack_sz].to_vec(),
2152                str_val: None,
2153                exit_fields: vec![],
2154                exit_ts: None,
2155            };
2156
2157            events_clone.lock().unwrap().push(event);
2158        }
2159
2160        0
2161    }) {
2162        tracing::error!(%e, "failed to register ring buffer callback");
2163        let error = format!("probe ring-buffer callback registration failed: {e}");
2164        diag.trigger_attach_error = Some(error.clone());
2165        publish_phase_a(Err(error));
2166        return (None, diag, Vec::new());
2167    }
2168
2169    let rb = match rb_builder.build() {
2170        Ok(rb) => rb,
2171        Err(e) => {
2172            tracing::error!(%e, "failed to build ring buffer");
2173            let error = format!("probe ring-buffer build failed: {e}");
2174            diag.trigger_attach_error = Some(error.clone());
2175            publish_phase_a(Err(error));
2176            return (None, diag, Vec::new());
2177        }
2178    };
2179
2180    // Enable is handled by the BPF program reading the volatile const.
2181    // Since we can't mutate rodata after load, the program starts enabled.
2182    // (ktstr_enabled defaults to false in BPF, but we always want probes
2183    // active once attached — remove the gate or set it before load.)
2184
2185    tracing::debug!(
2186        funcs = func_ips.len(),
2187        links = links.len(),
2188        trigger_type = %diag.trigger_type,
2189        "polling for probe data",
2190    );
2191
2192    // Signal Phase A probes attached (kprobes + kernel fexit +
2193    // trigger). When phase_b_rx is None, this means all probes.
2194    // When Some, BPF fentry is deferred to Phase B.
2195    publish_phase_a(Ok(()));
2196
2197    // Phase B: receive BPF fentry targets and attach them while
2198    // polling the ring buffer. The channel is consumed once; after
2199    // that phase_b_done prevents re-checking.
2200    let mut phase_b_rx = phase_b_rx;
2201    let mut phase_b_done = false;
2202    // Accumulates BTF from Phase B functions so the readout phase
2203    // can resolve field keys for both Phase A and Phase B functions.
2204    let mut phase_b_btf: Vec<BtfFunc> = Vec::new();
2205
2206    // Poll until trigger fires or stop requested.  When stop is
2207    // signaled, iterate all probe_data entries instead of waiting
2208    // for the trigger.
2209    loop {
2210        let _ = rb.poll(Duration::from_millis(100));
2211
2212        // Check for Phase B input while polling.
2213        if !phase_b_done && let Some(ref rx) = phase_b_rx {
2214            match rx.try_recv() {
2215                Ok(pb) => {
2216                    tracing::debug!(
2217                        bpf_funcs = pb.functions.len(),
2218                        "Phase B: attaching BPF fentry/fexit"
2219                    );
2220                    // Save Phase B BTF for readout field key resolution.
2221                    phase_b_btf = pb.btf_funcs.clone();
2222                    // Attach Phase B probes: kernel callers (kprobes
2223                    // + kernel fexit) and BPF callbacks (fentry/fexit).
2224                    attach_phase_b_fentry(
2225                        &skel,
2226                        &pb,
2227                        &mut func_ips,
2228                        &mut fentry_links,
2229                        &mut fexit_links,
2230                        &mut links,
2231                        &mut probed_ips,
2232                        &mut diag,
2233                    );
2234                    pb.done.set();
2235                    phase_b_done = true;
2236                    // Drop the receiver to release the channel.
2237                    phase_b_rx = None;
2238                }
2239                Err(std::sync::mpsc::TryRecvError::Empty) => {
2240                    // No Phase B input yet; if trigger already fired,
2241                    // break immediately — the crash happened before
2242                    // Phase B could attach.
2243                    if triggered.load(Ordering::Acquire) {
2244                        tracing::debug!("trigger fired during Phase B wait, skipping fentry");
2245                        phase_b_done = true;
2246                        phase_b_rx = None;
2247                    }
2248                }
2249                Err(std::sync::mpsc::TryRecvError::Disconnected) => {
2250                    // Channel sender dropped without delivering Phase B
2251                    // input. Without a Phase B payload there is nothing
2252                    // left for this probe loop to attach; if `stop` is
2253                    // also set, return empty diagnostics rather than
2254                    // poll an empty ringbuf for the rest of the timeout.
2255                    tracing::debug!("Phase B channel disconnected");
2256                    phase_b_done = true;
2257                    phase_b_rx = None;
2258                    if stop.load(Ordering::Relaxed) {
2259                        return (None, diag, Vec::new());
2260                    }
2261                }
2262            }
2263        }
2264
2265        // Also check BSS err_exit_detected: stall exits skip the
2266        // ring buffer (bpf_get_current_task is unrelated to the
2267        // stall cause) but still latch the BSS flag. Volatile read
2268        // because the BPF program writes via the kernel-side mmap
2269        // and Rust's aliasing rules let the compiler hoist a normal
2270        // read out of the loop.
2271        let bss_triggered = skel.maps.bss_data.as_ref().is_some_and(|bss| unsafe {
2272            std::ptr::read_volatile(&bss.ktstr_err_exit_detected as *const u32) != 0
2273        });
2274        // Mirror the latch into the cross-thread atomic so scenario
2275        // code can synchronously classify SchedulerDied as Clean vs
2276        // Crashed without owning the skeleton. Monotonic 0→1→2 —
2277        // once a poll iteration completes we know the probe armed,
2278        // and the BSS latch only ever transitions 0→1 within a
2279        // single test run. Release pairs with the Acquire load in
2280        // `sched_exit_kind()`.
2281        PROBE_SCHED_EXIT_STATE.store(
2282            if bss_triggered {
2283                PROBE_EXIT_STATE_CRASHED
2284            } else {
2285                PROBE_EXIT_STATE_CLEAN
2286            },
2287            Ordering::Release,
2288        );
2289        // Snapshot `triggered` once: a second `triggered.load` for the
2290        // diag assignment would observe an unrelated state if a racing
2291        // trigger fires between the two reads, so the gate decision
2292        // and the recorded `trigger_fired` could disagree.
2293        let triggered_snapshot = triggered.load(Ordering::Acquire);
2294        if triggered_snapshot || bss_triggered || stop.load(Ordering::Acquire) {
2295            // Final ringbuf drain before breaking. BPF-side ordering:
2296            // the trigger handler does
2297            // `__sync_val_compare_and_swap(&ktstr_err_exit_detected, 0u, 1u)`
2298            // BEFORE `bpf_ringbuf_reserve` + `bpf_ringbuf_submit`
2299            // (see src/bpf/probe.bpf.c near line 687-731). A userspace
2300            // observer can therefore see `bss_triggered=true` (CAS
2301            // visible) while the ringbuf event is still in transit
2302            // (submit not yet visible to `rb.poll`). Without an
2303            // explicit drain here, breaking out of the loop on
2304            // `bss_triggered` would lose the trigger event — the
2305            // readout phase below would see `events.last() = None`
2306            // (or the prior probe's trailing event), drop into the
2307            // no-causal-tptr fallback path, and produce
2308            // grouped-by-frequency output instead of a verified
2309            // stitch even though the kernel did publish a real
2310            // causal task pointer.
2311            //
2312            // 100 ms is bounded by the same teardown budget the loop's
2313            // top-level `rb.poll(100ms)` uses; libbpf's
2314            // `RingBuffer::poll` returns as soon as events are
2315            // consumed (it uses epoll-edge under the hood), so the
2316            // worst case is one full timeout window when no event
2317            // arrives — acceptable since we already know the trigger
2318            // fired (bss_triggered is true).
2319            let _ = rb.poll(Duration::from_millis(100));
2320            diag.trigger_fired = triggered_snapshot || bss_triggered;
2321
2322            // Read BPF-side diagnostic counters from BSS. The hot
2323            // counters live in the per-CPU `ktstr_pcpu_counters`
2324            // 2D array (`[MAX_CPUS][KTSTR_PCPU_NR]`); each per-CPU
2325            // slot is a cacheline-aligned `pcpu_counter { long
2326            // value; }`. Sum across CPUs to recover the cumulative
2327            // count — see `enum ktstr_pcpu_idx` in
2328            // src/bpf/probe.bpf.c for the slot indices.
2329            //
2330            // Every read below uses [`std::ptr::read_volatile`].
2331            // The BSS struct is mapped to userspace via the BPF
2332            // map's mmap region; the BPF program writes through
2333            // its own kernel-side mapping concurrently with these
2334            // reads. Without `read_volatile`, the userspace
2335            // compiler is free to hoist the loads (Rust's
2336            // aliasing rules: the compiler does not know a
2337            // `&types::bss` reference is shared with a kernel
2338            // writer through an unrelated mapping), miss the
2339            // post-trigger updates, and leave the diagnostic
2340            // counters / miss log / first-trigger timestamp
2341            // showing pre-trigger zeroes. Mirrors the existing
2342            // `ktstr_err_exit_detected` `read_volatile` site
2343            // upstream — same hazard, same fix, applied to every
2344            // BPF-mutated field the diag block reads.
2345            if let Some(bss) = skel.maps.bss_data.as_ref() {
2346                // Slot indices must match `enum ktstr_pcpu_idx`. A
2347                // reorder in the BPF source breaks every reader; the
2348                // explicit constants here surface that drift at the
2349                // call site instead of silently aliasing two
2350                // counters.
2351                const PCPU_PROBE_COUNT: usize = 0;
2352                const PCPU_KPROBE_RETURNS: usize = 1;
2353                const PCPU_META_MISS: usize = 2;
2354                const PCPU_RINGBUF_DROPS: usize = 3;
2355                const PCPU_TIMELINE_COUNT: usize = 4;
2356                const PCPU_TIMELINE_DROPS: usize = 5;
2357                const PCPU_TRIGGER_COUNT: usize = 14;
2358                let counters = &bss.ktstr_pcpu_counters;
2359                // SAFETY: each `pcpu_counter::value` is a plain
2360                // 64-bit integer at a stable BSS offset; the BPF
2361                // side updates it via `__sync_add_and_fetch`
2362                // (atomic add). A volatile load reads whatever
2363                // the kernel-side mmap currently shows — torn
2364                // reads are impossible because aligned 64-bit
2365                // loads are atomic on every supported arch
2366                // (x86_64, aarch64). The volatile qualifier is
2367                // what prevents the compiler from hoisting the
2368                // load out of the `sum` reduction across the
2369                // outer poll loop.
2370                let sum_pcpu = |idx: usize| -> u64 {
2371                    counters
2372                        .iter()
2373                        .map(|cpu_slots| unsafe {
2374                            std::ptr::read_volatile(&cpu_slots[idx].value as *const _) as u64
2375                        })
2376                        .sum()
2377                };
2378                diag.bpf_kprobe_fires = sum_pcpu(PCPU_PROBE_COUNT);
2379                diag.bpf_kprobe_returns = sum_pcpu(PCPU_KPROBE_RETURNS);
2380                diag.bpf_trigger_fires = sum_pcpu(PCPU_TRIGGER_COUNT);
2381                diag.bpf_meta_misses = sum_pcpu(PCPU_META_MISS);
2382                // SAFETY: `ktstr_miss_log_idx` is a `u32` written
2383                // via `__sync_fetch_and_add` from the BPF side
2384                // (see `src/bpf/probe.bpf.c` near the
2385                // `ktstr_miss_log[idx] = ip;` line). Aligned u32
2386                // loads are atomic on x86_64/aarch64. The BPF
2387                // writer increments-then-stores; a volatile read
2388                // observes either the pre- or post-update value
2389                // — both are bounded by the array length, so the
2390                // subsequent `.min(ktstr_miss_log.len())` keeps
2391                // the slice safe even if the kernel-side write
2392                // races this read.
2393                let miss_idx = unsafe {
2394                    std::ptr::read_volatile(&bss.ktstr_miss_log_idx as *const u32) as usize
2395                };
2396                let n = miss_idx.min(bss.ktstr_miss_log.len());
2397                // Element-wise volatile reads of the miss-log
2398                // entries that fall within `n`. A bulk
2399                // `to_vec()` over the `bss.ktstr_miss_log[..n]`
2400                // slice would let the compiler vectorise the
2401                // copy and elide the volatile semantics; pulling
2402                // each `u64` through `read_volatile` keeps every
2403                // load ordered against the BPF-side write.
2404                //
2405                // SAFETY: each entry is a 64-bit IP value the
2406                // BPF writer stores after its CAS-like
2407                // increment of `ktstr_miss_log_idx`. Aligned
2408                // u64 loads are atomic on every supported
2409                // arch; the BPF write order
2410                // (increment of `ktstr_miss_log_idx` via
2411                // `__sync_fetch_and_add` BEFORE the store
2412                // `ktstr_miss_log[idx] = ip`) means a
2413                // volatile read of `[..miss_idx]` may cover a
2414                // slot whose index was already reserved but
2415                // whose store has not yet landed. We tolerate
2416                // that race: a stale-zero entry is harmless
2417                // diagnostic noise compared with the
2418                // alternative (compiler-hoisted loads of
2419                // pre-trigger zeroes).
2420                diag.bpf_miss_ips = (0..n)
2421                    .map(|i| unsafe {
2422                        std::ptr::read_volatile(&bss.ktstr_miss_log[i] as *const u64)
2423                    })
2424                    .collect();
2425                diag.bpf_ringbuf_drops = sum_pcpu(PCPU_RINGBUF_DROPS);
2426                // SAFETY: `ktstr_last_trigger_ts` is a `u64`
2427                // written by the BPF trigger handler via
2428                // `bpf_ktime_get_ns()` (see
2429                // `src/bpf/probe.bpf.c::ktstr_last_trigger_ts`).
2430                // Aligned u64 loads are atomic; the volatile
2431                // qualifier prevents hoisting across the outer
2432                // poll loop so the userspace reader observes the
2433                // post-trigger timestamp instead of a cached
2434                // pre-trigger zero.
2435                diag.bpf_first_trigger_ns =
2436                    unsafe { std::ptr::read_volatile(&bss.ktstr_last_trigger_ts as *const u64) };
2437                // SAFETY: `ktstr_exit_kind_snap` is a `u32` written
2438                // by the BPF trigger handler in the same publishing
2439                // CAS sequence as `ktstr_err_exit_detected` (see
2440                // `src/bpf/probe.bpf.c::ktstr_exit_kind_snap`).
2441                // Aligned u32 loads are atomic on every supported
2442                // arch; the volatile qualifier prevents the compiler
2443                // from hoisting the load across the outer poll loop
2444                // so userspace observes the post-trigger SCX_EXIT_*
2445                // value rather than the pre-trigger zero.
2446                diag.bpf_exit_kind_snap =
2447                    unsafe { std::ptr::read_volatile(&bss.ktstr_exit_kind_snap as *const u32) };
2448                diag.bpf_timeline_count = sum_pcpu(PCPU_TIMELINE_COUNT);
2449                diag.bpf_timeline_drops = sum_pcpu(PCPU_TIMELINE_DROPS);
2450            }
2451
2452            let key_size = std::mem::size_of::<types::probe_key>();
2453            let mut probe_events = Vec::new();
2454            let mut total_keys = 0u32;
2455            let mut unmatched_ips = 0u32;
2456
2457            // Build IP → (func_idx, display_name) lookup once. The
2458            // event-drain loop below would otherwise scan every entry
2459            // in `func_ips` per probe_data key — O(events × funcs).
2460            // With thousands of funcs and tens of thousands of events
2461            // on a normal run, the linear scan dominates dump time.
2462            // HashMap turns the per-event lookup into O(1).
2463            let func_ips_by_ip: std::collections::HashMap<u64, (u32, &str)> = func_ips
2464                .iter()
2465                .map(|(idx, ip, name)| (*ip, (*idx, name.as_str())))
2466                .collect();
2467
2468            // Pre-compute per-function `field_keys_hints` once. The
2469            // previous code recomputed `build_field_keys` for every
2470            // event, even when many events share the same function —
2471            // O(events × funcs) field-key construction. Building the
2472            // map keyed by function name turns the per-event work
2473            // into a single HashMap lookup.
2474            let field_keys_by_func: std::collections::HashMap<&str, Vec<(String, RenderHint)>> =
2475                btf_funcs
2476                    .iter()
2477                    .chain(phase_b_btf.iter())
2478                    .map(|f| (f.name.as_str(), build_field_keys(f)))
2479                    .collect();
2480
2481            for key_bytes in skel.maps.probe_data.keys() {
2482                if key_bytes.len() < key_size {
2483                    continue;
2484                }
2485                total_keys += 1;
2486                let key: &types::probe_key =
2487                    unsafe { &*(key_bytes.as_ptr() as *const types::probe_key) };
2488
2489                // Find which function this IP belongs to.
2490                let (func_idx, display_name) = match func_ips_by_ip.get(&key.func_ip) {
2491                    Some(&(idx, name)) => (idx, name),
2492                    None => {
2493                        unmatched_ips += 1;
2494                        continue;
2495                    }
2496                };
2497
2498                if let Ok(Some(val_bytes)) = skel.maps.probe_data.lookup(&key_bytes, MapFlags::ANY)
2499                {
2500                    let entry: &types::probe_entry =
2501                        unsafe { &*(val_bytes.as_ptr() as *const types::probe_entry) };
2502                    if entry.ts == 0 {
2503                        continue;
2504                    }
2505
2506                    // Borrow the pre-computed hints for this function;
2507                    // an empty slice for unknown funcs preserves the
2508                    // previous `unwrap_or_default()` behaviour.
2509                    let empty: Vec<(String, RenderHint)> = Vec::new();
2510                    let field_keys_hints: &Vec<(String, RenderHint)> =
2511                        field_keys_by_func.get(display_name).unwrap_or(&empty);
2512
2513                    let nr = (entry.nr_fields as usize).min(16);
2514                    let fields: Vec<(String, u64)> = entry.fields[..nr]
2515                        .iter()
2516                        .enumerate()
2517                        .filter_map(|(i, &val)| {
2518                            field_keys_hints.get(i).map(|(k, _)| (k.clone(), val))
2519                        })
2520                        .collect();
2521
2522                    let str_val = if entry.has_str != 0 {
2523                        let s = &entry.str_val;
2524                        let bytes: Vec<u8> = s.iter().map(|&b| b as u8).collect();
2525                        let len = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len());
2526                        let text = std::str::from_utf8(&bytes[..len]).unwrap_or("").to_string();
2527                        if text.is_empty() { None } else { Some(text) }
2528                    } else {
2529                        None
2530                    };
2531
2532                    // Extract exit-side fields if fexit fired.
2533                    let (exit_fields, exit_ts) = if entry.has_exit != 0 {
2534                        let nr_exit = (entry.nr_exit_fields as usize).min(16);
2535                        let ef: Vec<(String, u64)> = entry.exit_fields[..nr_exit]
2536                            .iter()
2537                            .enumerate()
2538                            .filter_map(|(i, &val)| {
2539                                field_keys_hints.get(i).map(|(k, _)| (k.clone(), val))
2540                            })
2541                            .collect();
2542                        (ef, Some(entry.exit_ts))
2543                    } else {
2544                        (Vec::new(), None)
2545                    };
2546
2547                    probe_events.push(ProbeEvent {
2548                        func_idx,
2549                        task_ptr: key.task_ptr,
2550                        ts: entry.ts,
2551                        args: entry.args,
2552                        fields,
2553                        kstack: vec![],
2554                        str_val,
2555                        exit_fields,
2556                        exit_ts,
2557                    });
2558                }
2559            }
2560
2561            probe_events.sort_by_key(|e| e.ts);
2562
2563            diag.probe_data_keys = total_keys;
2564            diag.probe_data_unmatched_ips = unmatched_ips;
2565            diag.events_before_stitch = probe_events.len() as u32;
2566
2567            tracing::debug!(
2568                events = probe_events.len(),
2569                total_keys,
2570                unmatched_ips,
2571                "probe_data readout",
2572            );
2573
2574            if probe_events.is_empty() {
2575                return (None, diag, Vec::new());
2576            }
2577
2578            // Stitch by task_struct pointer. Build a map of func_idx ->
2579            // task_struct param index from BPF_OP_CALLERS and BTF, then
2580            // filter events to those referencing the same task_struct
2581            // pointer as the causal task.
2582            //
2583            // The selected BPF trigger handler sets args[0] to
2584            // bpf_get_current_task() ONLY for
2585            // SCX_EXIT_ERROR_BPF (1025), where a BPF scheduler
2586            // callback faulted in the running task's context — so
2587            // `current` IS the causal task. For SCX_EXIT_ERROR
2588            // (1024), args[0] is 0 because that exit can fire from
2589            // kworker context (async unregistration, sysrq), where
2590            // `current` is the worker thread rather than the task
2591            // that triggered the exit. For SCX_EXIT_ERROR_STALL
2592            // (1026) the trigger handler returns early without
2593            // submitting an event at all (watchdog/timer context).
2594            // The filter below therefore drops args[0] == 0 to
2595            // suppress non-causal probe output: no causal task
2596            // means no useful stitch chain.
2597            let task_param_idx = build_task_param_idx(&func_ips, btf_funcs, &phase_b_btf);
2598
2599            // Extract tptr and kstack from the trigger event in one
2600            // lock acquisition. When the trigger did not fire (stop-
2601            // signaled) or the exit kind lacks a causal task, probe
2602            // output is suppressed.
2603            let (target_tptr, trigger_kstack) = {
2604                let guard = events.lock().unwrap();
2605                let tptr = guard.last().and_then(|e| causal_tptr(e.task_ptr));
2606                let kstack = guard.last().map(|e| e.kstack.clone()).unwrap_or_default();
2607                (tptr, kstack)
2608            };
2609
2610            let Some(tptr) = target_tptr else {
2611                // No causal task identified — the trigger fired with
2612                // args[0]==0 (kind=STALL or generic ERROR from kworker
2613                // context), or never fired at all (probe lifecycle
2614                // race, scheduler clean-exited).
2615                //
2616                // Best-effort fallback: instead of returning empty,
2617                // group the captured events by task_struct pointer
2618                // (resolved via task_param_idx for events with a
2619                // task_struct param, otherwise key.task_ptr from the
2620                // probe map) and pick the top-N most-frequent
2621                // pointers. The events that fired during the actual
2622                // crash window ARE causal data — the host renderer
2623                // will mark the output as "trigger absent —
2624                // best-effort grouped by frequency" so the operator
2625                // doesn't mistake the candidate chain for a verified
2626                // stitch. Cap at 3 candidates to keep the output
2627                // readable; one task usually dominates a single
2628                // crash window.
2629                use std::collections::HashMap;
2630                let mut counts: HashMap<u64, u32> = HashMap::new();
2631                for ev in &probe_events {
2632                    let key = if let Some(&pidx) = task_param_idx.get(&ev.func_idx) {
2633                        ev.args[pidx]
2634                    } else {
2635                        ev.task_ptr
2636                    };
2637                    if key != 0 {
2638                        *counts.entry(key).or_default() += 1;
2639                    }
2640                }
2641                if counts.is_empty() {
2642                    tracing::debug!(
2643                        "no causal tptr and no candidate task_ptrs — suppressing probe output"
2644                    );
2645                    return (None, diag, Vec::new());
2646                }
2647                let mut sorted: Vec<(u64, u32)> = counts.into_iter().collect();
2648                sorted.sort_by(|a, b| b.1.cmp(&a.1));
2649                const MAX_CANDIDATES: usize = 3;
2650                sorted.truncate(MAX_CANDIDATES);
2651                let candidate_set: std::collections::HashSet<u64> =
2652                    sorted.iter().map(|(k, _)| *k).collect();
2653                let before = probe_events.len();
2654                probe_events.retain(|e| {
2655                    let key = if let Some(&pidx) = task_param_idx.get(&e.func_idx) {
2656                        e.args[pidx]
2657                    } else {
2658                        e.task_ptr
2659                    };
2660                    candidate_set.contains(&key)
2661                });
2662                tracing::debug!(
2663                    candidates = sorted.len(),
2664                    kept = probe_events.len(),
2665                    total = before,
2666                    "stitched by frequency (fallback — no causal tptr)"
2667                );
2668                diag.events_after_stitch = probe_events.len() as u32;
2669                diag.stitch_fallback_used = true;
2670                let fnames: Vec<(u32, String)> = func_ips
2671                    .iter()
2672                    .map(|(idx, _, name)| (*idx, name.clone()))
2673                    .collect();
2674                return (Some(probe_events), diag, fnames);
2675            };
2676
2677            let before = probe_events.len();
2678            probe_events.retain(|e| {
2679                if let Some(&pidx) = task_param_idx.get(&e.func_idx) {
2680                    e.args[pidx] == tptr
2681                } else {
2682                    e.task_ptr == tptr // no task_struct param — match on current
2683                }
2684            });
2685            tracing::debug!(
2686                tptr = format_args!("0x{tptr:x}"),
2687                kept = probe_events.len(),
2688                total = before,
2689                "stitched by task_struct arg",
2690            );
2691
2692            diag.events_after_stitch = probe_events.len() as u32;
2693
2694            // Attach trigger kstack if available.
2695            if let Some(last) = probe_events.last_mut() {
2696                last.kstack = trigger_kstack;
2697            }
2698
2699            let fnames: Vec<(u32, String)> = func_ips
2700                .iter()
2701                .map(|(idx, _, name)| (*idx, name.clone()))
2702                .collect();
2703            return (Some(probe_events), diag, fnames);
2704        }
2705    }
2706}
2707
2708/// Attach Phase B probes after the scheduler starts.
2709///
2710/// Handles both kernel callers (kprobes + kernel fexit) and BPF
2711/// callbacks (fentry/fexit). Uses `pb.func_idx_offset` for all
2712/// func_idx values to avoid collisions with Phase A indices.
2713///
2714/// Kprobes are attached via the Phase A kprobe skeleton (`skel`),
2715/// which stays alive throughout. BPF fentry/fexit use separate
2716/// skeleton loads that share `probe_data` and `func_meta_map` via
2717/// `reuse_fd`.
2718///
2719/// `probed_ips` carries the addresses Phase A already planted a kprobe
2720/// on; targets landing on one of them get their `func_meta` but no
2721/// second kprobe.
2722#[allow(clippy::too_many_arguments)]
2723fn attach_phase_b_fentry(
2724    skel: &crate::bpf_skel::ProbeSkel<'_>,
2725    pb: &PhaseBInput,
2726    func_ips: &mut Vec<(u32, u64, String)>,
2727    fentry_links: &mut Vec<libbpf_rs::Link>,
2728    fexit_links: &mut Vec<libbpf_rs::Link>,
2729    links: &mut Vec<(libbpf_rs::Link, String)>,
2730    probed_ips: &mut HashSet<u64>,
2731    diag: &mut ProbeDiagnostics,
2732) {
2733    use libbpf_rs::MapCore;
2734    use libbpf_rs::MapFlags;
2735
2736    let offset = pb.func_idx_offset;
2737
2738    // Pre-load vmlinux BTF once for the Phase B kprobe + fentry
2739    // loops below. Same rationale as the Phase A path in
2740    // `run_probe_skeleton`: per-call `resolve_field_specs(_, None)`
2741    // would re-parse the multi-MB vmlinux BTF on every iteration,
2742    // dominating Phase B attach time on a kernel with thousands of
2743    // probed functions.
2744    //
2745    // `cached_vmlinux_btf` shares the process-global memo with Phase
2746    // A so a single auto-repro VM pays one parse for both phases, and
2747    // a multi-VM nextest run pays one parse total.
2748    let cached_btf = crate::monitor::btf_offsets::cached_vmlinux_btf();
2749
2750    // --- Phase B kernel functions: kprobes + func_meta ---
2751    //
2752    // Two-pass shape: first populate `func_meta_map` for every
2753    // resolvable target and stage the (raw_name, idx, ip,
2754    // display_name) tuples, then run all kprobe attaches in
2755    // parallel via [`parallel_attach_kprobes`]. Splitting the loop
2756    // matters because the attach is the slow syscall pair (one
2757    // `perf_event_open` + one `BPF_LINK_CREATE` each); meta
2758    // population is a quick map update. Doing them as a single
2759    // sequential loop forces the slow attach to gate the next
2760    // iteration's meta update for no good reason.
2761    struct PhaseBKprobeTarget {
2762        raw_name: String,
2763        display_name: String,
2764        idx: u32,
2765        ip: u64,
2766    }
2767    let mut kprobe_targets: Vec<PhaseBKprobeTarget> = Vec::new();
2768    for (i, func) in pb.functions.iter().enumerate() {
2769        if func.is_bpf {
2770            continue;
2771        }
2772        let idx = offset + i as u32;
2773        let ip = match resolve_func_ip(&func.raw_name) {
2774            Some(ip) => ip,
2775            None => {
2776                tracing::warn!(func = %func.raw_name, "phase_b: could not resolve function IP");
2777                diag.kprobe_resolve_failed.push(func.raw_name.clone());
2778                continue;
2779            }
2780        };
2781
2782        let mut meta = types::func_meta {
2783            func_idx: idx,
2784            ..Default::default()
2785        };
2786
2787        if let Some(btf_func) = pb.btf_funcs.iter().find(|f| f.name == func.raw_name) {
2788            let field_specs = match cached_btf.as_ref() {
2789                Some(btf) => super::btf::resolve_field_specs_with_btf(btf_func, btf),
2790                None => Vec::new(),
2791            };
2792            populate_field_specs(&mut meta, &field_specs);
2793            meta.str_param_idx = detect_str_param(btf_func);
2794        }
2795
2796        let key_bytes = ip.to_ne_bytes();
2797        let meta_bytes = unsafe {
2798            std::slice::from_raw_parts(
2799                &meta as *const _ as *const u8,
2800                std::mem::size_of::<types::func_meta>(),
2801            )
2802        };
2803
2804        if let Err(e) = skel
2805            .maps
2806            .func_meta_map
2807            .update(&key_bytes, meta_bytes, MapFlags::ANY)
2808        {
2809            tracing::warn!(%e, func = %func.raw_name, "phase_b: failed to update func_meta_map");
2810            continue;
2811        }
2812
2813        kprobe_targets.push(PhaseBKprobeTarget {
2814            raw_name: func.raw_name.clone(),
2815            display_name: func.display_name.clone(),
2816            idx,
2817            ip,
2818        });
2819    }
2820
2821    // Skip any target Phase A already has a kprobe on: that kprobe runs
2822    // the same `ktstr_probe` program and keys off the `func_meta` written
2823    // above, so the Phase B entry capture is unaffected, while a second
2824    // kprobe on the address would only aggregate them in the kernel. See
2825    // the Phase A attach loop.
2826    let mut attach_slots: Vec<usize> = Vec::new();
2827    let mut attach_input: Vec<String> = Vec::new();
2828    for (slot, target) in kprobe_targets.iter().enumerate() {
2829        if !probed_ips.insert(target.ip) {
2830            tracing::debug!(
2831                func = %target.raw_name,
2832                ip = target.ip,
2833                "phase_b: kprobe already attached at ip, skipping",
2834            );
2835            continue;
2836        }
2837        attach_slots.push(slot);
2838        attach_input.push(target.raw_name.clone());
2839    }
2840    // Kernel fexit must mirror the set that resolved and received func_meta,
2841    // not the raw Phase-B plan. Feeding an absent generation-specific alias to
2842    // set_attach_target can poison the whole batch even though the matching
2843    // kprobe was already rejected above.
2844    struct KernelFexitTarget {
2845        idx: u32,
2846        name: String,
2847    }
2848    let kernel_fexit_targets: Vec<KernelFexitTarget> = kprobe_targets
2849        .iter()
2850        .map(|target| KernelFexitTarget {
2851            idx: target.idx,
2852            name: target.raw_name.clone(),
2853        })
2854        .collect();
2855    let attach_results = parallel_attach_kprobes(&skel.progs.ktstr_probe, &attach_input);
2856    // Scatter the results back onto their targets. `attach_results` is in
2857    // the same order as `attach_input`, which `attach_slots` maps onto
2858    // `kprobe_targets`, so the post-attach bookkeeping (links push,
2859    // func_ips push, counter bumps) still runs in target order. A `None`
2860    // slot is a target skipped by the dedupe above.
2861    let mut results_by_slot: Vec<Option<libbpf_rs::Result<libbpf_rs::Link>>> =
2862        (0..kprobe_targets.len()).map(|_| None).collect();
2863    for (slot, (raw_name, result)) in attach_slots.into_iter().zip(attach_results.into_iter()) {
2864        debug_assert_eq!(kprobe_targets[slot].raw_name, raw_name);
2865        results_by_slot[slot] = Some(result);
2866    }
2867    for (target, result) in kprobe_targets.into_iter().zip(results_by_slot.into_iter()) {
2868        match result {
2869            Some(Ok(link)) => {
2870                links.push((link, target.raw_name));
2871                diag.kprobe_attached += 1;
2872            }
2873            Some(Err(e)) => {
2874                // Nothing planted here after all — release the address.
2875                probed_ips.remove(&target.ip);
2876                tracing::warn!(%e, func = %target.raw_name, "phase_b kprobe attach failed");
2877                diag.kprobe_attach_failed
2878                    .push((target.raw_name, e.to_string()));
2879            }
2880            None => {}
2881        }
2882        diag.kprobe_resolved += 1;
2883        func_ips.push((target.idx, target.ip, target.display_name));
2884    }
2885
2886    // --- Phase B kernel function fexit batches (fd=0 = vmlinux BTF) ---
2887    use std::os::unix::io::AsFd;
2888    for chunk in kernel_fexit_targets.chunks(FENTRY_BATCH) {
2889        let names: Vec<(u32, &str)> = chunk.iter().map(|kt| (kt.idx, kt.name.as_str())).collect();
2890        if let Err((e, targeted)) = load_kernel_fexit_batch(
2891            &names,
2892            FENTRY_BATCH,
2893            skel.maps.probe_data.as_fd(),
2894            skel.maps.func_meta_map.as_fd(),
2895            fexit_links,
2896            "phase_b kernel fexit",
2897        ) {
2898            tracing::warn!(
2899                %e,
2900                kind = ?e.kind(),
2901                attempts = KERNEL_FEXIT_LOAD_ATTEMPTS,
2902                targets = ?targeted,
2903                "phase_b kernel fexit: batch load failed",
2904            );
2905            for name in targeted {
2906                diag.fentry_attach_failed
2907                    .push((name, format!("batch load: {e}")));
2908            }
2909        }
2910    }
2911
2912    // --- Phase B BPF callback fentry/fexit batches ---
2913    let valid_bpf: Vec<(u32, &StackFunction)> = pb
2914        .functions
2915        .iter()
2916        .enumerate()
2917        .filter(|(_, f)| f.bpf_prog_id.is_some())
2918        .map(|(i, f)| (offset + i as u32, f))
2919        .collect();
2920    diag.fentry_candidates += valid_bpf.len() as u32;
2921
2922    attach_bpf_fentry_batches(
2923        skel,
2924        &valid_bpf,
2925        &pb.functions,
2926        &pb.btf_funcs,
2927        &pb.bpf_prog_fds,
2928        cached_btf.as_ref(),
2929        func_ips,
2930        fentry_links,
2931        fexit_links,
2932        diag,
2933        "phase_b fentry",
2934    );
2935
2936    diag.fentry_attached = fentry_links.len() as u32;
2937    tracing::debug!(
2938        fentry = fentry_links.len(),
2939        fexit = fexit_links.len(),
2940        bpf_targets = valid_bpf.len(),
2941        kernel_targets = kernel_fexit_targets.len(),
2942        "Phase B probes attached",
2943    );
2944}
2945
2946#[cfg(test)]
2947#[path = "process_tests.rs"]
2948mod tests;