ktstr/
host_heap.rs

1//! Heap-state snapshot for the running ktstr binary.
2//!
3//! [`HostHeapState`] is a thin snapshot of the process's jemalloc
4//! allocator state at capture time: active / allocated / resident /
5//! mapped bytes plus the arena count. It rides along inside
6//! [`HostContext::heap_state`](crate::host_context::HostContext::heap_state)
7//! so a sidecar reader can correlate scheduler-test outcomes with
8//! the ktstr tool's own memory footprint — e.g. distinguish a
9//! legitimate regression from one where the runner itself OOM-pressured
10//! the host.
11//!
12//! # jemalloc is always linked
13//!
14//! `tikv-jemalloc-ctl` declares a non-optional dependency on
15//! `tikv-jemalloc-sys`, which builds and links libjemalloc
16//! unconditionally. So even consumers that do NOT install
17//! `tikv_jemallocator::Jemalloc` as `#[global_allocator]` carry
18//! libjemalloc in their binary, and every `mallctl` call from this
19//! module resolves to libjemalloc's implementation rather than a
20//! libc stub. `mallctl` reads succeed regardless of which allocator
21//! `#[global_allocator]` resolves to.
22//!
23//! What differs is the *meaning* of the numbers:
24//!
25//! - When jemalloc IS `#[global_allocator]` (every shipped binary in
26//!   this workspace declares it at its binary crate root), every heap
27//!   allocation flows through jemalloc and `stats.allocated` /
28//!   `stats.active` report
29//!   real application usage in the tens-to-hundreds of MiB range.
30//! - When jemalloc is linked but is NOT `#[global_allocator]`
31//!   (downstream consumers using ktstr as a library without opting
32//!   into jemallocator), the application never allocates through it.
33//!   Calling mallctl still initializes jemalloc and can give its own
34//!   small metadata allocations non-zero `stats.allocated` /
35//!   `stats.active` values, so counter magnitude is not a sound way
36//!   to identify the process allocator.
37//!
38//! Shipped binaries declare jemalloc at their crate roots and mark that
39//! fact once at process entry. Host-context capture checks the marker
40//! before calling [`collect`], so allocator-agnostic integration-test
41//! binaries neither initialize jemalloc nor publish its irrelevant
42//! metadata as application heap usage.
43//!
44//! # `stats` feature is required for stats reads
45//!
46//! libjemalloc only tracks `stats.*` counters when the C library is
47//! built with `--enable-stats`; without it the mallctl reads still
48//! succeed but return zero. The `stats` feature on both
49//! `tikv-jemalloc-ctl` and `tikv-jemallocator` in `Cargo.toml` forces
50//! the C build flag — `host_heap::collect` depends on this.
51//! `arenas.narenas` is independent of `--enable-stats`; it reports
52//! correctly either way.
53//!
54//! # Epoch discipline
55//!
56//! `stats.*` reads return cached values; the cache refreshes when
57//! the `epoch` mallctl is advanced. [`collect`] advances the epoch
58//! exactly once before issuing reads so each snapshot reflects
59//! post-advance state. Callers that invoke [`collect`] back-to-back
60//! see fresh reads every time because each call advances the epoch
61//! again.
62//!
63//! Side effect of `epoch::advance()`: libjemalloc flushes per-thread
64//! stat caches into the shared counters under a mallctl-internal
65//! lock. The operation is thread-safe per jemalloc's mallctl
66//! contract — concurrent `collect` calls from multiple ktstr
67//! threads are defined-behavior (though pointless; each caller sees
68//! its own refreshed snapshot and the last writer wins in the
69//! caches).
70
71use std::sync::atomic::{AtomicBool, Ordering};
72
73/// Explicit process-level identity for the allocator selected by the binary
74/// crate root. Mallctl counters cannot infer this: merely reading them
75/// initializes the linked jemalloc instance and creates small allocations even
76/// when Rust uses the system allocator.
77static JEMALLOC_IS_GLOBAL: AtomicBool = AtomicBool::new(false);
78
79/// Mark that this binary crate installed `tikv_jemallocator::Jemalloc` as its
80/// global allocator. Shipped binary entry points call this before doing work;
81/// library and integration-test binaries deliberately do not.
82#[doc(hidden)]
83pub fn mark_jemalloc_global_allocator() {
84    JEMALLOC_IS_GLOBAL.store(true, Ordering::Release);
85}
86
87/// Whether the current binary explicitly selected jemalloc as its global
88/// allocator.
89pub(crate) fn jemalloc_is_global_allocator() -> bool {
90    JEMALLOC_IS_GLOBAL.load(Ordering::Acquire)
91}
92
93/// Heap-state snapshot for the running process's jemalloc allocator.
94///
95/// Every field is `Option<u64>` (or `Option<usize>` for the arena
96/// count) so a partial read lands what succeeded and consumers can
97/// distinguish "jemalloc reported X" from "jemalloc did not report
98/// this field". The `Default` impl lands every field as `None`,
99/// matching the non-jemalloc fallback path and serving as the
100/// fixture for test call sites that want the empty shape.
101///
102/// # Constructing instances in tests
103///
104/// `HostHeapState` is `#[non_exhaustive]` — see
105/// [`crate::non_exhaustive`] for the cross-crate construction and
106/// pattern-match rules. Start from [`HostHeapState::test_fixture`]
107/// (populated baseline) or [`HostHeapState::default`] (all-`None`)
108/// and mutate fields.
109#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
110#[non_exhaustive]
111pub struct HostHeapState {
112    /// `stats.active` — bytes in active pages allocated by the
113    /// application. A multiple of the page size and `>=`
114    /// [`Self::allocated_bytes`]. Populated whenever libjemalloc
115    /// was built with `--enable-stats` (the `stats` feature on
116    /// `tikv-jemalloc-ctl` forces this). Host-context capture only
117    /// publishes this structure when the binary explicitly selected
118    /// jemalloc as `#[global_allocator]` (see module doc).
119    #[serde(default, skip_serializing_if = "Option::is_none")]
120    pub active_bytes: Option<u64>,
121    /// `stats.allocated` — total bytes allocated by the
122    /// application (sum of live allocations, excluding allocator
123    /// metadata and padding).
124    #[serde(default, skip_serializing_if = "Option::is_none")]
125    pub allocated_bytes: Option<u64>,
126    /// `stats.resident` — bytes in physically resident data pages
127    /// mapped by the allocator. Overestimates by including
128    /// demand-zeroed pages that have not been touched; jemalloc
129    /// documents this. A multiple of the page size and `>=`
130    /// [`Self::active_bytes`]. Reflects jemalloc's own metadata
131    /// footprint even when jemalloc is not `#[global_allocator]`.
132    #[serde(default, skip_serializing_if = "Option::is_none")]
133    pub resident_bytes: Option<u64>,
134    /// `stats.mapped` — bytes in active extents mapped by the
135    /// allocator. Excludes inactive extents even those with
136    /// unused dirty pages, so there is no strict ordering between
137    /// this and [`Self::resident_bytes`]. A multiple of the page
138    /// size and `>=` [`Self::active_bytes`]. Reflects jemalloc's
139    /// own metadata footprint even when jemalloc is not
140    /// `#[global_allocator]`.
141    #[serde(default, skip_serializing_if = "Option::is_none")]
142    pub mapped_bytes: Option<u64>,
143    /// `arenas.narenas` — current limit on the number of arenas.
144    /// Initialized at jemalloc startup (typically `4 * ncpus` on a
145    /// multi-core Linux host) and updated as the allocator grows
146    /// new arenas. Populated whenever libjemalloc is linked into
147    /// the binary, including on consumers that use ktstr as a
148    /// library without opting into jemallocator as
149    /// `#[global_allocator]` (see the module doc). `None` only on
150    /// the rare mallctl-error path.
151    #[serde(default, skip_serializing_if = "Option::is_none")]
152    pub narenas: Option<usize>,
153}
154
155impl HostHeapState {
156    /// Populated fixture for unit tests. Values are plausible for a
157    /// small ktstr run on a 16-CPU host: a few hundred KiB
158    /// allocated, rounded up to page-size multiples for active /
159    /// resident / mapped, and `narenas = 64` (jemalloc's
160    /// `4 * ncpus` default on a 16-CPU box).
161    ///
162    /// Call sites mutate the fields they care about:
163    ///
164    /// ```
165    /// use ktstr::prelude::HostHeapState;
166    /// let mut h = HostHeapState::test_fixture();
167    /// h.allocated_bytes = Some(0);
168    /// ```
169    pub fn test_fixture() -> HostHeapState {
170        HostHeapState {
171            active_bytes: Some(1 << 20),
172            allocated_bytes: Some(512 * 1024),
173            resident_bytes: Some(2 << 20),
174            mapped_bytes: Some(4 << 20),
175            narenas: Some(64),
176        }
177    }
178
179    /// Render as a human-readable multi-line block. Each field is
180    /// one `key: value` line; absent fields render `(unknown)` so
181    /// operators see which reads failed. The block ends with a
182    /// newline. Matches [`HostContext::format_human`](crate::host_context::HostContext::format_human)'s
183    /// shape — pair the two in `cargo ktstr show-host` for a
184    /// single-block host summary.
185    pub fn format_human(&self) -> String {
186        use std::fmt::Write;
187        // Destructuring bind forces every field of HostHeapState to
188        // appear by name here; adding a field will break the build
189        // until it's rendered.
190        let HostHeapState {
191            active_bytes,
192            allocated_bytes,
193            resident_bytes,
194            mapped_bytes,
195            narenas,
196        } = self;
197        fn row<T: std::fmt::Display>(out: &mut String, key: &str, value: Option<&T>) {
198            match value {
199                Some(v) => {
200                    let _ = writeln!(out, "{key}: {v}");
201                }
202                None => {
203                    let _ = writeln!(out, "{key}: (unknown)");
204                }
205            }
206        }
207        let mut out = String::new();
208        row(&mut out, "allocated_bytes", allocated_bytes.as_ref());
209        row(&mut out, "active_bytes", active_bytes.as_ref());
210        row(&mut out, "resident_bytes", resident_bytes.as_ref());
211        row(&mut out, "mapped_bytes", mapped_bytes.as_ref());
212        row(&mut out, "narenas", narenas.as_ref());
213        out
214    }
215
216    /// Render a field-by-field diff as `key: before → after` lines.
217    /// Omits unchanged fields; an empty return means the two
218    /// snapshots are identical. `None` renders as `(unknown)` so a
219    /// `None → Some(..)` transition is visible.
220    pub fn diff(&self, other: &HostHeapState) -> String {
221        use std::fmt::Write;
222        let HostHeapState {
223            active_bytes: a_active,
224            allocated_bytes: a_allocated,
225            resident_bytes: a_resident,
226            mapped_bytes: a_mapped,
227            narenas: a_narenas,
228        } = self;
229        let HostHeapState {
230            active_bytes: b_active,
231            allocated_bytes: b_allocated,
232            resident_bytes: b_resident,
233            mapped_bytes: b_mapped,
234            narenas: b_narenas,
235        } = other;
236        let mut out = String::new();
237        fn row_opt<T: std::fmt::Display + PartialEq>(
238            out: &mut String,
239            key: &str,
240            a: Option<&T>,
241            b: Option<&T>,
242        ) {
243            if a == b {
244                return;
245            }
246            let render = |v: Option<&T>| match v {
247                Some(x) => format!("{x}"),
248                None => "(unknown)".to_string(),
249            };
250            let _ = writeln!(out, "{key}: {} → {}", render(a), render(b));
251        }
252        row_opt(
253            &mut out,
254            "allocated_bytes",
255            a_allocated.as_ref(),
256            b_allocated.as_ref(),
257        );
258        row_opt(
259            &mut out,
260            "active_bytes",
261            a_active.as_ref(),
262            b_active.as_ref(),
263        );
264        row_opt(
265            &mut out,
266            "resident_bytes",
267            a_resident.as_ref(),
268            b_resident.as_ref(),
269        );
270        row_opt(
271            &mut out,
272            "mapped_bytes",
273            a_mapped.as_ref(),
274            b_mapped.as_ref(),
275        );
276        row_opt(&mut out, "narenas", a_narenas.as_ref(), b_narenas.as_ref());
277        out
278    }
279}
280
281/// Capture the running process's jemalloc heap state.
282///
283/// Advances the jemalloc `epoch` exactly once so cached `stats.*`
284/// values refresh (this is a jemalloc-internal operation —
285/// libjemalloc flushes per-thread stat caches into the shared
286/// counters under its mallctl lock, and the operation is thread-safe
287/// per jemalloc's mallctl contract), then reads five mallctl values.
288/// Any individual read error lands that field as `None`; an
289/// `epoch::advance()` error short-circuits the whole function to
290/// [`HostHeapState::default`] because without a refreshed epoch the
291/// stats reads would return values from an arbitrary prior snapshot.
292///
293/// Since libjemalloc is linked unconditionally via
294/// `tikv-jemalloc-sys` (see module doc), `epoch::advance()` and the
295/// subsequent mallctl reads always succeed on a well-formed build.
296/// The `is_err()` branch below is a defensive guard against future
297/// jemalloc versions changing the error surface, not an expected
298/// fallback.
299///
300/// Direct callers may use this even when jemalloc is not the global
301/// allocator; the reads then describe only jemalloc's own mallctl-triggered
302/// initialization. [`HostContext::heap_state`](crate::host_context::HostContext)
303/// checks the explicit process allocator marker before calling this function,
304/// so sidecars only publish these fields when they represent real application
305/// usage.
306///
307/// # Cost
308///
309/// One `mallctl("epoch", ...)` call plus five
310/// `mallctl("stats.*"/"arenas.narenas", ...)` reads. Each is a
311/// `memcpy` from a cached value after a short tree walk inside
312/// jemalloc — microseconds total. Safe to call on every sidecar
313/// write.
314pub fn collect() -> HostHeapState {
315    // epoch advance refreshes jemalloc's stat cache. libjemalloc is
316    // always linked (tikv-jemalloc-sys is a hard dep of
317    // tikv-jemalloc-ctl), so this only fails on an unexpected
318    // jemalloc-internal error path. Defensive fall-through to the
319    // all-None default keeps `collect` infallible.
320    if tikv_jemalloc_ctl::epoch::advance().is_err() {
321        return HostHeapState::default();
322    }
323    // Each read is independent — a single error on `stats.allocated`
324    // does not poison `arenas.narenas`. `arenas.narenas` is
325    // initialized at jemalloc startup (typically `4 * ncpus`) and
326    // always readable on a libjemalloc-linked build regardless of
327    // `--enable-stats`. `.ok().map(|v| v as _)` lands Err as None.
328    let active_bytes = tikv_jemalloc_ctl::stats::active::read()
329        .ok()
330        .map(|v| v as u64);
331    let allocated_bytes = tikv_jemalloc_ctl::stats::allocated::read()
332        .ok()
333        .map(|v| v as u64);
334    let resident_bytes = tikv_jemalloc_ctl::stats::resident::read()
335        .ok()
336        .map(|v| v as u64);
337    let mapped_bytes = tikv_jemalloc_ctl::stats::mapped::read()
338        .ok()
339        .map(|v| v as u64);
340    let narenas = tikv_jemalloc_ctl::arenas::narenas::read()
341        .ok()
342        .map(|v| v as usize);
343    HostHeapState {
344        active_bytes,
345        allocated_bytes,
346        resident_bytes,
347        mapped_bytes,
348        narenas,
349    }
350}
351
352#[cfg(test)]
353mod tests {
354    use super::*;
355
356    #[test]
357    fn default_lands_all_none() {
358        let h = HostHeapState::default();
359        assert!(h.active_bytes.is_none());
360        assert!(h.allocated_bytes.is_none());
361        assert!(h.resident_bytes.is_none());
362        assert!(h.mapped_bytes.is_none());
363        assert!(h.narenas.is_none());
364    }
365
366    #[test]
367    fn test_fixture_populates_every_field() {
368        let h = HostHeapState::test_fixture();
369        assert!(h.active_bytes.is_some());
370        assert!(h.allocated_bytes.is_some());
371        assert!(h.resident_bytes.is_some());
372        assert!(h.mapped_bytes.is_some());
373        assert!(h.narenas.is_some());
374    }
375
376    #[test]
377    fn format_human_lists_every_field() {
378        let out = HostHeapState::test_fixture().format_human();
379        assert!(out.contains("allocated_bytes:"));
380        assert!(out.contains("active_bytes:"));
381        assert!(out.contains("resident_bytes:"));
382        assert!(out.contains("mapped_bytes:"));
383        assert!(out.contains("narenas:"));
384        assert!(out.ends_with('\n'));
385    }
386
387    /// Snapshot-style pin of the label sequence `format_human`
388    /// emits. Mirrors
389    /// `host_context::tests::format_human_field_order_is_stable` —
390    /// `HostContext::format_human` embeds this block indented under
391    /// the `heap_state:` parent label, and downstream diff tools +
392    /// operator-eye scanning depend on a stable
393    /// `allocated → active → resident → mapped → narenas`
394    /// top-to-bottom ordering. A silent reorder from a future edit
395    /// that shuffles the `row(...)` calls inside `format_human`
396    /// would slip past the order-blind `.contains(...)` checks in
397    /// the sibling tests. This test fails the moment the sequence
398    /// drifts; updating it forces the author to acknowledge the
399    /// reorder and double-check the HostContext ctprof sub-block
400    /// still reads coherently.
401    #[test]
402    fn format_human_field_order_is_stable() {
403        let out = HostHeapState::default().format_human();
404        let labels: Vec<&str> = out
405            .lines()
406            .filter_map(|l| l.split(':').next())
407            .filter(|s| !s.starts_with(' '))
408            .collect();
409        assert_eq!(
410            labels,
411            vec![
412                "allocated_bytes",
413                "active_bytes",
414                "resident_bytes",
415                "mapped_bytes",
416                "narenas",
417            ],
418            "format_human field order drifted — if intentional, update \
419             the expected vector and verify the HostContext heap_state \
420             sub-block still reads in the expected top-to-bottom order",
421        );
422    }
423
424    #[test]
425    fn format_human_renders_none_as_unknown() {
426        let out = HostHeapState::default().format_human();
427        // Every line should end with `: (unknown)`.
428        for line in out.lines() {
429            assert!(
430                line.ends_with(": (unknown)"),
431                "expected unknown, got {line:?}"
432            );
433        }
434    }
435
436    #[test]
437    fn diff_is_empty_on_equal_snapshots() {
438        let a = HostHeapState::test_fixture();
439        let b = HostHeapState::test_fixture();
440        assert_eq!(a.diff(&b), "");
441    }
442
443    #[test]
444    fn diff_reports_only_changed_fields() {
445        let a = HostHeapState::test_fixture();
446        let mut b = a.clone();
447        b.allocated_bytes = Some(9 * 1024 * 1024);
448        let d = a.diff(&b);
449        assert!(d.contains("allocated_bytes:"));
450        assert!(!d.contains("active_bytes:"));
451        assert!(!d.contains("resident_bytes:"));
452        assert!(!d.contains("mapped_bytes:"));
453        assert!(!d.contains("narenas:"));
454        assert!(d.contains("→"));
455    }
456
457    #[test]
458    fn diff_renders_none_transitions() {
459        let a = HostHeapState::default();
460        let b = HostHeapState::test_fixture();
461        let d = a.diff(&b);
462        // Every field changed, every line should carry the unknown→x arrow.
463        assert!(d.contains("allocated_bytes: (unknown) →"));
464        assert!(d.contains("narenas: (unknown) →"));
465    }
466
467    #[test]
468    fn diff_renders_some_to_none_transitions() {
469        // Symmetric case to `diff_renders_none_transitions`: a full
470        // fixture diffed against `default()` must surface each field
471        // as `x → (unknown)`, not be silently absorbed. Without this
472        // test a one-sided `(unknown) → x` match could mask a
473        // formatting bug in the reverse direction (e.g. the renderer
474        // inadvertently suppressing `Some → None` as unchanged).
475        let a = HostHeapState::test_fixture();
476        let b = HostHeapState::default();
477        let d = a.diff(&b);
478        assert!(
479            d.contains("allocated_bytes:") && d.contains("→ (unknown)"),
480            "expected allocated_bytes → (unknown), got:\n{d}",
481        );
482        assert!(d.contains("active_bytes:"));
483        assert!(d.contains("resident_bytes:"));
484        assert!(d.contains("mapped_bytes:"));
485        assert!(d.contains("narenas:"));
486    }
487
488    #[test]
489    fn serde_round_trip_preserves_fields() {
490        let h = HostHeapState::test_fixture();
491        let s = serde_json::to_string(&h).expect("serialize");
492        let back: HostHeapState = serde_json::from_str(&s).expect("deserialize");
493        assert_eq!(back, h);
494    }
495
496    #[test]
497    fn serde_skips_none_fields() {
498        let h = HostHeapState::default();
499        let s = serde_json::to_string(&h).expect("serialize");
500        // Every field is None and skip_serializing_if drops all of
501        // them, so the JSON object is empty.
502        assert_eq!(s, "{}");
503    }
504
505    #[test]
506    fn serde_accepts_missing_fields_via_defaults() {
507        // Older sidecar with no heap_state fields still
508        // deserializes to a valid (all-None) snapshot.
509        let back: HostHeapState = serde_json::from_str("{}").expect("deserialize");
510        assert_eq!(back, HostHeapState::default());
511    }
512
513    /// Under the library-crate test harness, `tikv-jemallocator` is
514    /// NOT installed as `#[global_allocator]` — the ktstr library
515    /// itself declares no allocator so downstream consumers can
516    /// pick their own. Libjemalloc is still linked (hard dep of
517    /// `tikv-jemalloc-ctl`) and `collect()` initializes it, so its
518    /// own small metadata allocations may make `stats.allocated`
519    /// and `stats.active` non-zero even though Rust allocations use
520    /// libc.
521    ///
522    /// Under this shape, the jemalloc invariants only describe its
523    /// own initialization. They do NOT validate the application's
524    /// allocator behavior.
525    /// Real invariant coverage lives in the shipped ktstr binaries,
526    /// whose crate roots declare `tikv_jemallocator::Jemalloc` as the
527    /// `#[global_allocator]`; a
528    /// live production run of any of those binaries exercises the
529    /// non-trivial invariants. Documenting rather than
530    /// feature-gating because a lib-crate integration test with its
531    /// own `#[global_allocator]` would add its own binary target
532    /// and is heavier than the coverage warrants.
533    ///
534    /// What this test DOES pin, non-trivially:
535    /// - libjemalloc-linked-build contract: `narenas` is always
536    ///   populated after `epoch::advance()` because arena count is
537    ///   a jemalloc-init-time constant, independent of whether
538    ///   jemalloc served any allocations.
539    /// - `collect()` infallibility on a libjemalloc build — the
540    ///   epoch::advance defensive guard does not fire.
541    #[test]
542    fn collect_populates_linked_jemalloc_without_selecting_it() {
543        let h = collect();
544        // libjemalloc is linked unconditionally, so every field
545        // populates. `narenas` in particular is a jemalloc-init
546        // constant (4*ncpus by default) and always non-zero on a
547        // multi-core host.
548        assert!(
549            h.narenas.is_some(),
550            "narenas must populate on a libjemalloc build"
551        );
552        assert!(
553            h.narenas.unwrap() > 0,
554            "narenas must be > 0; jemalloc computes 4*ncpus at init",
555        );
556        // stats reads also populate (they're not None) because
557        // `--enable-stats` is forced by the `stats` feature in
558        // Cargo.toml. Their VALUES depend on whether jemalloc is
559        // `#[global_allocator]` — see the doc comment on this test.
560        assert!(h.allocated_bytes.is_some());
561        assert!(h.active_bytes.is_some());
562        assert!(h.resident_bytes.is_some());
563        assert!(h.mapped_bytes.is_some());
564        assert!(
565            !jemalloc_is_global_allocator(),
566            "the allocator-agnostic library test binary must not be marked as a jemalloc-global process",
567        );
568        let before = h.allocated_bytes.expect("allocated populated");
569        let held = vec![0u8; 8 * 1024 * 1024];
570        std::hint::black_box(&held);
571        let after = collect().allocated_bytes.expect("allocated populated");
572        assert!(
573            after.saturating_sub(before) < 1024 * 1024,
574            "an 8 MiB Rust allocation must not flow through the linked-but-unselected jemalloc: before={before}, after={after}",
575        );
576    }
577}