ktstr/host_heap.rs
1//! Heap-state snapshot for the running ktstr binary.
2//!
3//! [`HostHeapState`] is a thin snapshot of the process's jemalloc
4//! allocator state at capture time: active / allocated / resident /
5//! mapped bytes plus the arena count. It rides along inside
6//! [`HostContext::heap_state`](crate::host_context::HostContext::heap_state)
7//! so a sidecar reader can correlate scheduler-test outcomes with
8//! the ktstr tool's own memory footprint — e.g. distinguish a
9//! legitimate regression from one where the runner itself OOM-pressured
10//! the host.
11//!
12//! # jemalloc is always linked
13//!
14//! `tikv-jemalloc-ctl` declares a non-optional dependency on
15//! `tikv-jemalloc-sys`, which builds and links libjemalloc
16//! unconditionally. So even consumers that do NOT install
17//! `tikv_jemallocator::Jemalloc` as `#[global_allocator]` carry
18//! libjemalloc in their binary, and every `mallctl` call from this
19//! module resolves to libjemalloc's implementation rather than a
20//! libc stub. `mallctl` reads succeed regardless of which allocator
21//! `#[global_allocator]` resolves to.
22//!
23//! What differs is the *meaning* of the numbers:
24//!
25//! - When jemalloc IS `#[global_allocator]` (every shipped binary in
26//! this workspace declares it at its binary crate root), every heap
27//! allocation flows through jemalloc and `stats.allocated` /
28//! `stats.active` report
29//! real application usage in the tens-to-hundreds of MiB range.
30//! - When jemalloc is linked but is NOT `#[global_allocator]`
31//! (downstream consumers using ktstr as a library without opting
32//! into jemallocator), the application never allocates through it.
33//! Calling mallctl still initializes jemalloc and can give its own
34//! small metadata allocations non-zero `stats.allocated` /
35//! `stats.active` values, so counter magnitude is not a sound way
36//! to identify the process allocator.
37//!
38//! Shipped binaries declare jemalloc at their crate roots and mark that
39//! fact once at process entry. Host-context capture checks the marker
40//! before calling [`collect`], so allocator-agnostic integration-test
41//! binaries neither initialize jemalloc nor publish its irrelevant
42//! metadata as application heap usage.
43//!
44//! # `stats` feature is required for stats reads
45//!
46//! libjemalloc only tracks `stats.*` counters when the C library is
47//! built with `--enable-stats`; without it the mallctl reads still
48//! succeed but return zero. The `stats` feature on both
49//! `tikv-jemalloc-ctl` and `tikv-jemallocator` in `Cargo.toml` forces
50//! the C build flag — `host_heap::collect` depends on this.
51//! `arenas.narenas` is independent of `--enable-stats`; it reports
52//! correctly either way.
53//!
54//! # Epoch discipline
55//!
56//! `stats.*` reads return cached values; the cache refreshes when
57//! the `epoch` mallctl is advanced. [`collect`] advances the epoch
58//! exactly once before issuing reads so each snapshot reflects
59//! post-advance state. Callers that invoke [`collect`] back-to-back
60//! see fresh reads every time because each call advances the epoch
61//! again.
62//!
63//! Side effect of `epoch::advance()`: libjemalloc flushes per-thread
64//! stat caches into the shared counters under a mallctl-internal
65//! lock. The operation is thread-safe per jemalloc's mallctl
66//! contract — concurrent `collect` calls from multiple ktstr
67//! threads are defined-behavior (though pointless; each caller sees
68//! its own refreshed snapshot and the last writer wins in the
69//! caches).
70
71use std::sync::atomic::{AtomicBool, Ordering};
72
73/// Explicit process-level identity for the allocator selected by the binary
74/// crate root. Mallctl counters cannot infer this: merely reading them
75/// initializes the linked jemalloc instance and creates small allocations even
76/// when Rust uses the system allocator.
77static JEMALLOC_IS_GLOBAL: AtomicBool = AtomicBool::new(false);
78
79/// Mark that this binary crate installed `tikv_jemallocator::Jemalloc` as its
80/// global allocator. Shipped binary entry points call this before doing work;
81/// library and integration-test binaries deliberately do not.
82#[doc(hidden)]
83pub fn mark_jemalloc_global_allocator() {
84 JEMALLOC_IS_GLOBAL.store(true, Ordering::Release);
85}
86
87/// Whether the current binary explicitly selected jemalloc as its global
88/// allocator.
89pub(crate) fn jemalloc_is_global_allocator() -> bool {
90 JEMALLOC_IS_GLOBAL.load(Ordering::Acquire)
91}
92
93/// Heap-state snapshot for the running process's jemalloc allocator.
94///
95/// Every field is `Option<u64>` (or `Option<usize>` for the arena
96/// count) so a partial read lands what succeeded and consumers can
97/// distinguish "jemalloc reported X" from "jemalloc did not report
98/// this field". The `Default` impl lands every field as `None`,
99/// matching the non-jemalloc fallback path and serving as the
100/// fixture for test call sites that want the empty shape.
101///
102/// # Constructing instances in tests
103///
104/// `HostHeapState` is `#[non_exhaustive]` — see
105/// [`crate::non_exhaustive`] for the cross-crate construction and
106/// pattern-match rules. Start from [`HostHeapState::test_fixture`]
107/// (populated baseline) or [`HostHeapState::default`] (all-`None`)
108/// and mutate fields.
109#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
110#[non_exhaustive]
111pub struct HostHeapState {
112 /// `stats.active` — bytes in active pages allocated by the
113 /// application. A multiple of the page size and `>=`
114 /// [`Self::allocated_bytes`]. Populated whenever libjemalloc
115 /// was built with `--enable-stats` (the `stats` feature on
116 /// `tikv-jemalloc-ctl` forces this). Host-context capture only
117 /// publishes this structure when the binary explicitly selected
118 /// jemalloc as `#[global_allocator]` (see module doc).
119 #[serde(default, skip_serializing_if = "Option::is_none")]
120 pub active_bytes: Option<u64>,
121 /// `stats.allocated` — total bytes allocated by the
122 /// application (sum of live allocations, excluding allocator
123 /// metadata and padding).
124 #[serde(default, skip_serializing_if = "Option::is_none")]
125 pub allocated_bytes: Option<u64>,
126 /// `stats.resident` — bytes in physically resident data pages
127 /// mapped by the allocator. Overestimates by including
128 /// demand-zeroed pages that have not been touched; jemalloc
129 /// documents this. A multiple of the page size and `>=`
130 /// [`Self::active_bytes`]. Reflects jemalloc's own metadata
131 /// footprint even when jemalloc is not `#[global_allocator]`.
132 #[serde(default, skip_serializing_if = "Option::is_none")]
133 pub resident_bytes: Option<u64>,
134 /// `stats.mapped` — bytes in active extents mapped by the
135 /// allocator. Excludes inactive extents even those with
136 /// unused dirty pages, so there is no strict ordering between
137 /// this and [`Self::resident_bytes`]. A multiple of the page
138 /// size and `>=` [`Self::active_bytes`]. Reflects jemalloc's
139 /// own metadata footprint even when jemalloc is not
140 /// `#[global_allocator]`.
141 #[serde(default, skip_serializing_if = "Option::is_none")]
142 pub mapped_bytes: Option<u64>,
143 /// `arenas.narenas` — current limit on the number of arenas.
144 /// Initialized at jemalloc startup (typically `4 * ncpus` on a
145 /// multi-core Linux host) and updated as the allocator grows
146 /// new arenas. Populated whenever libjemalloc is linked into
147 /// the binary, including on consumers that use ktstr as a
148 /// library without opting into jemallocator as
149 /// `#[global_allocator]` (see the module doc). `None` only on
150 /// the rare mallctl-error path.
151 #[serde(default, skip_serializing_if = "Option::is_none")]
152 pub narenas: Option<usize>,
153}
154
155impl HostHeapState {
156 /// Populated fixture for unit tests. Values are plausible for a
157 /// small ktstr run on a 16-CPU host: a few hundred KiB
158 /// allocated, rounded up to page-size multiples for active /
159 /// resident / mapped, and `narenas = 64` (jemalloc's
160 /// `4 * ncpus` default on a 16-CPU box).
161 ///
162 /// Call sites mutate the fields they care about:
163 ///
164 /// ```
165 /// use ktstr::prelude::HostHeapState;
166 /// let mut h = HostHeapState::test_fixture();
167 /// h.allocated_bytes = Some(0);
168 /// ```
169 pub fn test_fixture() -> HostHeapState {
170 HostHeapState {
171 active_bytes: Some(1 << 20),
172 allocated_bytes: Some(512 * 1024),
173 resident_bytes: Some(2 << 20),
174 mapped_bytes: Some(4 << 20),
175 narenas: Some(64),
176 }
177 }
178
179 /// Render as a human-readable multi-line block. Each field is
180 /// one `key: value` line; absent fields render `(unknown)` so
181 /// operators see which reads failed. The block ends with a
182 /// newline. Matches [`HostContext::format_human`](crate::host_context::HostContext::format_human)'s
183 /// shape — pair the two in `cargo ktstr show-host` for a
184 /// single-block host summary.
185 pub fn format_human(&self) -> String {
186 use std::fmt::Write;
187 // Destructuring bind forces every field of HostHeapState to
188 // appear by name here; adding a field will break the build
189 // until it's rendered.
190 let HostHeapState {
191 active_bytes,
192 allocated_bytes,
193 resident_bytes,
194 mapped_bytes,
195 narenas,
196 } = self;
197 fn row<T: std::fmt::Display>(out: &mut String, key: &str, value: Option<&T>) {
198 match value {
199 Some(v) => {
200 let _ = writeln!(out, "{key}: {v}");
201 }
202 None => {
203 let _ = writeln!(out, "{key}: (unknown)");
204 }
205 }
206 }
207 let mut out = String::new();
208 row(&mut out, "allocated_bytes", allocated_bytes.as_ref());
209 row(&mut out, "active_bytes", active_bytes.as_ref());
210 row(&mut out, "resident_bytes", resident_bytes.as_ref());
211 row(&mut out, "mapped_bytes", mapped_bytes.as_ref());
212 row(&mut out, "narenas", narenas.as_ref());
213 out
214 }
215
216 /// Render a field-by-field diff as `key: before → after` lines.
217 /// Omits unchanged fields; an empty return means the two
218 /// snapshots are identical. `None` renders as `(unknown)` so a
219 /// `None → Some(..)` transition is visible.
220 pub fn diff(&self, other: &HostHeapState) -> String {
221 use std::fmt::Write;
222 let HostHeapState {
223 active_bytes: a_active,
224 allocated_bytes: a_allocated,
225 resident_bytes: a_resident,
226 mapped_bytes: a_mapped,
227 narenas: a_narenas,
228 } = self;
229 let HostHeapState {
230 active_bytes: b_active,
231 allocated_bytes: b_allocated,
232 resident_bytes: b_resident,
233 mapped_bytes: b_mapped,
234 narenas: b_narenas,
235 } = other;
236 let mut out = String::new();
237 fn row_opt<T: std::fmt::Display + PartialEq>(
238 out: &mut String,
239 key: &str,
240 a: Option<&T>,
241 b: Option<&T>,
242 ) {
243 if a == b {
244 return;
245 }
246 let render = |v: Option<&T>| match v {
247 Some(x) => format!("{x}"),
248 None => "(unknown)".to_string(),
249 };
250 let _ = writeln!(out, "{key}: {} → {}", render(a), render(b));
251 }
252 row_opt(
253 &mut out,
254 "allocated_bytes",
255 a_allocated.as_ref(),
256 b_allocated.as_ref(),
257 );
258 row_opt(
259 &mut out,
260 "active_bytes",
261 a_active.as_ref(),
262 b_active.as_ref(),
263 );
264 row_opt(
265 &mut out,
266 "resident_bytes",
267 a_resident.as_ref(),
268 b_resident.as_ref(),
269 );
270 row_opt(
271 &mut out,
272 "mapped_bytes",
273 a_mapped.as_ref(),
274 b_mapped.as_ref(),
275 );
276 row_opt(&mut out, "narenas", a_narenas.as_ref(), b_narenas.as_ref());
277 out
278 }
279}
280
281/// Capture the running process's jemalloc heap state.
282///
283/// Advances the jemalloc `epoch` exactly once so cached `stats.*`
284/// values refresh (this is a jemalloc-internal operation —
285/// libjemalloc flushes per-thread stat caches into the shared
286/// counters under its mallctl lock, and the operation is thread-safe
287/// per jemalloc's mallctl contract), then reads five mallctl values.
288/// Any individual read error lands that field as `None`; an
289/// `epoch::advance()` error short-circuits the whole function to
290/// [`HostHeapState::default`] because without a refreshed epoch the
291/// stats reads would return values from an arbitrary prior snapshot.
292///
293/// Since libjemalloc is linked unconditionally via
294/// `tikv-jemalloc-sys` (see module doc), `epoch::advance()` and the
295/// subsequent mallctl reads always succeed on a well-formed build.
296/// The `is_err()` branch below is a defensive guard against future
297/// jemalloc versions changing the error surface, not an expected
298/// fallback.
299///
300/// Direct callers may use this even when jemalloc is not the global
301/// allocator; the reads then describe only jemalloc's own mallctl-triggered
302/// initialization. [`HostContext::heap_state`](crate::host_context::HostContext)
303/// checks the explicit process allocator marker before calling this function,
304/// so sidecars only publish these fields when they represent real application
305/// usage.
306///
307/// # Cost
308///
309/// One `mallctl("epoch", ...)` call plus five
310/// `mallctl("stats.*"/"arenas.narenas", ...)` reads. Each is a
311/// `memcpy` from a cached value after a short tree walk inside
312/// jemalloc — microseconds total. Safe to call on every sidecar
313/// write.
314pub fn collect() -> HostHeapState {
315 // epoch advance refreshes jemalloc's stat cache. libjemalloc is
316 // always linked (tikv-jemalloc-sys is a hard dep of
317 // tikv-jemalloc-ctl), so this only fails on an unexpected
318 // jemalloc-internal error path. Defensive fall-through to the
319 // all-None default keeps `collect` infallible.
320 if tikv_jemalloc_ctl::epoch::advance().is_err() {
321 return HostHeapState::default();
322 }
323 // Each read is independent — a single error on `stats.allocated`
324 // does not poison `arenas.narenas`. `arenas.narenas` is
325 // initialized at jemalloc startup (typically `4 * ncpus`) and
326 // always readable on a libjemalloc-linked build regardless of
327 // `--enable-stats`. `.ok().map(|v| v as _)` lands Err as None.
328 let active_bytes = tikv_jemalloc_ctl::stats::active::read()
329 .ok()
330 .map(|v| v as u64);
331 let allocated_bytes = tikv_jemalloc_ctl::stats::allocated::read()
332 .ok()
333 .map(|v| v as u64);
334 let resident_bytes = tikv_jemalloc_ctl::stats::resident::read()
335 .ok()
336 .map(|v| v as u64);
337 let mapped_bytes = tikv_jemalloc_ctl::stats::mapped::read()
338 .ok()
339 .map(|v| v as u64);
340 let narenas = tikv_jemalloc_ctl::arenas::narenas::read()
341 .ok()
342 .map(|v| v as usize);
343 HostHeapState {
344 active_bytes,
345 allocated_bytes,
346 resident_bytes,
347 mapped_bytes,
348 narenas,
349 }
350}
351
352#[cfg(test)]
353mod tests {
354 use super::*;
355
356 #[test]
357 fn default_lands_all_none() {
358 let h = HostHeapState::default();
359 assert!(h.active_bytes.is_none());
360 assert!(h.allocated_bytes.is_none());
361 assert!(h.resident_bytes.is_none());
362 assert!(h.mapped_bytes.is_none());
363 assert!(h.narenas.is_none());
364 }
365
366 #[test]
367 fn test_fixture_populates_every_field() {
368 let h = HostHeapState::test_fixture();
369 assert!(h.active_bytes.is_some());
370 assert!(h.allocated_bytes.is_some());
371 assert!(h.resident_bytes.is_some());
372 assert!(h.mapped_bytes.is_some());
373 assert!(h.narenas.is_some());
374 }
375
376 #[test]
377 fn format_human_lists_every_field() {
378 let out = HostHeapState::test_fixture().format_human();
379 assert!(out.contains("allocated_bytes:"));
380 assert!(out.contains("active_bytes:"));
381 assert!(out.contains("resident_bytes:"));
382 assert!(out.contains("mapped_bytes:"));
383 assert!(out.contains("narenas:"));
384 assert!(out.ends_with('\n'));
385 }
386
387 /// Snapshot-style pin of the label sequence `format_human`
388 /// emits. Mirrors
389 /// `host_context::tests::format_human_field_order_is_stable` —
390 /// `HostContext::format_human` embeds this block indented under
391 /// the `heap_state:` parent label, and downstream diff tools +
392 /// operator-eye scanning depend on a stable
393 /// `allocated → active → resident → mapped → narenas`
394 /// top-to-bottom ordering. A silent reorder from a future edit
395 /// that shuffles the `row(...)` calls inside `format_human`
396 /// would slip past the order-blind `.contains(...)` checks in
397 /// the sibling tests. This test fails the moment the sequence
398 /// drifts; updating it forces the author to acknowledge the
399 /// reorder and double-check the HostContext ctprof sub-block
400 /// still reads coherently.
401 #[test]
402 fn format_human_field_order_is_stable() {
403 let out = HostHeapState::default().format_human();
404 let labels: Vec<&str> = out
405 .lines()
406 .filter_map(|l| l.split(':').next())
407 .filter(|s| !s.starts_with(' '))
408 .collect();
409 assert_eq!(
410 labels,
411 vec![
412 "allocated_bytes",
413 "active_bytes",
414 "resident_bytes",
415 "mapped_bytes",
416 "narenas",
417 ],
418 "format_human field order drifted — if intentional, update \
419 the expected vector and verify the HostContext heap_state \
420 sub-block still reads in the expected top-to-bottom order",
421 );
422 }
423
424 #[test]
425 fn format_human_renders_none_as_unknown() {
426 let out = HostHeapState::default().format_human();
427 // Every line should end with `: (unknown)`.
428 for line in out.lines() {
429 assert!(
430 line.ends_with(": (unknown)"),
431 "expected unknown, got {line:?}"
432 );
433 }
434 }
435
436 #[test]
437 fn diff_is_empty_on_equal_snapshots() {
438 let a = HostHeapState::test_fixture();
439 let b = HostHeapState::test_fixture();
440 assert_eq!(a.diff(&b), "");
441 }
442
443 #[test]
444 fn diff_reports_only_changed_fields() {
445 let a = HostHeapState::test_fixture();
446 let mut b = a.clone();
447 b.allocated_bytes = Some(9 * 1024 * 1024);
448 let d = a.diff(&b);
449 assert!(d.contains("allocated_bytes:"));
450 assert!(!d.contains("active_bytes:"));
451 assert!(!d.contains("resident_bytes:"));
452 assert!(!d.contains("mapped_bytes:"));
453 assert!(!d.contains("narenas:"));
454 assert!(d.contains("→"));
455 }
456
457 #[test]
458 fn diff_renders_none_transitions() {
459 let a = HostHeapState::default();
460 let b = HostHeapState::test_fixture();
461 let d = a.diff(&b);
462 // Every field changed, every line should carry the unknown→x arrow.
463 assert!(d.contains("allocated_bytes: (unknown) →"));
464 assert!(d.contains("narenas: (unknown) →"));
465 }
466
467 #[test]
468 fn diff_renders_some_to_none_transitions() {
469 // Symmetric case to `diff_renders_none_transitions`: a full
470 // fixture diffed against `default()` must surface each field
471 // as `x → (unknown)`, not be silently absorbed. Without this
472 // test a one-sided `(unknown) → x` match could mask a
473 // formatting bug in the reverse direction (e.g. the renderer
474 // inadvertently suppressing `Some → None` as unchanged).
475 let a = HostHeapState::test_fixture();
476 let b = HostHeapState::default();
477 let d = a.diff(&b);
478 assert!(
479 d.contains("allocated_bytes:") && d.contains("→ (unknown)"),
480 "expected allocated_bytes → (unknown), got:\n{d}",
481 );
482 assert!(d.contains("active_bytes:"));
483 assert!(d.contains("resident_bytes:"));
484 assert!(d.contains("mapped_bytes:"));
485 assert!(d.contains("narenas:"));
486 }
487
488 #[test]
489 fn serde_round_trip_preserves_fields() {
490 let h = HostHeapState::test_fixture();
491 let s = serde_json::to_string(&h).expect("serialize");
492 let back: HostHeapState = serde_json::from_str(&s).expect("deserialize");
493 assert_eq!(back, h);
494 }
495
496 #[test]
497 fn serde_skips_none_fields() {
498 let h = HostHeapState::default();
499 let s = serde_json::to_string(&h).expect("serialize");
500 // Every field is None and skip_serializing_if drops all of
501 // them, so the JSON object is empty.
502 assert_eq!(s, "{}");
503 }
504
505 #[test]
506 fn serde_accepts_missing_fields_via_defaults() {
507 // Older sidecar with no heap_state fields still
508 // deserializes to a valid (all-None) snapshot.
509 let back: HostHeapState = serde_json::from_str("{}").expect("deserialize");
510 assert_eq!(back, HostHeapState::default());
511 }
512
513 /// Under the library-crate test harness, `tikv-jemallocator` is
514 /// NOT installed as `#[global_allocator]` — the ktstr library
515 /// itself declares no allocator so downstream consumers can
516 /// pick their own. Libjemalloc is still linked (hard dep of
517 /// `tikv-jemalloc-ctl`) and `collect()` initializes it, so its
518 /// own small metadata allocations may make `stats.allocated`
519 /// and `stats.active` non-zero even though Rust allocations use
520 /// libc.
521 ///
522 /// Under this shape, the jemalloc invariants only describe its
523 /// own initialization. They do NOT validate the application's
524 /// allocator behavior.
525 /// Real invariant coverage lives in the shipped ktstr binaries,
526 /// whose crate roots declare `tikv_jemallocator::Jemalloc` as the
527 /// `#[global_allocator]`; a
528 /// live production run of any of those binaries exercises the
529 /// non-trivial invariants. Documenting rather than
530 /// feature-gating because a lib-crate integration test with its
531 /// own `#[global_allocator]` would add its own binary target
532 /// and is heavier than the coverage warrants.
533 ///
534 /// What this test DOES pin, non-trivially:
535 /// - libjemalloc-linked-build contract: `narenas` is always
536 /// populated after `epoch::advance()` because arena count is
537 /// a jemalloc-init-time constant, independent of whether
538 /// jemalloc served any allocations.
539 /// - `collect()` infallibility on a libjemalloc build — the
540 /// epoch::advance defensive guard does not fire.
541 #[test]
542 fn collect_populates_linked_jemalloc_without_selecting_it() {
543 let h = collect();
544 // libjemalloc is linked unconditionally, so every field
545 // populates. `narenas` in particular is a jemalloc-init
546 // constant (4*ncpus by default) and always non-zero on a
547 // multi-core host.
548 assert!(
549 h.narenas.is_some(),
550 "narenas must populate on a libjemalloc build"
551 );
552 assert!(
553 h.narenas.unwrap() > 0,
554 "narenas must be > 0; jemalloc computes 4*ncpus at init",
555 );
556 // stats reads also populate (they're not None) because
557 // `--enable-stats` is forced by the `stats` feature in
558 // Cargo.toml. Their VALUES depend on whether jemalloc is
559 // `#[global_allocator]` — see the doc comment on this test.
560 assert!(h.allocated_bytes.is_some());
561 assert!(h.active_bytes.is_some());
562 assert!(h.resident_bytes.is_some());
563 assert!(h.mapped_bytes.is_some());
564 assert!(
565 !jemalloc_is_global_allocator(),
566 "the allocator-agnostic library test binary must not be marked as a jemalloc-global process",
567 );
568 let before = h.allocated_bytes.expect("allocated populated");
569 let held = vec![0u8; 8 * 1024 * 1024];
570 std::hint::black_box(&held);
571 let after = collect().allocated_bytes.expect("allocated populated");
572 assert!(
573 after.saturating_sub(before) < 1024 * 1024,
574 "an 8 MiB Rust allocation must not flow through the linked-but-unselected jemalloc: before={before}, after={after}",
575 );
576 }
577}