ktstr/monitor/dump/
display.rs

1//! Human-readable [`std::fmt::Display`] impls for the failure-dump
2//! types.
3//!
4//! All `Display` impls for [`super::FailureDumpReport`],
5//! [`super::DualFailureDumpReport`], [`super::FailureDumpReportAny`],
6//! [`super::DegradedFailureDumpReport`], [`super::FailureDumpMap`],
7//! [`super::FailureDumpEntry`],
8//! [`super::FailureDumpArrayEntry`], [`super::FailureDumpPercpuEntry`],
9//! and [`super::FailureDumpPercpuHashEntry`] live here so the type
10//! definitions in [`super`] stay focused on the data shape and the
11//! formatting concerns are isolated in one place.
12//!
13//! JSON remains the programmatic form via `serde_json`; these impls
14//! are the default presentation used in test-failure output.
15
16use super::super::btf_render::{
17    RenderedMember, RenderedValue, is_inline_scalar, is_zero, write_value_at_depth,
18};
19use super::{
20    DegradedFailureDumpReport, DualFailureDumpReport, EventCounterSample, FailureDumpArrayEntry,
21    FailureDumpEntry, FailureDumpMap, FailureDumpPercpuEntry, FailureDumpPercpuHashEntry,
22    FailureDumpReport, FailureDumpReportAny, render_sparkline_i64,
23};
24
25/// Minimum entry count for [`FailureDumpMap`] table rendering. Two
26/// entries are still meaningful as a table (column headers vs two
27/// per-entry blocks is denser); below that the table header
28/// overhead exceeds the savings.
29const TABLE_MIN_ENTRIES: usize = 2;
30
31/// Try to render a [`FailureDumpMap`]'s `entries` as a homogeneous
32/// table. Returns `Ok(true)` when the table was rendered (caller
33/// must skip per-entry rendering); `Ok(false)` when the entries
34/// don't qualify (caller falls through to per-entry rendering).
35///
36/// Table eligibility (every condition required):
37///   - at least [`TABLE_MIN_ENTRIES`] entries,
38///   - every entry has BOTH `key.is_some()` AND `value.is_some()`,
39///   - no entry has a `payload` (typed sdt_alloc payloads need
40///     block rendering below the entry; the table format can't
41///     represent them),
42///   - every key is a `RenderedValue::Struct` with the same
43///     `type_name` and same member names AND every member is an
44///     inline scalar (matches [`is_inline_scalar`]'s definition),
45///   - every value is a `RenderedValue::Struct` with the same
46///     `type_name` and same member names AND every member is an
47///     inline scalar.
48///
49/// Output shape, with `|` separating key columns from value columns
50/// and numeric columns right-aligned:
51/// ```text
52///   cgrp_id  llc_id | llcx
53///         1       5 | 17592186046336
54///        61       3 | 17592186047616
55/// ```
56fn try_write_entry_table(
57    f: &mut std::fmt::Formatter<'_>,
58    entries: &[FailureDumpEntry],
59) -> Result<bool, std::fmt::Error> {
60    if entries.len() < TABLE_MIN_ENTRIES {
61        return Ok(false);
62    }
63    // Reject if any entry has a payload — the typed payload renders
64    // in a block below the entry; the table can't carry it without
65    // breaking the row layout.
66    if entries.iter().any(|e| e.payload.is_some()) {
67        return Ok(false);
68    }
69    // Collect every entry's key + value as Struct references. Any
70    // missing render (None key or None value) bails immediately —
71    // a hex-only entry can't be a table cell.
72    let pairs: Option<Vec<(&Vec<RenderedMember>, &Vec<RenderedMember>)>> = entries
73        .iter()
74        .map(|e| match (&e.key, &e.value) {
75            (
76                Some(RenderedValue::Struct { members: k, .. }),
77                Some(RenderedValue::Struct { members: v, .. }),
78            ) => Some((k, v)),
79            _ => None,
80        })
81        .collect();
82    let Some(pairs) = pairs else {
83        return Ok(false);
84    };
85    if pairs.is_empty() {
86        return Ok(false);
87    }
88
89    // Type names + member names must match across every entry. The
90    // first entry sets the template; subsequent entries must agree.
91    let (first_key_name, first_value_name) = match (&entries[0].key, &entries[0].value) {
92        (
93            Some(RenderedValue::Struct { type_name: kn, .. }),
94            Some(RenderedValue::Struct { type_name: vn, .. }),
95        ) => (kn.clone(), vn.clone()),
96        _ => return Ok(false),
97    };
98    for e in &entries[1..] {
99        match (&e.key, &e.value) {
100            (
101                Some(RenderedValue::Struct { type_name: kn, .. }),
102                Some(RenderedValue::Struct { type_name: vn, .. }),
103            ) => {
104                if *kn != first_key_name || *vn != first_value_name {
105                    return Ok(false);
106                }
107            }
108            _ => return Ok(false),
109        }
110    }
111
112    let (k0, v0) = pairs[0];
113    let key_names: Vec<&str> = k0.iter().map(|m| m.name.as_str()).collect();
114    let value_names: Vec<&str> = v0.iter().map(|m| m.name.as_str()).collect();
115
116    // Member names + counts must match across every entry. A
117    // mismatch means the structs aren't actually homogeneous (the
118    // BTF rendered different fields per entry — possible if the
119    // entry value type id changed mid-iteration, or a torn read
120    // produced a Truncated partial inside the Struct).
121    for (k, v) in &pairs {
122        if k.len() != k0.len() || v.len() != v0.len() {
123            return Ok(false);
124        }
125        for (a, b) in k.iter().zip(k0.iter()) {
126            if a.name != b.name {
127                return Ok(false);
128            }
129        }
130        for (a, b) in v.iter().zip(v0.iter()) {
131            if a.name != b.name {
132                return Ok(false);
133            }
134        }
135    }
136
137    // Every member in every entry's key + value must be an inline
138    // scalar. A composite member (Struct, Array, CpuList, etc.)
139    // breaks the single-line-per-row contract.
140    for (k, v) in &pairs {
141        if !k.iter().all(|m| is_inline_scalar(&m.value)) {
142            return Ok(false);
143        }
144        if !v.iter().all(|m| is_inline_scalar(&m.value)) {
145            return Ok(false);
146        }
147    }
148
149    // Pre-render every cell so column widths can be measured. A
150    // numeric cell uses the same Display impl that produces
151    // "<value>" (e.g. "1024") so widths reflect the rendered
152    // form, not the raw integer.
153    let key_rows: Vec<Vec<String>> = pairs
154        .iter()
155        .map(|(k, _)| k.iter().map(|m| format!("{}", m.value)).collect())
156        .collect();
157    let value_rows: Vec<Vec<String>> = pairs
158        .iter()
159        .map(|(_, v)| v.iter().map(|m| format!("{}", m.value)).collect())
160        .collect();
161
162    // Per-column width: max of header (member name) and any cell
163    // in the column. Header is the member name; cells come from
164    // the pre-rendered row vectors.
165    let key_widths: Vec<usize> = (0..key_names.len())
166        .map(|c| {
167            let cell_max = key_rows.iter().map(|r| r[c].len()).max().unwrap_or(0);
168            key_names[c].len().max(cell_max)
169        })
170        .collect();
171    let value_widths: Vec<usize> = (0..value_names.len())
172        .map(|c| {
173            let cell_max = value_rows.iter().map(|r| r[c].len()).max().unwrap_or(0);
174            value_names[c].len().max(cell_max)
175        })
176        .collect();
177
178    // Header row: key names | value names.
179    f.write_str("\n  ")?;
180    for (i, name) in key_names.iter().enumerate() {
181        if i > 0 {
182            f.write_str("  ")?;
183        }
184        write!(f, "{:>width$}", name, width = key_widths[i])?;
185    }
186    f.write_str(" | ")?;
187    for (i, name) in value_names.iter().enumerate() {
188        if i > 0 {
189            f.write_str("  ")?;
190        }
191        write!(f, "{:>width$}", name, width = value_widths[i])?;
192    }
193
194    // Data rows. Right-align every cell to the column width — the
195    // values are scalar Display output (integers, hex pointers,
196    // booleans), and right-alignment makes a vertical tens/hundreds
197    // alignment immediately readable.
198    for (key_row, value_row) in key_rows.iter().zip(value_rows.iter()) {
199        f.write_str("\n  ")?;
200        for (i, cell) in key_row.iter().enumerate() {
201            if i > 0 {
202                f.write_str("  ")?;
203            }
204            write!(f, "{:>width$}", cell, width = key_widths[i])?;
205        }
206        f.write_str(" | ")?;
207        for (i, cell) in value_row.iter().enumerate() {
208            if i > 0 {
209                f.write_str("  ")?;
210            }
211            write!(f, "{:>width$}", cell, width = value_widths[i])?;
212        }
213    }
214
215    Ok(true)
216}
217
218impl std::fmt::Display for DualFailureDumpReport {
219    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
220        // Summary header: a one-line at-a-glance description so an
221        // operator scanning logs sees the shape (early present /
222        // absent, late map + vcpu_regs counts, plus the trigger
223        // metric and threshold when early fired) before paging
224        // through the full body.
225        let n_maps = self.late.maps.len();
226        let m_vcpu_regs = self.late.vcpu_regs.len();
227        if self.early.is_some() {
228            // Both jiffies fields zero means the early-snapshot
229            // bookkeeping never recorded a trigger metric (e.g. the
230            // snapshot was attached without the runnable_at scan
231            // populating the threshold). Render a distinct line so
232            // operators don't read "max_age=0j, threshold=0j" as a
233            // legitimate sub-tick trigger.
234            if self.early_max_age_jiffies == 0 && self.early_threshold_jiffies == 0 {
235                write!(
236                    f,
237                    "DualFailureDumpReport: early=present (jiffies not captured), \
238                     late=({n_maps} maps, {m_vcpu_regs} vcpu_regs)\n\n",
239                )?;
240            } else {
241                write!(
242                    f,
243                    "DualFailureDumpReport: early=present (max_age={}j, threshold={}j), \
244                     late=({n_maps} maps, {m_vcpu_regs} vcpu_regs)\n\n",
245                    self.early_max_age_jiffies, self.early_threshold_jiffies,
246                )?;
247            }
248        } else if let Some(reason) = self.early_skipped_reason.as_deref() {
249            // Structured reason populated by the freeze coordinator
250            // (one of: scan prerequisites unavailable, max_age never
251            // crossed threshold, scx_tick stall — see
252            // `DualFailureDumpReport::early_skipped_reason` doc for
253            // the full set). Surface it directly so the operator
254            // does not have to re-run with RUST_LOG=ktstr=debug to
255            // recover the cause.
256            write!(
257                f,
258                "DualFailureDumpReport: early=absent ({reason}), \
259                 late=({n_maps} maps, {m_vcpu_regs} vcpu_regs)\n\n",
260            )?;
261        } else {
262            // Legacy generic message. Reached only on dumps written
263            // before the freeze coordinator started populating
264            // `early_skipped_reason` (no field on the JSON, deserialised
265            // as None). Keep the RUST_LOG hint so old dumps remain
266            // actionable; new dumps take the structured branch above.
267            write!(
268                f,
269                "DualFailureDumpReport: early=absent, late=({n_maps} maps, \
270                 {m_vcpu_regs} vcpu_regs)\n\n",
271            )?;
272        }
273        match &self.early {
274            Some(early) => {
275                f.write_str("early snapshot (sched_ext watchdog half-way):\n")?;
276                std::fmt::Display::fmt(early, f)?;
277                f.write_str("\n\nlate snapshot (error-exit):\n")?;
278                std::fmt::Display::fmt(&self.late, f)
279            }
280            None => {
281                if let Some(reason) = self.early_skipped_reason.as_deref() {
282                    writeln!(
283                        f,
284                        "late snapshot (error-exit; early snapshot absent: \
285                         {reason}):",
286                    )?;
287                } else {
288                    f.write_str(
289                        "late snapshot (error-exit; early snapshot absent \
290                         (stall fired before half-way threshold, or runnable_at \
291                         scan setup failed) — re-run with RUST_LOG=ktstr=debug \
292                         for scan resolution diagnostics):\n",
293                    )?;
294                }
295                std::fmt::Display::fmt(&self.late, f)
296            }
297        }
298    }
299}
300
301impl std::fmt::Display for FailureDumpReportAny {
302    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
303        match self {
304            Self::Single(r) => std::fmt::Display::fmt(r, f),
305            Self::Dual(r) => std::fmt::Display::fmt(r.as_ref(), f),
306            Self::Degraded(r) => std::fmt::Display::fmt(r.as_ref(), f),
307        }
308    }
309}
310
311impl std::fmt::Display for DegradedFailureDumpReport {
312    /// Renders the degraded report as a short operator-oriented
313    /// banner: schema label, the human reason, the per-vCPU
314    /// `parked` / `not_parked` pattern that identifies which vCPUs
315    /// stalled, the watchpoint + bss latch state, the optional
316    /// live `exit_kind`, and the elapsed-ms budget the coordinator
317    /// spent before giving up. Designed to fit a single terminal
318    /// scroll without paging — the full diagnostic surface lives
319    /// in the structured fields.
320    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
321        f.write_str("degraded failure dump:\n")?;
322        writeln!(f, "  reason: {}", self.reason)?;
323        if !self.vcpu_regs.is_empty() {
324            let parked = self.vcpu_regs.iter().filter(|s| s.is_some()).count();
325            let total = self.vcpu_regs.len();
326            writeln!(f, "  vcpus_parked: {parked}/{total}")?;
327            for (i, slot) in self.vcpu_regs.iter().enumerate() {
328                match slot {
329                    Some(s) => writeln!(f, "    vcpu {i}: parked, {s}")?,
330                    None => writeln!(f, "    vcpu {i}: not_parked")?,
331                }
332            }
333        }
334        writeln!(f, "  watchpoint_hit: {}", self.watchpoint_hit)?;
335        writeln!(f, "  bss_latch_state: {}", self.bss_latch_state)?;
336        if let Some(kind) = self.exit_kind {
337            writeln!(f, "  exit_kind: {kind}")?;
338        }
339        if self.elapsed_ms != 0 {
340            writeln!(f, "  elapsed_ms: {}", self.elapsed_ms)?;
341        }
342        Ok(())
343    }
344}
345
346impl FailureDumpReport {
347    /// Renders the scx walker section: per-CPU rq->scx / DSQ / scx_sched
348    /// counts, then the walker-unavailable reason when the walk failed.
349    /// Renders the pre-capture guest-setup-failure evidence: stage,
350    /// final phase, exit code, attach outcome, then the embedded log
351    /// tails (scheduler log first — it carries the proximate error when
352    /// the scheduler died on its own argv/config).
353    fn fmt_guest_setup_failure(
354        &self,
355        f: &mut std::fmt::Formatter<'_>,
356        first: &mut bool,
357    ) -> std::fmt::Result {
358        let Some(context) = self
359            .guest_setup_failure
360            .as_ref()
361            .filter(|context| !context.is_empty())
362        else {
363            return Ok(());
364        };
365        if !*first {
366            f.write_str("\n\n")?;
367        }
368        *first = false;
369        f.write_str("guest setup failure:")?;
370        if let Some(stage) = &context.init_stage {
371            write!(f, "\n  stage: {stage}")?;
372        }
373        if let Some(phase) = &context.final_guest_phase {
374            write!(f, "\n  final_guest_phase: {phase}")?;
375        }
376        if let Some(code) = context.guest_exit_code {
377            write!(f, "\n  guest_exit_code: {code}")?;
378        }
379        if let Some(attach) = &context.scheduler_attach_outcome {
380            write!(f, "\n  scheduler_attach: {attach}")?;
381        }
382        for (label, tail) in [
383            ("scheduler log tail", &context.scheduler_log_tail),
384            ("init log tail (COM2)", &context.init_log_tail),
385            ("console tail (COM1)", &context.console_tail),
386        ] {
387            if let Some(tail) = tail {
388                write!(f, "\n  {label}:")?;
389                for line in tail.lines() {
390                    write!(f, "\n    {line}")?;
391                }
392            }
393        }
394        Ok(())
395    }
396
397    fn fmt_scx_walker(
398        &self,
399        f: &mut std::fmt::Formatter<'_>,
400        first: &mut bool,
401    ) -> std::fmt::Result {
402        if !self.rq_scx_states.is_empty()
403            || !self.dsq_states.is_empty()
404            || self.scx_sched_state.is_some()
405        {
406            if !*first {
407                f.write_str("\n\n")?;
408            }
409            *first = false;
410            // scx walker section: counts of each sub-walk's output.
411            // JSON carries the full per-CPU rq->scx state, per-DSQ
412            // task lists, and scx_sched scalars; the Display surface
413            // tells the operator what the walker reached.
414            write!(
415                f,
416                "scx_walker: rq_scx={} dsq={} sched={}",
417                self.rq_scx_states.len(),
418                self.dsq_states.len(),
419                if self.scx_sched_state.is_some() {
420                    "captured"
421                } else {
422                    "absent"
423                },
424            )?;
425        }
426        if !self.is_placeholder
427            && let Some(reason) = &self.scx_walker_unavailable
428        {
429            if !*first {
430                f.write_str("\n\n")?;
431            }
432            *first = false;
433            write!(f, "scx_walker: <unavailable: {reason}>")?;
434        }
435        Ok(())
436    }
437
438    /// Renders the event-counter timeline: sample-count header plus a
439    /// per-counter sparkline row for each non-zero SCX_EV_* counter.
440    fn fmt_event_counter_timeline(
441        &self,
442        f: &mut std::fmt::Formatter<'_>,
443        first: &mut bool,
444    ) -> std::fmt::Result {
445        if !self.event_counter_timeline.is_empty() {
446            if !*first {
447                f.write_str("\n\n")?;
448            }
449            *first = false;
450            // Clear `first` like the sibling sections so a dump whose
451            // only populated sections are this one and
452            // vcpu_perf_at_freeze still renders the blank-line
453            // separator between them. The per-counter sparkline rows
454            // below append their own `\n`-prefixed lines.
455            write!(
456                f,
457                "event_counter_timeline: {} samples ({}–{}ms)",
458                self.event_counter_timeline.len(),
459                self.event_counter_timeline
460                    .first()
461                    .map(|s| s.elapsed_ms)
462                    .unwrap_or(0),
463                self.event_counter_timeline
464                    .last()
465                    .map(|s| s.elapsed_ms)
466                    .unwrap_or(0),
467            )?;
468            // Per-counter sparkline. Each row is one of the 13
469            // SCX_EV_* counters across all samples in the
470            // timeline. Skips counters that stayed at zero across
471            // every sample to keep the rendering compact (a
472            // counter at zero everywhere has no signal worth
473            // surfacing in the human-readable view).
474            type EventCounterExtract = (&'static str, fn(&EventCounterSample) -> i64);
475            let extract: [EventCounterExtract; 13] = [
476                ("select_cpu_fallback", |s| s.select_cpu_fallback),
477                ("dispatch_local_dsq_offline", |s| {
478                    s.dispatch_local_dsq_offline
479                }),
480                ("dispatch_keep_last", |s| s.dispatch_keep_last),
481                ("enq_skip_exiting", |s| s.enq_skip_exiting),
482                ("enq_skip_migration_disabled", |s| {
483                    s.enq_skip_migration_disabled
484                }),
485                ("reenq_immed", |s| s.reenq_immed),
486                ("reenq_local_repeat", |s| s.reenq_local_repeat),
487                ("refill_slice_dfl", |s| s.refill_slice_dfl),
488                ("bypass_duration", |s| s.bypass_duration),
489                ("bypass_dispatch", |s| s.bypass_dispatch),
490                ("bypass_activate", |s| s.bypass_activate),
491                ("insert_not_owned", |s| s.insert_not_owned),
492                ("sub_bypass_dispatch", |s| s.sub_bypass_dispatch),
493            ];
494            for (name, ext) in extract {
495                let series: Vec<i64> = self.event_counter_timeline.iter().map(ext).collect();
496                if series.iter().all(|&v| v == 0) {
497                    continue;
498                }
499                let line = render_sparkline_i64(&series);
500                let last = series.last().copied().unwrap_or(0);
501                write!(f, "\n  {name:>30}  {line}  (last={last})")?;
502            }
503        }
504        Ok(())
505    }
506
507    /// Renders the per-vCPU performance-counter snapshot captured at
508    /// freeze: cycles / instructions / IPC / cache + branch misses.
509    fn fmt_vcpu_perf_at_freeze(
510        &self,
511        f: &mut std::fmt::Formatter<'_>,
512        first: &mut bool,
513    ) -> std::fmt::Result {
514        if !self.vcpu_perf_at_freeze.is_empty() {
515            if !*first {
516                f.write_str("\n\n")?;
517            }
518            *first = false;
519            f.write_str("vcpu_perf_at_freeze:")?;
520            for (i, slot) in self.vcpu_perf_at_freeze.iter().enumerate() {
521                f.write_str("\n  ")?;
522                match slot {
523                    Some(s) => write!(
524                        f,
525                        "vcpu {i}: cycles={} insns={} ipc={:.3} cache_misses={} branch_misses={} (en/ru={}/{} ns)",
526                        s.cycles,
527                        s.instructions,
528                        s.ipc(),
529                        s.cache_misses,
530                        s.branch_misses,
531                        s.time_enabled_ns,
532                        s.time_running_ns,
533                    )?,
534                    None => write!(f, "vcpu {i}: <unavailable>")?,
535                }
536            }
537        }
538        Ok(())
539    }
540}
541
542impl std::fmt::Display for FailureDumpReport {
543    /// Human-readable rendering of every map plus per-vCPU register
544    /// snapshots, per-program runtime stats, per-CPU CPU-time /
545    /// softirq / IRQ counters, per-node NUMA stats, per-task
546    /// enrichments, scx walker output (rq->scx, DSQ, scx_sched
547    /// state), and event-counter timeline. JSON remains the
548    /// programmatic form via `serde_json`; this Display is the
549    /// default presentation used in test-failure output.
550    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
551        if self.maps.is_empty()
552            && self.vcpu_regs.is_empty()
553            && self.sdt_allocations.is_empty()
554            && self.scx_static_ranges.is_empty()
555            && self.prog_runtime_stats.is_empty()
556            && self.prog_runtime_stats_unavailable.is_none()
557            && self.per_cpu_time.is_empty()
558            && self.per_node_numa.is_empty()
559            && self.per_node_numa_unavailable.is_none()
560            && self.task_enrichments.is_empty()
561            && self.task_enrichments_unavailable.is_none()
562            && self.event_counter_timeline.is_empty()
563            && self.rq_scx_states.is_empty()
564            && self.dsq_states.is_empty()
565            && self.scx_sched_state.is_none()
566            && self.scx_walker_unavailable.is_none()
567            && self.vcpu_perf_at_freeze.is_empty()
568            && self.dump_truncated_at_us.is_none()
569            && self.maps_truncated == 0
570            && self.sdt_alloc_unavailable.is_none()
571            && self
572                .guest_setup_failure
573                .as_ref()
574                .is_none_or(|context| context.is_empty())
575        {
576            return f.write_str("(empty failure dump)");
577        }
578        // A failure dump is rendered on the retry/error path, where
579        // pthread creation may already be the scarce resource. Using
580        // `par_iter()` here lazily initialized Rayon's process-global
581        // host-sized pool and retained every worker for the rest of the
582        // test process. Render in the same stable map order without
583        // creating any helper threads. Keeping the pre-rendered `String`
584        // vector preserves the prior byte output and write/error boundary.
585        let rendered_maps: Vec<String> = self.maps.iter().map(|m| format!("{m}")).collect();
586        let mut first = true;
587        self.fmt_guest_setup_failure(f, &mut first)?;
588        // A placeholder duplicates one reason string across every
589        // `*_unavailable` field; render it once here and skip the
590        // per-section `<unavailable: ...>` repeats below.
591        if self.is_placeholder {
592            let reason = self
593                .prog_runtime_stats_unavailable
594                .as_deref()
595                .or(self.per_node_numa_unavailable.as_deref())
596                .or(self.task_enrichments_unavailable.as_deref())
597                .or(self.scx_walker_unavailable.as_deref())
598                .or(self.sdt_alloc_unavailable.as_deref());
599            if let Some(reason) = reason {
600                if !first {
601                    f.write_str("\n\n")?;
602                }
603                first = false;
604                write!(f, "capture unavailable: {reason}")?;
605            }
606        }
607        for s in &rendered_maps {
608            if !first {
609                f.write_str("\n\n")?;
610            }
611            first = false;
612            f.write_str(s)?;
613        }
614        if !self.vcpu_regs.is_empty() {
615            if !first {
616                f.write_str("\n\n")?;
617            }
618            first = false;
619            f.write_str("vcpu_regs:")?;
620            for (i, slot) in self.vcpu_regs.iter().enumerate() {
621                f.write_str("\n  ")?;
622                match slot {
623                    Some(s) => write!(f, "vcpu {i}: {s}")?,
624                    None => write!(f, "vcpu {i}: <unavailable>")?,
625                }
626            }
627        }
628        for snap in &self.sdt_allocations {
629            if !first {
630                f.write_str("\n\n")?;
631            }
632            first = false;
633            std::fmt::Display::fmt(snap, f)?;
634        }
635        if !self.is_placeholder
636            && let Some(reason) = &self.sdt_alloc_unavailable
637        {
638            if !first {
639                f.write_str("\n\n")?;
640            }
641            first = false;
642            write!(f, "sdt_allocations: <unavailable: {reason}>")?;
643        }
644        if !self.scx_static_ranges.is_empty() {
645            if !first {
646                f.write_str("\n\n")?;
647            }
648            first = false;
649            std::fmt::Display::fmt(&self.scx_static_ranges, f)?;
650        }
651        if !self.prog_runtime_stats.is_empty() {
652            if !first {
653                f.write_str("\n\n")?;
654            }
655            first = false;
656            f.write_str("prog_runtime_stats:")?;
657            for stats in &self.prog_runtime_stats {
658                f.write_str("\n  ")?;
659                std::fmt::Display::fmt(stats, f)?;
660            }
661        }
662        if !self.is_placeholder
663            && let Some(reason) = &self.prog_runtime_stats_unavailable
664        {
665            if !first {
666                f.write_str("\n\n")?;
667            }
668            first = false;
669            write!(f, "prog_runtime_stats: <unavailable: {reason}>")?;
670        }
671        if !self.per_cpu_time.is_empty() {
672            if !first {
673                f.write_str("\n\n")?;
674            }
675            first = false;
676            // Per-CPU CPU-time / softirq / IRQ summary. JSON carries
677            // the full per-CPU breakdown; this Display surfaces
678            // counts so the test-failure log shows what was captured
679            // without paging through every CPU.
680            write!(f, "per_cpu_time: {} CPUs captured", self.per_cpu_time.len())?;
681        }
682        if !self.per_node_numa.is_empty() {
683            if !first {
684                f.write_str("\n\n")?;
685            }
686            first = false;
687            write!(
688                f,
689                "per_node_numa: {} nodes captured",
690                self.per_node_numa.len()
691            )?;
692        }
693        if !self.is_placeholder
694            && let Some(reason) = &self.per_node_numa_unavailable
695        {
696            if !first {
697                f.write_str("\n\n")?;
698            }
699            first = false;
700            write!(f, "per_node_numa: <unavailable: {reason}>")?;
701        }
702        if !self.task_enrichments.is_empty() {
703            if !first {
704                f.write_str("\n\n")?;
705            }
706            first = false;
707            write!(
708                f,
709                "task_enrichments: {} tasks captured",
710                self.task_enrichments.len(),
711            )?;
712        }
713        if !self.is_placeholder
714            && let Some(reason) = &self.task_enrichments_unavailable
715        {
716            if !first {
717                f.write_str("\n\n")?;
718            }
719            first = false;
720            write!(f, "task_enrichments: <unavailable: {reason}>")?;
721        }
722        self.fmt_scx_walker(f, &mut first)?;
723        self.fmt_event_counter_timeline(f, &mut first)?;
724        self.fmt_vcpu_perf_at_freeze(f, &mut first)?;
725        // Truncation footer: the per-map render loop bounds the freeze
726        // window by skipping work once the soft deadline is crossed.
727        // Surface both WHEN (`dump_truncated_at_us`) and HOW MANY maps
728        // (`maps_truncated`) were dropped so a degraded dump reads as
729        // "incomplete by truncation" rather than "this is everything".
730        if self.dump_truncated_at_us.is_some() || self.maps_truncated > 0 {
731            if !first {
732                f.write_str("\n\n")?;
733            }
734            f.write_str("dump truncated:")?;
735            if let Some(us) = self.dump_truncated_at_us {
736                write!(f, " deadline crossed at {us}us")?;
737            }
738            if self.maps_truncated > 0 {
739                write!(f, " ({} map(s) skipped)", self.maps_truncated)?;
740            }
741        }
742        Ok(())
743    }
744}
745
746impl std::fmt::Display for FailureDumpMap {
747    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
748        // Render `map_type` as the symbolic `BPF_MAP_TYPE_<NAME>`
749        // suffix when known; fall through to the raw integer for
750        // forward-compatibility with kernels newer than this dump
751        // renderer. Operators see "type=ringbuf" instead of
752        // "type=27"; the rare unknown discriminant still surfaces
753        // a stable numeric handle.
754        let type_str: std::borrow::Cow<'_, str> =
755            match super::render_map::map_type_name(self.map_type) {
756                Some(name) => std::borrow::Cow::Borrowed(name),
757                None => std::borrow::Cow::Owned(format!("{}", self.map_type)),
758            };
759        write!(
760            f,
761            "map {} (type={}, value_size={}, max_entries={})",
762            self.name, type_str, self.value_size, self.max_entries
763        )?;
764        if let Some(err) = &self.error {
765            write!(f, " [error: {err}]")?;
766        }
767        if let Some(value) = &self.value {
768            f.write_str("\n")?;
769            std::fmt::Display::fmt(value, f)?;
770        }
771        // Try table layout for homogeneous entries: when every
772        // entry has BTF-rendered key + value, both sides are
773        // structs of the same shape, every member is an inline
774        // scalar, and no entry has a typed payload, render as a
775        // compact table. The table form replaces the per-entry
776        // `entry { ... }` blocks for the qualifying batch — a
777        // 30-entry hash map renders as 31 lines (header + 30 rows)
778        // instead of 30 four-line entry blocks.
779        if !try_write_entry_table(f, &self.entries)? {
780            for entry in &self.entries {
781                f.write_str("\n")?;
782                std::fmt::Display::fmt(entry, f)?;
783            }
784        }
785        for entry in &self.array_entries {
786            f.write_str("\n")?;
787            std::fmt::Display::fmt(entry, f)?;
788        }
789        for entry in &self.percpu_entries {
790            f.write_str("\n")?;
791            std::fmt::Display::fmt(entry, f)?;
792        }
793        for entry in &self.percpu_hash_entries {
794            f.write_str("\n")?;
795            std::fmt::Display::fmt(entry, f)?;
796        }
797        if let Some(arena) = &self.arena {
798            let total_pages = arena.pages.len();
799            let total_kib: usize = arena.pages.iter().map(|p| p.bytes.len()).sum::<usize>() / 1024;
800            let nonzero = arena
801                .pages
802                .iter()
803                .filter(|p| p.bytes.iter().any(|&b| b != 0))
804                .count();
805            write!(
806                f,
807                "\narena: {total_pages} pages captured ({total_kib} KiB), \
808                 {nonzero} non-zero (see sdt_alloc section + JSON for typed data)",
809            )?;
810            if arena.truncated {
811                write!(f, " (truncated, {} declared)", arena.declared_pages)?;
812            }
813        }
814        if let Some(rb) = &self.ringbuf {
815            // Show capacity, pending bytes (consumer lag), and the
816            // four position counters. The pending_pos vs producer_pos
817            // gap signals a producer mid-reserve; the pending_bytes
818            // vs capacity ratio signals consumer-stall pressure.
819            let pct = if rb.capacity == 0 {
820                0
821            } else {
822                (rb.pending_bytes.saturating_mul(100) / rb.capacity).min(100)
823            };
824            write!(
825                f,
826                "\nringbuf: capacity={}B, pending={}B ({pct}%), \
827                 consumer_pos={}, producer_pos={}, pending_pos={}",
828                rb.capacity, rb.pending_bytes, rb.consumer_pos, rb.producer_pos, rb.pending_pos,
829            )?;
830        }
831        if let Some(st) = &self.stack_trace {
832            write!(
833                f,
834                "\nstack_trace: {} of {} buckets populated",
835                st.entries.len(),
836                st.n_buckets,
837            )?;
838            if st.truncated {
839                f.write_str(" (truncated)")?;
840            }
841            if st.buckets_unreadable > 0 {
842                write!(f, " ({} unreadable)", st.buckets_unreadable)?;
843            }
844            for entry in &st.entries {
845                if entry.pcs.is_empty() {
846                    write!(f, "\n  bucket {}: nr={}", entry.bucket_id, entry.nr)?;
847                } else {
848                    // Show first up to 8 PCs as hex; full list is
849                    // in JSON. Write directly to the formatter
850                    // with a manual comma separator — no
851                    // intermediate Vec<String> + join allocation
852                    // per bucket. Stack-trace dumps with hundreds
853                    // of buckets compounded the per-bucket Vec
854                    // alloc into a measurable overhead.
855                    write!(f, "\n  bucket {}: nr={} pcs=[", entry.bucket_id, entry.nr)?;
856                    for (i, pc) in entry.pcs.iter().take(8).enumerate() {
857                        if i > 0 {
858                            f.write_str(", ")?;
859                        }
860                        write!(f, "{pc:#x}")?;
861                    }
862                    let extra = entry.pcs.len().saturating_sub(8);
863                    if extra > 0 {
864                        write!(f, ", +{extra} more")?;
865                    }
866                    f.write_str("]")?;
867                }
868            }
869        }
870        if let Some(fda) = &self.fd_array {
871            write!(
872                f,
873                "\nfd_array: {} of {} slots populated",
874                fda.populated, fda.scanned,
875            )?;
876            if fda.truncated {
877                f.write_str(" (slots truncated)")?;
878            }
879            if fda.unreadable > 0 {
880                write!(f, " ({} unreadable)", fda.unreadable)?;
881            }
882            if !fda.indices.is_empty() {
883                // Same pattern as stack-trace: stream directly
884                // to the formatter rather than allocating a Vec
885                // and joining.
886                f.write_str(" indices=[")?;
887                for (i, idx) in fda.indices.iter().take(16).enumerate() {
888                    if i > 0 {
889                        f.write_str(", ")?;
890                    }
891                    write!(f, "{idx}")?;
892                }
893                let extra = fda.indices.len().saturating_sub(16);
894                if extra > 0 {
895                    write!(f, ", +{extra} more")?;
896                }
897                f.write_str("]")?;
898            }
899        }
900        Ok(())
901    }
902}
903
904impl std::fmt::Display for FailureDumpEntry {
905    /// Render an entry using the indent-based format:
906    ///
907    /// ```text
908    /// entry: key=<rendered key>
909    ///   value: <rendered value>
910    ///   .data TypeName:
911    ///     field=val   field=val   field=val
912    /// ```
913    ///
914    /// `entry:` is a label and `key=` is a field assignment; the
915    /// `=` follows the field-assignment convention used elsewhere
916    /// in the dump output. `value:` is also a label introducing
917    /// the rendered value (which carries its own `TypeName{...}`
918    /// or breadcrumb form). The optional payload follows the
919    /// breadcrumb pattern: `.data <rendered>` where the value's
920    /// own Type breadcrumb completes the line.
921    ///
922    /// The renderer is invoked with `depth = 1` for the value and
923    /// payload positions so any multi-line struct / array body
924    /// indents one level deeper than the entry's own `  ` prefix.
925    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
926        // Header: `entry: key=<rendered>` on a single line. Small
927        // struct keys collapse to `Type{field=value}` via the
928        // btf_render inline path; non-struct keys (Uint, etc.)
929        // render as their own scalar form. The hex fallback path
930        // (no BTF render) emits the hex string with a trailing
931        // `(raw)` marker so the operator distinguishes "no BTF"
932        // from a parsed value.
933        f.write_str("entry: key=")?;
934        match &self.key {
935            Some(k) => write_value_at_depth(f, k, 1)?,
936            None => write!(f, "{} (raw)", self.key_hex)?,
937        }
938        // Value: `value: <rendered>` indented one level. The
939        // rendered value carries its own Type breadcrumb /
940        // braces form depending on size; depth=1 ensures any
941        // multi-line body indents two levels deep (4 spaces).
942        f.write_str("\n  value: ")?;
943        match &self.value {
944            Some(v) => write_value_at_depth(f, v, 1)?,
945            None => write!(f, "{} (raw)", self.value_hex)?,
946        }
947        // Typed sdt_alloc payload, when the entry value carried a
948        // `struct sdt_data __arena *` field that resolved into a
949        // captured arena page. Surfaced AFTER the surface value so
950        // the operator reads the surface struct first (with its
951        // tid / tptr / data fields) and then the typed payload —
952        // matching the order a kernel-side debugger would inspect:
953        // chase the pointer, then read the dereferenced struct.
954        // The space after `.data` lets the rendered value's own
955        // `TypeName:` breadcrumb (or inline `Type{...}` form) read
956        // as `.data TypeName:` on the same line.
957        if let Some(p) = &self.payload {
958            f.write_str("\n  .data ")?;
959            write_value_at_depth(f, p, 1)?;
960        }
961        Ok(())
962    }
963}
964
965impl std::fmt::Display for FailureDumpArrayEntry {
966    /// `key <N>: <rendered value>` — one line for a flat struct, the
967    /// value's own multi-line body indented one level (depth=1) for a
968    /// nested struct/array. `<unreadable>` marks a key whose guest
969    /// page was unmapped at the freeze instant.
970    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
971        write!(f, "key {}: ", self.key)?;
972        match &self.value {
973            Some(v) => write_value_at_depth(f, v, 1)?,
974            None => f.write_str("<unreadable>")?,
975        }
976        Ok(())
977    }
978}
979
980impl std::fmt::Display for FailureDumpPercpuEntry {
981    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
982        let structs: Vec<(usize, &[RenderedMember])> = self
983            .per_cpu
984            .iter()
985            .enumerate()
986            .filter_map(|(cpu, slot)| match slot {
987                Some(RenderedValue::Struct { members, .. }) => Some((cpu, members.as_slice())),
988                _ => None,
989            })
990            .collect();
991
992        if structs.len() < 2 || structs.iter().any(|(_, m)| m.is_empty()) {
993            write!(f, "key {}:", self.key)?;
994            for (cpu, slot) in self.per_cpu.iter().enumerate() {
995                f.write_str("\n")?;
996                match slot {
997                    Some(v) => {
998                        write!(f, "  cpu {cpu}: ")?;
999                        std::fmt::Display::fmt(v, f)?;
1000                    }
1001                    None => write!(f, "  cpu {cpu}: <unmapped>")?,
1002                }
1003            }
1004            return Ok(());
1005        }
1006
1007        let type_name = match &self.per_cpu.iter().flatten().next() {
1008            Some(RenderedValue::Struct { type_name, .. }) => type_name.clone(),
1009            _ => None,
1010        };
1011        let n_cpus = self.per_cpu.len();
1012        match &type_name {
1013            Some(name) => write!(f, "key {}: struct {name} ({n_cpus} CPUs)", self.key)?,
1014            None => write!(f, "key {}: ({n_cpus} CPUs)", self.key)?,
1015        }
1016
1017        // Skip the cross-CPU dedup pass on hosts with >64 CPUs:
1018        // the existing dedup loop is O(n²) in the number of
1019        // groups, so a 256-CPU host with mostly-unique values
1020        // ends up doing several million RenderedValue equality
1021        // comparisons (each itself a deep walk of the struct
1022        // tree). Above the threshold we just emit one row per
1023        // CPU — at scale, the table form would not fit in any
1024        // reasonable terminal anyway, and the per-CPU rows are
1025        // grep-friendly.
1026        const PERCPU_DEDUP_CPU_LIMIT: usize = 64;
1027        if n_cpus > PERCPU_DEDUP_CPU_LIMIT {
1028            for (cpu, slot) in self.per_cpu.iter().enumerate() {
1029                f.write_str("\n  ")?;
1030                match slot {
1031                    Some(v) => {
1032                        write!(f, "cpu {cpu}: ")?;
1033                        std::fmt::Display::fmt(v, f)?;
1034                    }
1035                    None => write!(f, "cpu {cpu}: <unmapped>")?,
1036                }
1037            }
1038            return Ok(());
1039        }
1040
1041        // Group CPUs by identical struct content. Show each unique
1042        // value once with its CPU list. The find() is O(n) per
1043        // CPU, which at 64 CPUs is bounded; the >64 fallback above
1044        // protects callers against the quadratic scaling.
1045        let mut groups: Vec<(Vec<usize>, &RenderedValue)> = Vec::new();
1046        let mut unmapped: Vec<usize> = Vec::new();
1047        for (cpu, slot) in self.per_cpu.iter().enumerate() {
1048            match slot {
1049                Some(val) => {
1050                    if let Some(g) = groups.iter_mut().find(|(_, v)| *v == val) {
1051                        g.0.push(cpu);
1052                    } else {
1053                        groups.push((vec![cpu], val));
1054                    }
1055                }
1056                None => unmapped.push(cpu),
1057            }
1058        }
1059        // Template detection: if every CPU has a unique struct but
1060        // most fields are identical, show the struct once with a
1061        // per-CPU table of varying fields.
1062        if groups.len() >= 3 && groups.iter().all(|(cpus, _)| cpus.len() == 1) {
1063            let all_structs: Vec<(usize, &[RenderedMember])> = groups
1064                .iter()
1065                .filter_map(|(cpus, val)| match val {
1066                    RenderedValue::Struct { members, .. } => Some((cpus[0], members.as_slice())),
1067                    _ => None,
1068                })
1069                .collect();
1070            if all_structs.len() == groups.len()
1071                && all_structs
1072                    .iter()
1073                    .all(|(_, m)| m.len() == all_structs[0].1.len())
1074            {
1075                let first = all_structs[0].1;
1076                let mut varying: Vec<usize> = Vec::new();
1077                for i in 0..first.len() {
1078                    if all_structs[1..]
1079                        .iter()
1080                        .any(|(_, m)| m[i].value != first[i].value)
1081                    {
1082                        varying.push(i);
1083                    }
1084                }
1085                if !varying.is_empty() && varying.len() < 8 {
1086                    // Show common fields once. Zero fields are
1087                    // suppressed silently — no count line.
1088                    f.write_str("\n  common:")?;
1089                    for (i, m) in first.iter().enumerate() {
1090                        if varying.contains(&i) {
1091                            continue;
1092                        }
1093                        if is_zero(&m.value) {
1094                            continue;
1095                        }
1096                        write!(f, "\n    {}: ", m.name)?;
1097                        std::fmt::Display::fmt(&m.value, f)?;
1098                    }
1099                    // Show varying fields as per-CPU table.
1100                    f.write_str("\n  per-cpu:")?;
1101                    f.write_str("\n    cpu")?;
1102                    for &vi in &varying {
1103                        write!(f, " | {}", first[vi].name)?;
1104                    }
1105                    for (cpu, members) in &all_structs {
1106                        write!(f, "\n    {cpu:>3}")?;
1107                        for &vi in &varying {
1108                            write!(f, " | {}", members[vi].value)?;
1109                        }
1110                    }
1111                    if !unmapped.is_empty() {
1112                        write!(f, "\n  cpus {unmapped:?}: <unmapped>")?;
1113                    }
1114                    return Ok(());
1115                }
1116            }
1117        }
1118        // Fallback: show each group with its CPU list.
1119        for (cpus, val) in &groups {
1120            let cpu_list = if cpus.len() == n_cpus {
1121                "all CPUs".to_string()
1122            } else if cpus.len() == 1 {
1123                format!("cpu {}", cpus[0])
1124            } else if cpus.windows(2).all(|w| w[1] == w[0] + 1) {
1125                // Contiguity: every adjacent pair differs by 1.
1126                // The endpoint-only `last - first + 1 == len` check
1127                // would falsely accept e.g. [0, 1, 1, 3] (span 4,
1128                // len 4, yet non-contiguous) if a duplicate or
1129                // gap somehow slipped past collection; `windows`
1130                // is robust to construction errors.
1131                format!("cpus {}-{}", cpus[0], cpus.last().unwrap())
1132            } else {
1133                format!("cpus {:?}", cpus)
1134            };
1135            write!(f, "\n  {cpu_list}: ")?;
1136            std::fmt::Display::fmt(val, f)?;
1137        }
1138        if !unmapped.is_empty() {
1139            write!(f, "\n  cpus {unmapped:?}: <unmapped>")?;
1140        }
1141        Ok(())
1142    }
1143}
1144
1145impl std::fmt::Display for FailureDumpPercpuHashEntry {
1146    /// Match [`FailureDumpEntry`]'s `entry: key=...` header so an
1147    /// operator scanning the human-readable failure dump sees the
1148    /// same shape regardless of whether the underlying map is a
1149    /// plain HASH or a PERCPU_HASH variant. Each per-CPU slot
1150    /// renders on its own indented line as `cpu N: <value>`.
1151    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1152        f.write_str("entry: key=")?;
1153        match &self.key {
1154            Some(k) => write_value_at_depth(f, k, 1)?,
1155            None => write!(f, "{} (raw)", self.key_hex)?,
1156        }
1157        // Per-CPU values: list each CPU's slot. Matches the
1158        // FailureDumpPercpuEntry simple branch (no group folding) for
1159        // readability — typical PERCPU_HASH maps have small
1160        // num_cpus * num_keys, so the verbose listing isn't a
1161        // problem in practice. CPU rows use `cpu N:` as a label
1162        // (the cpu id is metadata, not a struct field assignment).
1163        for (cpu, slot) in self.per_cpu.iter().enumerate() {
1164            f.write_str("\n  ")?;
1165            match slot {
1166                Some(v) => {
1167                    write!(f, "cpu {cpu}: ")?;
1168                    write_value_at_depth(f, v, 1)?;
1169                }
1170                None => write!(f, "cpu {cpu}: <unmapped>")?,
1171            }
1172        }
1173        Ok(())
1174    }
1175}
1176
1177#[cfg(test)]
1178mod tests {
1179    use super::*;
1180
1181    const DISPLAY_THREAD_PROBE_ENV: &str = "KTSTR_FAILURE_DUMP_DISPLAY_THREAD_PROBE";
1182    const DISPLAY_THREAD_PROBE_MARKER: &str = "failure-dump-display-thread-probe-ok";
1183
1184    /// Run the thread-count assertion in a fresh libtest process.
1185    ///
1186    /// Rayon's global pool is process-global and one-shot, so checking it
1187    /// in the ordinary parallel unit-test process would be order-dependent:
1188    /// an unrelated Rayon test may already have initialized the pool. The
1189    /// exact-test child starts with a private process and makes the before /
1190    /// after task count a deterministic regression signal.
1191    #[test]
1192    fn report_display_does_not_create_retained_workers() {
1193        if std::env::var_os(DISPLAY_THREAD_PROBE_ENV).is_some() {
1194            return;
1195        }
1196        let output = std::process::Command::new(std::env::current_exe().unwrap())
1197            .env(DISPLAY_THREAD_PROBE_ENV, "1")
1198            .arg("--exact")
1199            .arg(
1200                "monitor::dump::display::tests::\
1201                 report_display_does_not_create_retained_workers_child",
1202            )
1203            .arg("--nocapture")
1204            .arg("--test-threads=1")
1205            .output()
1206            .expect("spawn isolated failure-dump display probe");
1207        let transcript = format!(
1208            "{}{}",
1209            String::from_utf8_lossy(&output.stdout),
1210            String::from_utf8_lossy(&output.stderr)
1211        );
1212        assert!(
1213            output.status.success(),
1214            "isolated failure-dump display probe failed:\n{transcript}"
1215        );
1216        assert!(
1217            transcript.contains(DISPLAY_THREAD_PROBE_MARKER),
1218            "isolated helper did not run (wrong libtest exact name?):\n{transcript}"
1219        );
1220    }
1221
1222    #[test]
1223    fn report_display_does_not_create_retained_workers_child() {
1224        if std::env::var_os(DISPLAY_THREAD_PROBE_ENV).is_none() {
1225            return;
1226        }
1227        let task_count = || {
1228            std::fs::read_dir("/proc/self/task")
1229                .expect("read /proc/self/task")
1230                .count()
1231        };
1232        let report = FailureDumpReport {
1233            maps: (0..8)
1234                .map(|idx| FailureDumpMap {
1235                    name: format!("probe_map_{idx}"),
1236                    map_type: 2,
1237                    value_size: 8,
1238                    max_entries: 1,
1239                    ..Default::default()
1240                })
1241                .collect(),
1242            ..Default::default()
1243        };
1244        let before = task_count();
1245        let rendered = format!("{report}");
1246        let after = task_count();
1247        assert_eq!(
1248            rendered.matches("\n\n").count(),
1249            7,
1250            "all maps must retain their prior ordered separators"
1251        );
1252        assert_eq!(
1253            after, before,
1254            "failure-dump formatting retained worker threads: before={before}, after={after}"
1255        );
1256        eprintln!("{DISPLAY_THREAD_PROBE_MARKER}");
1257    }
1258
1259    /// Regression: when the event-counter timeline and
1260    /// vcpu_perf_at_freeze are the only populated sections, the
1261    /// timeline block must clear `first` so the blank-line separator
1262    /// still renders between the two. The timeline block previously
1263    /// left `first` set, which merged the two sections in this dump
1264    /// shape (every earlier section empty, so `first` was still true
1265    /// entering the timeline).
1266    #[test]
1267    fn report_display_event_counter_timeline_separated_from_vcpu_perf() {
1268        let report = FailureDumpReport {
1269            event_counter_timeline: vec![EventCounterSample {
1270                elapsed_ms: 1,
1271                select_cpu_fallback: 5,
1272                ..Default::default()
1273            }],
1274            vcpu_perf_at_freeze: vec![None],
1275            ..Default::default()
1276        };
1277        let out = format!("{report}");
1278        assert!(
1279            out.starts_with("event_counter_timeline: 1 samples (1–1ms)"),
1280            "timeline header missing: {out}"
1281        );
1282        assert!(
1283            out.contains("\n\nvcpu_perf_at_freeze:\n  vcpu 0: <unavailable>"),
1284            "missing blank-line separator before vcpu_perf section: {out}"
1285        );
1286    }
1287}