zng_app/
crash_handler.rs

1#![cfg(all(
2    feature = "crash_handler",
3    not(any(target_arch = "wasm32", target_os = "android", target_os = "ios"))
4))]
5
6//! App-process crash handler.
7//!
8//! See the `zng::app::crash_handler` documentation for more details.
9
10use parking_lot::Mutex;
11use std::{
12    fmt,
13    io::{BufRead, Write},
14    path::{Path, PathBuf},
15    sync::{Arc, atomic::AtomicBool},
16    time::SystemTime,
17};
18use zng_clone_move::clmv;
19use zng_layout::unit::TimeUnits as _;
20
21use zng_txt::{ToTxt as _, Txt};
22
23/// Environment variable that causes the crash handler to not start if set.
24///
25/// This is particularly useful to set in debugger launch configs. Crash handler spawns
26/// a different process for the app  so break points will not work.
27pub const NO_CRASH_HANDLER: &str = "ZNG_NO_CRASH_HANDLER";
28
29zng_env::on_process_start!(|process_start_args| {
30    if std::env::var(NO_CRASH_HANDLER).is_ok() {
31        return;
32    }
33    if zng_env::about().is_test {
34        tracing::debug!("ignoring crash_handler because is test process");
35        return;
36    }
37
38    let mut config = CrashConfig::new();
39    for ext in CRASH_CONFIG {
40        ext(&mut config);
41        if config.no_crash_handler {
42            return;
43        }
44    }
45
46    if process_start_args.next_handlers_count > 0 && process_start_args.yield_count < zng_env::ProcessStartArgs::MAX_YIELD_COUNT - 10 {
47        // extra sure that this is the app-process
48        return process_start_args.yield_once();
49    }
50
51    if std::env::var(APP_PROCESS) != Err(std::env::VarError::NotPresent) {
52        return crash_handler_app_process(config.dump_dir.is_some());
53    }
54
55    match std::env::var(DIALOG_PROCESS) {
56        Ok(args_file) => crash_handler_dialog_process(
57            config.dump_dir.is_some(),
58            config
59                .dialog
60                .or(config.default_dialog)
61                .expect("dialog-process spawned without dialog handler"),
62            args_file,
63        ),
64        Err(e) => match e {
65            std::env::VarError::NotPresent => {}
66            e => panic!("invalid dialog env args, {e:?}"),
67        },
68    }
69
70    crash_handler_monitor_process(
71        config.dump_dir,
72        config.app_process,
73        config.dialog_process,
74        config.default_dialog.is_some() || config.dialog.is_some(),
75    );
76});
77
78/// Gets the number of crash restarts in the app-process.
79///
80/// Always returns zero if called in other processes.
81pub fn restart_count() -> usize {
82    match std::env::var(APP_PROCESS) {
83        Ok(c) => c.strip_prefix("restart-").unwrap_or("0").parse().unwrap_or(0),
84        Err(_) => 0,
85    }
86}
87
88const APP_PROCESS: &str = "ZNG_CRASH_HANDLER_APP";
89const DIALOG_PROCESS: &str = "ZNG_CRASH_HANDLER_DIALOG";
90const DUMP_CHANNEL: &str = "ZNG_MINIDUMP_CHANNEL";
91const RESPONSE_PREFIX: &str = "zng_crash_response: ";
92
93#[doc(hidden)]
94#[linkme::distributed_slice]
95pub static CRASH_CONFIG: [fn(&mut CrashConfig)];
96
97#[doc(hidden)]
98pub use linkme as __linkme;
99
100/// <span data-del-macro-root></span> Register a `FnOnce(&mut CrashConfig)` closure to be
101/// called on process init to configure the crash handler.
102///
103/// See [`CrashConfig`] for more details.
104#[macro_export]
105macro_rules! crash_handler_config {
106    ($closure:expr) => {
107        // expanded from:
108        #[$crate::crash_handler::__linkme::distributed_slice($crate::crash_handler::CRASH_CONFIG)]
109        #[linkme(crate = $crate::crash_handler::__linkme)]
110        #[doc(hidden)]
111        static _CRASH_CONFIG: fn(&mut $crate::crash_handler::CrashConfig) = _crash_config;
112        #[doc(hidden)]
113        fn _crash_config(cfg: &mut $crate::crash_handler::CrashConfig) {
114            fn crash_config(cfg: &mut $crate::crash_handler::CrashConfig, handler: impl FnOnce(&mut $crate::crash_handler::CrashConfig)) {
115                handler(cfg)
116            }
117            crash_config(cfg, $closure)
118        }
119    };
120}
121pub use crate::crash_handler_config;
122
123type ConfigProcess = Vec<Box<dyn for<'a, 'b> FnMut(&'a mut std::process::Command, &'b CrashArgs) -> &'a mut std::process::Command>>;
124type CrashDialogHandler = Box<dyn FnOnce(CrashArgs)>;
125
126/// Crash handler config.
127///
128/// Use [`crash_handler_config!`] to set config.
129///
130/// [`crash_handler_config!`]: crate::crash_handler_config!
131pub struct CrashConfig {
132    default_dialog: Option<CrashDialogHandler>,
133    dialog: Option<CrashDialogHandler>,
134    app_process: ConfigProcess,
135    dialog_process: ConfigProcess,
136    dump_dir: Option<PathBuf>,
137    no_crash_handler: bool,
138}
139impl CrashConfig {
140    fn new() -> Self {
141        Self {
142            default_dialog: None,
143            dialog: None,
144            app_process: vec![],
145            dialog_process: vec![],
146            dump_dir: Some(zng_env::cache("zng_minidump")),
147            no_crash_handler: false,
148        }
149    }
150
151    /// Set the crash dialog process handler.
152    ///
153    /// The dialog `handler` can run an app or show a native dialog, it must use the [`CrashArgs`] process
154    /// terminating methods to respond, if it returns [`CrashArgs::exit`] will run.
155    ///
156    /// Note that the handler does not need to actually show any dialog, it can just save crash info and
157    /// restart the app for example.
158    pub fn dialog(&mut self, handler: impl FnOnce(CrashArgs) + 'static) {
159        if self.dialog.is_none() {
160            self.dialog = Some(Box::new(handler));
161        }
162    }
163
164    /// Set the crash dialog-handler used if `crash_dialog` is not set.
165    ///
166    /// This is used by app libraries or themes to provide a default dialog.
167    pub fn default_dialog(&mut self, handler: impl FnOnce(CrashArgs) + 'static) {
168        self.default_dialog = Some(Box::new(handler));
169    }
170
171    /// Add a closure that is called just before the app-process is spawned.
172    pub fn app_process(
173        &mut self,
174        cfg: impl for<'a, 'b> FnMut(&'a mut std::process::Command, &'b CrashArgs) -> &'a mut std::process::Command + 'static,
175    ) {
176        self.app_process.push(Box::new(cfg));
177    }
178
179    /// Add a closure that is called just before the dialog-process is spawned.
180    pub fn dialog_process(
181        &mut self,
182        cfg: impl for<'a, 'b> FnMut(&'a mut std::process::Command, &'b CrashArgs) -> &'a mut std::process::Command + 'static,
183    ) {
184        self.dialog_process.push(Box::new(cfg));
185    }
186
187    /// Change the minidump directory.
188    ///
189    /// Is `zng::env::cache("zng_minidump")` by default.
190    pub fn minidump_dir(&mut self, dir: impl Into<PathBuf>) {
191        self.dump_dir = Some(dir.into());
192    }
193
194    /// Do not collect a minidump.
195    pub fn no_minidump(&mut self) {
196        self.dump_dir = None;
197    }
198
199    /// Does not run with crash handler.
200    ///
201    /// This is equivalent of running with `NO_ZNG_CRASH_HANDLER` env var.
202    pub fn no_crash_handler(&mut self) {
203        self.no_crash_handler = true;
204    }
205}
206
207/// Arguments for the crash handler dialog function.
208#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
209#[non_exhaustive]
210pub struct CrashArgs {
211    /// Info about the app-process crashes.
212    ///
213    /// Has at least one entry, latest is last. Includes all crashes since the start of the monitor-process.
214    pub app_crashes: Vec<CrashError>,
215
216    /// Info about a crash in the dialog-process spawned to handle the latest app-process crash.
217    ///
218    /// If set this is the last chance to show something to the end user, if the current dialog crashes too
219    /// the monitor-process will give up. If you started an `APP` to show a crash dialog try using a native
220    /// dialog directly now, or just give up, clearly things are far from ok.
221    pub dialog_crash: Option<CrashError>,
222}
223impl CrashArgs {
224    /// Latest crash.
225    pub fn latest(&self) -> &CrashError {
226        self.app_crashes.last().unwrap()
227    }
228
229    /// Restart the app-process with same argument as the latest crash.
230    pub fn restart(&self) -> ! {
231        let json_args = serde_json::to_string(&self.latest().args[..]).unwrap();
232        println!("{RESPONSE_PREFIX}restart {json_args}");
233        zng_env::exit(0)
234    }
235
236    /// Restart the app-process with custom arguments.
237    pub fn restart_with(&self, args: &[Txt]) -> ! {
238        let json_args = serde_json::to_string(&args).unwrap();
239        println!("{RESPONSE_PREFIX}restart {json_args}");
240        zng_env::exit(0)
241    }
242
243    /// Exit the monitor-process (application) with code.
244    pub fn exit(&self, code: i32) -> ! {
245        println!("{RESPONSE_PREFIX}exit {code}");
246        zng_env::exit(0)
247    }
248}
249impl fmt::Display for CrashArgs {
250    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
251        writeln!(f, "APP CRASHES:\n")?;
252
253        for c in self.app_crashes.iter() {
254            writeln!(f, "{c}")?;
255        }
256
257        if let Some(c) = &self.dialog_crash {
258            writeln!(f, "\nDIALOG CRASH:\n")?;
259            writeln!(f, "{c}")?;
260        }
261
262        Ok(())
263    }
264}
265
266/// Info about an app-process crash.
267#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
268#[non_exhaustive]
269pub struct CrashError {
270    /// Crash moment.
271    pub timestamp: SystemTime,
272    /// Process exit code.
273    pub code: Option<i32>,
274    /// Unix signal that terminated the process.
275    pub signal: Option<i32>,
276    /// Full capture of the app stdout.
277    pub stdout: Txt,
278    /// Full capture of the app stderr.
279    pub stderr: Txt,
280    /// Arguments used.
281    pub args: Box<[Txt]>,
282    /// Minidump file.
283    pub minidump: Option<PathBuf>,
284    /// Operating system.
285    ///
286    /// See [`std::env::consts::OS`] for details.
287    pub os: Txt,
288}
289/// Alternate mode `{:#}` prints plain stdout and stderr (no ANSI escape sequences).
290impl fmt::Display for CrashError {
291    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
292        writeln!(f, "timestamp: {}", self.unix_time())?;
293        if let Some(c) = self.code {
294            writeln!(f, "exit code: {c:#X}")?
295        }
296        if let Some(c) = self.signal {
297            writeln!(f, "exit signal: {c}")?
298        }
299        if let Some(p) = self.minidump.as_ref() {
300            writeln!(f, "minidump: {}", p.display())?
301        }
302        if f.alternate() {
303            write!(f, "\nSTDOUT:\n{}\nSTDERR:\n{}\n", self.stdout_plain(), self.stderr_plain())
304        } else {
305            write!(f, "\nSTDOUT:\n{}\nSTDERR:\n{}\n", self.stdout, self.stderr)
306        }
307    }
308}
309impl CrashError {
310    fn new(
311        timestamp: SystemTime,
312        code: Option<i32>,
313        signal: Option<i32>,
314        stdout: Txt,
315        stderr: Txt,
316        minidump: Option<PathBuf>,
317        args: Box<[Txt]>,
318    ) -> Self {
319        Self {
320            timestamp,
321            code,
322            signal,
323            stdout,
324            stderr,
325            args,
326            minidump,
327            os: std::env::consts::OS.into(),
328        }
329    }
330
331    /// Seconds since Unix epoch.
332    pub fn unix_time(&self) -> u64 {
333        self.timestamp.duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default().as_secs()
334    }
335
336    /// Gets if `stdout` does not contain any ANSI scape sequences.
337    pub fn is_stdout_plain(&self) -> bool {
338        !self.stdout.contains(CSI)
339    }
340
341    /// Gets if `stderr` does not contain any ANSI scape sequences.
342    pub fn is_stderr_plain(&self) -> bool {
343        !self.stderr.contains(CSI)
344    }
345
346    /// Get `stdout` without any ANSI escape sequences (CSI).
347    pub fn stdout_plain(&self) -> Txt {
348        remove_ansi_csi(&self.stdout)
349    }
350
351    /// Get `stderr` without any ANSI escape sequences (CSI).
352    pub fn stderr_plain(&self) -> Txt {
353        remove_ansi_csi(&self.stderr)
354    }
355
356    /// Gets if `stderr` contains a crash panic.
357    pub fn has_panic(&self) -> bool {
358        if self.code == Some(101) {
359            CrashPanic::contains(&self.stderr_plain())
360        } else {
361            false
362        }
363    }
364
365    /// Gets if `stderr` contains a crash panic that traced widget/window path.
366    pub fn has_panic_widget(&self) -> bool {
367        if self.code == Some(101) {
368            CrashPanic::contains_widget(&self.stderr_plain())
369        } else {
370            false
371        }
372    }
373
374    /// Try parse `stderr` for the crash panic.
375    ///
376    /// Only reliably works if the panic fully printed correctly and was formatted by the panic
377    /// hook installed by `crash_handler` or by the display print of [`CrashPanic`].
378    pub fn find_panic(&self) -> Option<CrashPanic> {
379        if self.code == Some(101) {
380            CrashPanic::find(&self.stderr_plain())
381        } else {
382            None
383        }
384    }
385
386    /// Best attempt at generating a readable error message.
387    ///
388    /// Is the panic message, or the minidump exception, with the exit code and signal.
389    pub fn message(&self) -> Txt {
390        let mut msg = if let Some(msg) = self.find_panic().map(|p| p.message) {
391            msg
392        } else if let Some(msg) = self.minidump_message() {
393            msg
394        } else {
395            "".into()
396        };
397        use std::fmt::Write as _;
398
399        if let Some(c) = self.code {
400            let sep = if msg.is_empty() { "" } else { "\n" };
401            write!(&mut msg, "{sep}Code: {c:#X}").unwrap();
402        }
403        if let Some(c) = self.signal {
404            let sep = if msg.is_empty() { "" } else { "\n" };
405            write!(&mut msg, "{sep}Signal: {c}").unwrap();
406        }
407        msg.end_mut();
408        msg
409    }
410
411    fn minidump_message(&self) -> Option<Txt> {
412        use minidump::*;
413
414        let dump = match Minidump::read_path(self.minidump.as_ref()?) {
415            Ok(d) => d,
416            Err(e) => {
417                tracing::error!("error reading minidump, {e}");
418                return None;
419            }
420        };
421
422        let exception = match dump.get_stream::<MinidumpException>() {
423            Ok(s) => s,
424            Err(e) => {
425                tracing::error!("error reading minidump exception, {e}");
426                return None;
427            }
428        };
429
430        #[cfg(debug_assertions)]
431        {
432            // nice error messages, but adds >1MB of binary code
433            let system_info = match dump.get_stream::<MinidumpSystemInfo>() {
434                Ok(s) => s,
435                Err(e) => {
436                    tracing::error!("error reading minidump system info, {e}");
437                    return None;
438                }
439            };
440            let crash_reason = exception.get_crash_reason(system_info.os, system_info.cpu);
441            Some(zng_txt::formatx!("{crash_reason}"))
442        }
443
444        #[cfg(not(debug_assertions))]
445        {
446            // raw error code, only common names
447            let raw = exception.raw;
448
449            let code = raw.exception_record.exception_code;
450            let addr = raw.exception_record.exception_address;
451
452            cfg_select! {
453                windows => {
454                    let name = match code {
455                        0xC0000005 => "ACCESS_VIOLATION",
456                        0xC0000409 => "STACK_BUFFER_OVERRUN",
457                        0x80000003 => "BREAKPOINT",
458                        0xC000001D => "ILLEGAL_INSTRUCTION",
459                        0xC0000094 => "INTEGER_DIVIDE_BY_ZERO",
460                        0xC00000FD => "STACK_OVERFLOW",
461                        0xC0000096 => "PRIVILEGED_INSTRUCTION",
462                        0xC0000008 => "INVALID_HANDLE",
463                        0xC0000135 => "DLL_NOT_FOUND",
464                        _ => "",
465                    };
466                }
467                any(target_os = "linux", target_os = "android") => {
468                    let name = match code as i32 {
469                        4 => "SIGILL",
470                        5 => "SIGTRAP",
471                        6 => "SIGABRT",
472                        7 => "SIGBUS",
473                        8 => "SIGFPE",
474                        9 => "SIGKILL",
475                        11 => "SIGSEGV",
476                        13 => "SIGPIPE",
477                        _ => "",
478                    };
479                }
480                any(target_os = "macos", target_os = "ios") => {
481                    let name = match code as i32 {
482                        4 => "SIGILL",
483                        5 => "SIGTRAP",
484                        6 => "SIGABRT",
485                        8 => "SIGFPE",
486                        10 => "SIGBUS",
487                        11 => "SIGSEGV",
488                        _ => "",
489                    };
490                }
491                _ => {
492                    let name = "";
493                }
494            }
495            if name.is_empty() {
496                Some(zng_txt::formatx!("exception 0x{code:08X} at 0x{addr:X}"))
497            } else {
498                Some(zng_txt::formatx!("exception 0x{code:08X} ({name}) at 0x{addr:X}"))
499            }
500        }
501    }
502}
503
504const CSI: &str = "\x1b[";
505
506/// Remove ANSI escape sequences (CSI) from `s`.
507pub fn remove_ansi_csi(mut s: &str) -> Txt {
508    fn is_esc_end(byte: u8) -> bool {
509        (0x40..=0x7e).contains(&byte)
510    }
511
512    let mut r = String::new();
513    while let Some(i) = s.find(CSI) {
514        r.push_str(&s[..i]);
515        s = &s[i + CSI.len()..];
516        let mut esc_end = 0;
517        while esc_end < s.len() && !is_esc_end(s.as_bytes()[esc_end]) {
518            esc_end += 1;
519        }
520        esc_end += 1;
521        s = &s[esc_end..];
522    }
523    r.push_str(s);
524    r.into()
525}
526
527/// Panic parsed from a `stderr` dump.
528#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
529#[non_exhaustive]
530pub struct CrashPanic {
531    /// Name of thread that panicked.
532    pub thread: Txt,
533    /// Panic message.
534    pub message: Txt,
535    /// Path to file that defines the panic.
536    pub file: Txt,
537    /// Line of code that defines the panic.
538    pub line: u32,
539    /// Column in the line of code that defines the panic.
540    pub column: u32,
541    /// Widget where the panic happened.
542    pub widget_path: Txt,
543    /// Stack backtrace.
544    pub backtrace: Txt,
545}
546
547/// Alternate mode `{:#}` prints full backtrace.
548impl fmt::Display for CrashPanic {
549    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
550        writeln!(
551            f,
552            "thread '{}' panicked at {}:{}:{}:",
553            self.thread, self.file, self.line, self.column
554        )?;
555        for line in self.message.lines() {
556            writeln!(f, "   {line}")?;
557        }
558        writeln!(f, "widget path:\n   {}", self.widget_path)?;
559
560        if f.alternate() {
561            writeln!(f, "stack backtrace:\n{}", self.backtrace)
562        } else {
563            writeln!(f, "stack backtrace:")?;
564            let mut snippet = 9;
565            for frame in self.backtrace_frames().skip_while(|f| f.is_after_panic) {
566                write!(f, "{frame}")?;
567                if snippet > 0 {
568                    let code = frame.code_snippet();
569                    if !code.is_empty() {
570                        snippet -= 1;
571                        writeln!(f, "{code}")?;
572                    }
573                }
574            }
575            Ok(())
576        }
577    }
578}
579impl CrashPanic {
580    /// Gets if `stderr` contains a panic that can be parsed by [`find`].
581    ///
582    /// [`find`]: Self::find
583    pub fn contains(stderr: &str) -> bool {
584        Self::find_impl(stderr, false).is_some()
585    }
586
587    /// Gets if `stderr` contains a panic that can be parsed by [`find`] and traced a widget/window path.
588    ///
589    /// [`find`]: Self::find
590    pub fn contains_widget(stderr: &str) -> bool {
591        match Self::find_impl(stderr, false) {
592            Some(p) => !p.widget_path.is_empty(),
593            None => false,
594        }
595    }
596
597    /// Try parse `stderr` for the crash panic.
598    ///
599    /// Only reliably works if the panic fully printed correctly and was formatted by the panic
600    /// hook installed by `crash_handler` or by the display print of this type.
601    pub fn find(stderr: &str) -> Option<Self> {
602        Self::find_impl(stderr, true)
603    }
604
605    fn find_impl(stderr: &str, parse: bool) -> Option<Self> {
606        let mut panic_at = usize::MAX;
607        let mut widget_path = usize::MAX;
608        let mut stack_backtrace = usize::MAX;
609        let mut i = 0;
610        for line in stderr.lines() {
611            if line.starts_with("thread '") && line.contains("' panicked at ") && line.ends_with(':') {
612                panic_at = i;
613                widget_path = usize::MAX;
614                stack_backtrace = usize::MAX;
615            } else if line == "widget path:" {
616                widget_path = i + "widget path:\n".len();
617            } else if line == "stack backtrace:" {
618                stack_backtrace = i + "stack backtrace:\n".len();
619            }
620            i += line.len() + "\n".len();
621        }
622
623        if panic_at == usize::MAX {
624            return None;
625        }
626
627        if !parse {
628            return Some(Self {
629                thread: Txt::from(""),
630                message: Txt::from(""),
631                file: Txt::from(""),
632                line: 0,
633                column: 0,
634                widget_path: if widget_path < stderr.len() {
635                    Txt::from("true")
636                } else {
637                    Txt::from("")
638                },
639                backtrace: Txt::from(""),
640            });
641        }
642
643        let panic_str = stderr[panic_at..].lines().next().unwrap();
644        let (thread, location) = panic_str.strip_prefix("thread '").unwrap().split_once("' panicked at ").unwrap();
645        let mut location = location.split(':');
646        let file = location.next().unwrap_or("");
647        let line: u32 = location.next().unwrap_or("0").parse().unwrap_or(0);
648        let column: u32 = location.next().unwrap_or("0").parse().unwrap_or(0);
649
650        let mut message = String::new();
651        let mut sep = "";
652        for line in stderr[panic_at + panic_str.len() + "\n".len()..].lines() {
653            if let Some(line) = line.strip_prefix("   ") {
654                message.push_str(sep);
655                message.push_str(line);
656                sep = "\n";
657            } else {
658                if message.is_empty() && line != "widget path:" && line != "stack backtrace:" {
659                    // not formatted by us, probably by Rust
660                    line.clone_into(&mut message);
661                }
662                break;
663            }
664        }
665
666        let widget_path = if widget_path < stderr.len() {
667            stderr[widget_path..].lines().next().unwrap().trim()
668        } else {
669            ""
670        };
671
672        let backtrace = if stack_backtrace < stderr.len() {
673            let mut i = stack_backtrace;
674            'backtrace_seek: for line in stderr[stack_backtrace..].lines() {
675                if !line.starts_with(' ') {
676                    'digit_check: for c in line.chars() {
677                        if !c.is_ascii_digit() {
678                            if c == ':' {
679                                break 'digit_check;
680                            } else {
681                                break 'backtrace_seek;
682                            }
683                        }
684                    }
685                }
686                i += line.len() + "\n".len();
687            }
688            &stderr[stack_backtrace..i]
689        } else {
690            ""
691        };
692
693        Some(Self {
694            thread: thread.to_txt(),
695            message: message.into(),
696            file: file.to_txt(),
697            line,
698            column,
699            widget_path: widget_path.to_txt(),
700            backtrace: backtrace.to_txt(),
701        })
702    }
703
704    /// Iterate over frames parsed from the `backtrace`.
705    pub fn backtrace_frames(&self) -> impl Iterator<Item = BacktraceFrame> + '_ {
706        BacktraceFrame::parse(&self.backtrace)
707    }
708}
709
710/// Represents a frame parsed from a stack backtrace.
711#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
712#[non_exhaustive]
713pub struct BacktraceFrame {
714    /// Position on the backtrace.
715    pub n: usize,
716
717    /// Function name.
718    pub name: Txt,
719    /// Source code file.
720    pub file: Txt,
721    /// Source code line.
722    pub line: u32,
723
724    /// If this frame is inside the Rust panic code.
725    pub is_after_panic: bool,
726}
727impl fmt::Display for BacktraceFrame {
728    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
729        writeln!(f, "{:>4}: {}", self.n, self.name)?;
730        if !self.file.is_empty() {
731            writeln!(f, "      at {}:{}", self.file, self.line)?;
732        }
733        Ok(())
734    }
735}
736impl BacktraceFrame {
737    /// Iterate over frames parsed from the `backtrace`.
738    pub fn parse(mut backtrace: &str) -> impl Iterator<Item = BacktraceFrame> + '_ {
739        let mut is_after_panic = backtrace.lines().any(|l| l.ends_with("core::panicking::panic_fmt"));
740        std::iter::from_fn(move || {
741            if backtrace.is_empty() {
742                None
743            } else {
744                let n_name = backtrace.lines().next().unwrap();
745                let (n, name) = if let Some((n, name)) = n_name.split_once(':') {
746                    let n = match n.trim_start().parse() {
747                        Ok(n) => n,
748                        Err(_) => {
749                            backtrace = "";
750                            return None;
751                        }
752                    };
753                    let name = name.trim();
754                    if name.is_empty() {
755                        backtrace = "";
756                        return None;
757                    }
758                    (n, name)
759                } else {
760                    backtrace = "";
761                    return None;
762                };
763
764                backtrace = &backtrace[n_name.len() + 1..];
765                let r = if backtrace.trim_start().starts_with("at ") {
766                    let file_line = backtrace.lines().next().unwrap();
767                    let (file, line) = if let Some((file, line)) = file_line.rsplit_once(':') {
768                        let file = file.trim_start().strip_prefix("at ").unwrap();
769                        let line = match line.trim_end().parse() {
770                            Ok(l) => l,
771                            Err(_) => {
772                                backtrace = "";
773                                return None;
774                            }
775                        };
776                        (file, line)
777                    } else {
778                        backtrace = "";
779                        return None;
780                    };
781
782                    backtrace = &backtrace[file_line.len() + 1..];
783
784                    BacktraceFrame {
785                        n,
786                        name: name.to_txt(),
787                        file: file.to_txt(),
788                        line,
789                        is_after_panic,
790                    }
791                } else {
792                    BacktraceFrame {
793                        n,
794                        name: name.to_txt(),
795                        file: Txt::from(""),
796                        line: 0,
797                        is_after_panic,
798                    }
799                };
800
801                if is_after_panic && name == "core::panicking::panic_fmt" {
802                    is_after_panic = false;
803                }
804
805                Some(r)
806            }
807        })
808    }
809
810    /// Reads the code line + four surrounding lines if the code file can be found.
811    pub fn code_snippet(&self) -> Txt {
812        if !self.file.is_empty()
813            && self.line > 0
814            && let Ok(file) = std::fs::File::open(&self.file)
815        {
816            use std::fmt::Write as _;
817            let mut r = String::new();
818
819            let reader = std::io::BufReader::new(file);
820
821            let line_s = self.line - 2.min(self.line - 1);
822            let lines = reader.lines().skip(line_s as usize - 1).take(5);
823            for (line, line_n) in lines.zip(line_s..) {
824                let line = match line {
825                    Ok(l) => l,
826                    Err(_) => return Txt::from(""),
827                };
828
829                if line_n == self.line {
830                    writeln!(&mut r, "      {line_n:>4} > {line}").unwrap();
831                } else {
832                    writeln!(&mut r, "      {line_n:>4} │ {line}").unwrap();
833                }
834            }
835
836            return r.into();
837        }
838        Txt::from("")
839    }
840}
841
842fn crash_handler_monitor_process(
843    dump_dir: Option<PathBuf>,
844    mut cfg_app: ConfigProcess,
845    mut cfg_dialog: ConfigProcess,
846    has_dialog_handler: bool,
847) -> ! {
848    zng_env::set_process_name("crash-handler-process");
849
850    let exe = std::env::current_exe()
851        .and_then(dunce::canonicalize)
852        .expect("failed to get the current executable");
853
854    let mut args: Box<[_]> = std::env::args().skip(1).map(Txt::from).collect();
855
856    let mut dialog_args = CrashArgs {
857        app_crashes: vec![],
858        dialog_crash: None,
859    };
860    loop {
861        let mut app_process = std::process::Command::new(&exe);
862        for cfg in &mut cfg_app {
863            cfg(&mut app_process, &dialog_args);
864        }
865
866        match run_process(
867            dump_dir.as_deref(),
868            app_process
869                .env(APP_PROCESS, format!("restart-{}", dialog_args.app_crashes.len()))
870                .args(args.iter()),
871        ) {
872            Ok((status, [stdout, stderr], dump_file)) => {
873                if status.success() {
874                    let code = status.code().unwrap_or(0);
875                    tracing::info!(
876                        "crash monitor-process exiting with success code ({code}), {} crashes",
877                        dialog_args.app_crashes.len()
878                    );
879                    zng_env::exit(code);
880                } else {
881                    let code = status.code();
882                    #[allow(unused_mut)] // Windows has no signal
883                    let mut signal = None::<i32>;
884
885                    #[cfg(windows)]
886                    if code == Some(1) {
887                        tracing::warn!(
888                            "app-process exit code (1), probably killed by the system, \
889                                        will exit monitor-process with the same code"
890                        );
891                        zng_env::exit(1);
892                    }
893                    #[cfg(unix)]
894                    if code.is_none() {
895                        use std::os::unix::process::ExitStatusExt as _;
896                        signal = status.signal();
897
898                        if let Some(sig) = signal
899                            && [2, 9, 17, 19, 23].contains(&sig)
900                        {
901                            tracing::warn!(
902                                "app-process exited by signal ({sig}), \
903                                                will exit monitor-process with code 1"
904                            );
905                            zng_env::exit(1);
906                        }
907                    }
908
909                    tracing::error!(
910                        "app-process crashed with exit code ({:#X}), signal ({:#?}), {} crashes previously",
911                        code.unwrap_or(0),
912                        signal.unwrap_or(0),
913                        dialog_args.app_crashes.len()
914                    );
915
916                    let timestamp = SystemTime::now();
917
918                    dialog_args.app_crashes.push(CrashError::new(
919                        timestamp,
920                        code,
921                        signal,
922                        stdout.into(),
923                        stderr.into(),
924                        dump_file,
925                        args.clone(),
926                    ));
927
928                    // show dialog, retries once if dialog crashes too.
929                    for _ in 0..2 {
930                        // serialize app-crashes to a temp JSON file
931                        let timestamp_nanos = timestamp.duration_since(SystemTime::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0);
932                        let mut timestamp = timestamp_nanos;
933                        let mut retries = 0;
934                        let crash_file = loop {
935                            let path = std::env::temp_dir().join(format!("zng-crash-errors-{timestamp:#x}"));
936                            match std::fs::File::create_new(&path) {
937                                Ok(f) => match serde_json::to_writer(std::io::BufWriter::new(f), &dialog_args) {
938                                    Ok(_) => break path,
939                                    Err(e) => {
940                                        if e.is_io() {
941                                            if retries > 20 {
942                                                panic!("error writing crash errors, {e}");
943                                            } else if retries > 5 {
944                                                timestamp += 1;
945                                            }
946                                            std::thread::sleep(100.ms());
947                                        } else {
948                                            panic!("error serializing crash errors, {e}");
949                                        }
950                                    }
951                                },
952                                Err(e) => {
953                                    if e.kind() == std::io::ErrorKind::AlreadyExists {
954                                        timestamp += 1;
955                                    } else {
956                                        if retries > 20 {
957                                            panic!("error creating crash errors file, {e}");
958                                        } else if retries > 5 {
959                                            timestamp += 1;
960                                        }
961                                        std::thread::sleep(100.ms());
962                                    }
963                                }
964                            }
965                            retries += 1;
966                        };
967
968                        let dialog_result = if has_dialog_handler {
969                            let mut dialog_process = std::process::Command::new(&exe);
970                            for cfg in &mut cfg_dialog {
971                                cfg(&mut dialog_process, &dialog_args);
972                            }
973                            run_process(dump_dir.as_deref(), dialog_process.env(DIALOG_PROCESS, &crash_file))
974                        } else {
975                            Ok((std::process::ExitStatus::default(), [String::new(), String::new()], None))
976                        };
977
978                        for _ in 0..5 {
979                            if !crash_file.exists() || std::fs::remove_file(&crash_file).is_ok() {
980                                break;
981                            }
982                            std::thread::sleep(100.ms());
983                        }
984
985                        let response = match dialog_result {
986                            Ok((dlg_status, [dlg_stdout, dlg_stderr], dlg_dump_file)) => {
987                                if dlg_status.success() {
988                                    dlg_stdout
989                                        .lines()
990                                        .filter_map(|l| l.trim().strip_prefix(RESPONSE_PREFIX))
991                                        .next_back()
992                                        .unwrap_or("exit 0")
993                                        .to_owned()
994                                } else {
995                                    let code = dlg_status.code();
996                                    #[allow(unused_mut)] // Windows has no signal
997                                    let mut signal = None::<i32>;
998
999                                    #[cfg(windows)]
1000                                    if code == Some(1) {
1001                                        tracing::warn!(
1002                                            "dialog-process exit code (1), probably killed by the system, \
1003                                                        will exit monitor-process with the same code"
1004                                        );
1005                                        zng_env::exit(1);
1006                                    }
1007                                    #[cfg(unix)]
1008                                    if code.is_none() {
1009                                        use std::os::unix::process::ExitStatusExt as _;
1010                                        signal = status.signal();
1011
1012                                        if let Some(sig) = signal
1013                                            && [2, 9, 17, 19, 23].contains(&sig)
1014                                        {
1015                                            tracing::warn!(
1016                                                "dialog-process exited by signal ({sig}), \
1017                                                                will exit monitor-process with code 1"
1018                                            );
1019                                            zng_env::exit(1);
1020                                        }
1021                                    }
1022
1023                                    let dialog_crash = CrashError::new(
1024                                        SystemTime::now(),
1025                                        code,
1026                                        signal,
1027                                        dlg_stdout.into(),
1028                                        dlg_stderr.into(),
1029                                        dlg_dump_file,
1030                                        Box::new([]),
1031                                    );
1032                                    tracing::error!("crash dialog-process crashed, {dialog_crash}");
1033
1034                                    if dialog_args.dialog_crash.is_none() {
1035                                        dialog_args.dialog_crash = Some(dialog_crash);
1036                                        continue;
1037                                    } else {
1038                                        let latest = dialog_args.latest();
1039                                        eprintln!("{latest}");
1040                                        zng_env::exit(latest.code.unwrap_or(1));
1041                                    }
1042                                }
1043                            }
1044                            Err(e) => panic!("error running dialog-process, {e}"),
1045                        };
1046
1047                        if let Some(args_json) = response.strip_prefix("restart ") {
1048                            args = serde_json::from_str(args_json).expect("crash dialog-process did not respond 'restart' correctly");
1049                            break;
1050                        } else if let Some(code) = response.strip_prefix("exit ") {
1051                            let code: i32 = code.parse().expect("crash dialog-process did not respond 'code' correctly");
1052                            zng_env::exit(code);
1053                        } else {
1054                            panic!("crash dialog-process did not respond correctly")
1055                        }
1056                    }
1057                }
1058            }
1059            Err(e) => panic!("error running app-process, {e}"),
1060        }
1061    }
1062}
1063fn run_process(
1064    dump_dir: Option<&Path>,
1065    command: &mut std::process::Command,
1066) -> std::io::Result<(std::process::ExitStatus, [String; 2], Option<PathBuf>)> {
1067    struct DumpServer {
1068        shutdown: Arc<AtomicBool>,
1069        runner: std::thread::JoinHandle<Option<PathBuf>>,
1070    }
1071    let mut dump_server = None;
1072    if let Some(dump_dir) = dump_dir {
1073        match std::fs::create_dir_all(dump_dir) {
1074            Ok(_) => {
1075                let uuid = uuid::Uuid::new_v4();
1076                let dump_file = dump_dir.join(format!("{}.dmp", uuid.simple()));
1077                let dump_channel = std::env::temp_dir().join(format!("zng-crash-{}", uuid.simple()));
1078                match minidumper::Server::with_name(minidumper::SocketName::Path(&dump_channel)) {
1079                    Ok(mut s) => {
1080                        command.env(DUMP_CHANNEL, &dump_channel);
1081                        let shutdown = Arc::new(AtomicBool::new(false));
1082                        let runner = std::thread::Builder::new()
1083                            .name("minidumper-server".into())
1084                            .stack_size(512 * 1024)
1085                            .spawn(clmv!(shutdown, || {
1086                                let created_file = Arc::new(Mutex::new(None));
1087                                if let Err(e) = s.run(
1088                                    Box::new(MinidumpServerHandler {
1089                                        dump_file,
1090                                        created_file: created_file.clone(),
1091                                    }),
1092                                    &shutdown,
1093                                    None,
1094                                ) {
1095                                    tracing::error!("minidump server exited with error, {e}");
1096                                }
1097                                created_file.lock().take()
1098                            }))
1099                            .expect("failed to spawn thread");
1100                        dump_server = Some(DumpServer { shutdown, runner });
1101                    }
1102                    Err(e) => tracing::error!("failed to spawn minidump server, will not enable crash handling, {e}"),
1103                }
1104            }
1105            Err(e) => tracing::error!("cannot create minidump dir, will not enable crash handling, {e}"),
1106        }
1107    }
1108
1109    let mut app_process = command
1110        .env("RUST_BACKTRACE", "full")
1111        .env("CLICOLOR_FORCE", "1")
1112        .stdout(std::process::Stdio::piped())
1113        .stderr(std::process::Stdio::piped())
1114        .spawn()?;
1115
1116    let stdout = capture_and_print(app_process.stdout.take().unwrap(), false);
1117    let stderr = capture_and_print(app_process.stderr.take().unwrap(), true);
1118
1119    let status = app_process.wait()?;
1120
1121    let stdout = match stdout.join() {
1122        Ok(r) => r,
1123        Err(p) => std::panic::resume_unwind(p),
1124    };
1125    let stderr = match stderr.join() {
1126        Ok(r) => r,
1127        Err(p) => std::panic::resume_unwind(p),
1128    };
1129
1130    let mut dump_file = None;
1131    if let Some(s) = dump_server {
1132        s.shutdown.store(true, atomic::Ordering::Relaxed);
1133        match s.runner.join() {
1134            Ok(r) => dump_file = r,
1135            Err(p) => std::panic::resume_unwind(p),
1136        };
1137    }
1138
1139    Ok((status, [stdout, stderr], dump_file))
1140}
1141struct MinidumpServerHandler {
1142    dump_file: PathBuf,
1143    created_file: Arc<Mutex<Option<PathBuf>>>,
1144}
1145impl minidumper::ServerHandler for MinidumpServerHandler {
1146    fn create_minidump_file(&self) -> Result<(std::fs::File, PathBuf), std::io::Error> {
1147        let file = std::fs::File::create_new(&self.dump_file)?;
1148        Ok((file, self.dump_file.clone()))
1149    }
1150
1151    fn on_minidump_created(&self, result: Result<minidumper::MinidumpBinary, minidumper::Error>) -> minidumper::LoopAction {
1152        match result {
1153            Ok(b) => *self.created_file.lock() = Some(b.path),
1154            Err(e) => tracing::error!("failed to write minidump file, {e}"),
1155        }
1156        minidumper::LoopAction::Exit
1157    }
1158
1159    fn on_message(&self, _: u32, _: Vec<u8>) {}
1160
1161    fn on_client_connected(&self, num_clients: usize) -> minidumper::LoopAction {
1162        if num_clients > 1 {
1163            tracing::error!("expected only one minidump client, {num_clients} connected, exiting server");
1164            minidumper::LoopAction::Exit
1165        } else {
1166            minidumper::LoopAction::Continue
1167        }
1168    }
1169
1170    fn on_client_disconnected(&self, num_clients: usize) -> minidumper::LoopAction {
1171        if num_clients != 0 {
1172            tracing::error!("expected only one minidump client disconnect, {num_clients} still connected");
1173        }
1174        minidumper::LoopAction::Exit
1175    }
1176}
1177fn capture_and_print(mut stream: impl std::io::Read + Send + 'static, is_err: bool) -> std::thread::JoinHandle<String> {
1178    std::thread::Builder::new()
1179        .name(format!("{}-reader", if is_err { "stderr" } else { "stdout" }))
1180        .stack_size(256 * 1024)
1181        .spawn(move || {
1182            let mut capture = vec![];
1183            let mut buffer = [0u8; 32];
1184            loop {
1185                match stream.read(&mut buffer) {
1186                    Ok(n) => {
1187                        if n == 0 {
1188                            break;
1189                        }
1190
1191                        let new = &buffer[..n];
1192                        capture.write_all(new).unwrap();
1193                        let r = if is_err {
1194                            let mut s = std::io::stderr();
1195                            s.write_all(new).and_then(|_| s.flush())
1196                        } else {
1197                            let mut s = std::io::stdout();
1198                            s.write_all(new).and_then(|_| s.flush())
1199                        };
1200                        if let Err(e) = r {
1201                            panic!("{} write error, {}", if is_err { "stderr" } else { "stdout" }, e)
1202                        }
1203                    }
1204                    Err(e) => panic!("{} read error, {}", if is_err { "stderr" } else { "stdout" }, e),
1205                }
1206            }
1207            String::from_utf8_lossy(&capture).into_owned()
1208        })
1209        .expect("failed to spawn thread")
1210}
1211
1212fn crash_handler_app_process(dump_enabled: bool) {
1213    std::panic::set_hook(Box::new(panic_handler));
1214    if dump_enabled {
1215        minidump_attach();
1216    }
1217
1218    // app-process execution happens after this.
1219}
1220
1221fn crash_handler_dialog_process(dump_enabled: bool, dialog: CrashDialogHandler, args_file: String) -> ! {
1222    zng_env::set_process_name("crash-dialog-process");
1223
1224    std::panic::set_hook(Box::new(panic_handler));
1225    if dump_enabled {
1226        minidump_attach();
1227    }
1228
1229    let mut retries = 0;
1230    let args = loop {
1231        match std::fs::read_to_string(&args_file) {
1232            Ok(args) => break args,
1233            Err(e) => {
1234                if e.kind() != std::io::ErrorKind::NotFound && retries < 10 {
1235                    retries += 1;
1236                    continue;
1237                }
1238                panic!("error reading args file, {e}");
1239            }
1240        }
1241    };
1242
1243    dialog(serde_json::from_str(&args).expect("error deserializing args"));
1244    CrashArgs {
1245        app_crashes: vec![],
1246        dialog_crash: None,
1247    }
1248    .exit(0)
1249}
1250
1251fn panic_handler(info: &std::panic::PanicHookInfo) {
1252    let backtrace = std::backtrace::Backtrace::capture();
1253    let path = crate::widget::WIDGET.trace_path();
1254    let panic = PanicInfo::from_hook(info);
1255    eprintln!("{panic}widget path:\n   {path}\nstack backtrace:\n{backtrace}");
1256}
1257
1258fn minidump_attach() {
1259    let channel_name = match std::env::var(DUMP_CHANNEL) {
1260        Ok(n) if !n.is_empty() => PathBuf::from(n),
1261        _ => {
1262            eprintln!("expected minidump channel name, this instance will not handle crashes");
1263            return;
1264        }
1265    };
1266    let client = match minidumper::Client::with_name(minidumper::SocketName::Path(&channel_name)) {
1267        Ok(c) => c,
1268        Err(e) => {
1269            eprintln!("failed to connect minidump client, this instance will not handle crashes, {e}");
1270            return;
1271        }
1272    };
1273    struct Handler(minidumper::Client);
1274    // SAFETY: on_crash does the minimal possible work
1275    unsafe impl crash_handler::CrashEvent for Handler {
1276        fn on_crash(&self, context: &crash_handler::CrashContext) -> crash_handler::CrashEventResult {
1277            crash_handler::CrashEventResult::Handled(self.0.request_dump(context).is_ok())
1278        }
1279    }
1280    let handler = match crash_handler::CrashHandler::attach(Box::new(Handler(client))) {
1281        Ok(h) => h,
1282        Err(e) => {
1283            eprintln!("failed attach minidump crash handler, this instance will not handle crashes, {e}");
1284            return;
1285        }
1286    };
1287
1288    *CRASH_HANDLER.lock() = Some(handler);
1289}
1290static CRASH_HANDLER: Mutex<Option<crash_handler::CrashHandler>> = Mutex::new(None);
1291
1292#[derive(Debug)]
1293struct PanicInfo {
1294    pub thread: Txt,
1295    pub msg: Txt,
1296    pub file: Txt,
1297    pub line: u32,
1298    pub column: u32,
1299}
1300impl PanicInfo {
1301    pub fn from_hook(info: &std::panic::PanicHookInfo) -> Self {
1302        let current_thread = std::thread::current();
1303        let thread = current_thread.name().unwrap_or("<unnamed>");
1304        let msg = Self::payload(info.payload());
1305
1306        let (file, line, column) = if let Some(l) = info.location() {
1307            (l.file(), l.line(), l.column())
1308        } else {
1309            ("<unknown>", 0, 0)
1310        };
1311        Self {
1312            thread: thread.to_txt(),
1313            msg,
1314            file: file.to_txt(),
1315            line,
1316            column,
1317        }
1318    }
1319
1320    fn payload(p: &dyn std::any::Any) -> Txt {
1321        match p.downcast_ref::<&'static str>() {
1322            Some(s) => s,
1323            None => match p.downcast_ref::<String>() {
1324                Some(s) => &s[..],
1325                None => "Box<dyn Any>",
1326            },
1327        }
1328        .to_txt()
1329    }
1330}
1331impl std::error::Error for PanicInfo {}
1332impl fmt::Display for PanicInfo {
1333    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1334        writeln!(
1335            f,
1336            "thread '{}' panicked at {}:{}:{}:",
1337            self.thread, self.file, self.line, self.column
1338        )?;
1339        for line in self.msg.lines() {
1340            writeln!(f, "   {line}")?;
1341        }
1342        Ok(())
1343    }
1344}
zng_app/crash_handler.rs

zng_app/
crash_handler.rs
