zng_app/
crash_handler.rs

1#![cfg(all(
2    feature = "crash_handler",
3    not(any(target_arch = "wasm32", target_os = "android", target_os = "ios"))
4))]
5
6//! App-process crash handler.
7//!
8//! See the `zng::app::crash_handler` documentation for more details.
9
10use std::{
11    fmt,
12    path::{Path, PathBuf},
13    sync::{Arc, atomic::AtomicBool},
14    time::SystemTime,
15};
16use zng_clone_move::clmv;
17use zng_layout::unit::TimeUnits as _;
18use zng_task::{
19    parking_lot::Mutex,
20    process::tap::{self, PanicInfo},
21};
22
23use zng_txt::Txt;
24
25/// Environment variable that causes the crash handler to not start if set.
26///
27/// This is particularly useful to set in debugger launch configs. Crash handler spawns
28/// a different process for the app  so break points will not work.
29pub const NO_CRASH_HANDLER: &str = "ZNG_NO_CRASH_HANDLER";
30
31zng_env::on_process_start!(|process_start_args| {
32    if std::env::var(NO_CRASH_HANDLER).is_ok() {
33        return;
34    }
35    if zng_env::about().is_test {
36        tracing::debug!("ignoring crash_handler because is test process");
37        return;
38    }
39
40    let mut config = CrashConfig::new();
41    for ext in CRASH_CONFIG {
42        ext(&mut config);
43        if config.no_crash_handler {
44            return;
45        }
46    }
47
48    if process_start_args.next_handlers_count > 0 && process_start_args.yield_count < zng_env::ProcessStartArgs::MAX_YIELD_COUNT - 10 {
49        // extra sure that this is the app-process
50        return process_start_args.yield_once();
51    }
52
53    if std::env::var(APP_PROCESS) != Err(std::env::VarError::NotPresent) {
54        return crash_handler_app_process(config.dump_dir.is_some());
55    }
56
57    match std::env::var(DIALOG_PROCESS) {
58        Ok(args_file) => crash_handler_dialog_process(
59            config.dump_dir.is_some(),
60            config
61                .dialog
62                .or(config.default_dialog)
63                .expect("dialog-process spawned without dialog handler"),
64            args_file,
65        ),
66        Err(e) => match e {
67            std::env::VarError::NotPresent => {}
68            e => panic!("invalid dialog env args, {e:?}"),
69        },
70    }
71
72    crash_handler_monitor_process(
73        config.dump_dir,
74        config.app_process,
75        config.dialog_process,
76        config.default_dialog.is_some() || config.dialog.is_some(),
77    );
78});
79
80/// Gets the number of crash restarts in the app-process.
81///
82/// Always returns zero if called in other processes.
83pub fn restart_count() -> usize {
84    match std::env::var(APP_PROCESS) {
85        Ok(c) => c.strip_prefix("restart-").unwrap_or("0").parse().unwrap_or(0),
86        Err(_) => 0,
87    }
88}
89
90const APP_PROCESS: &str = "ZNG_CRASH_HANDLER_APP";
91const DIALOG_PROCESS: &str = "ZNG_CRASH_HANDLER_DIALOG";
92const DUMP_CHANNEL: &str = "ZNG_MINIDUMP_CHANNEL";
93const RESPONSE_PREFIX: &str = "zng_crash_response: ";
94
95#[doc(hidden)]
96#[linkme::distributed_slice]
97pub static CRASH_CONFIG: [fn(&mut CrashConfig)];
98
99#[doc(hidden)]
100pub use linkme as __linkme;
101
102/// <span data-del-macro-root></span> Register a `FnOnce(&mut CrashConfig)` closure to be
103/// called on process init to configure the crash handler.
104///
105/// See [`CrashConfig`] for more details.
106#[macro_export]
107macro_rules! crash_handler_config {
108    ($closure:expr) => {
109        // expanded from:
110        #[$crate::crash_handler::__linkme::distributed_slice($crate::crash_handler::CRASH_CONFIG)]
111        #[linkme(crate = $crate::crash_handler::__linkme)]
112        #[doc(hidden)]
113        static _CRASH_CONFIG: fn(&mut $crate::crash_handler::CrashConfig) = _crash_config;
114        #[doc(hidden)]
115        fn _crash_config(cfg: &mut $crate::crash_handler::CrashConfig) {
116            fn crash_config(cfg: &mut $crate::crash_handler::CrashConfig, handler: impl FnOnce(&mut $crate::crash_handler::CrashConfig)) {
117                handler(cfg)
118            }
119            crash_config(cfg, $closure)
120        }
121    };
122}
123pub use crate::crash_handler_config;
124
125type ConfigProcess = Vec<Box<dyn for<'a, 'b> FnMut(&'a mut std::process::Command, &'b CrashArgs) -> &'a mut std::process::Command>>;
126type CrashDialogHandler = Box<dyn FnOnce(CrashArgs)>;
127
128/// Crash handler config.
129///
130/// Use [`crash_handler_config!`] to set config.
131///
132/// [`crash_handler_config!`]: crate::crash_handler_config!
133pub struct CrashConfig {
134    default_dialog: Option<CrashDialogHandler>,
135    dialog: Option<CrashDialogHandler>,
136    app_process: ConfigProcess,
137    dialog_process: ConfigProcess,
138    dump_dir: Option<PathBuf>,
139    no_crash_handler: bool,
140}
141impl CrashConfig {
142    fn new() -> Self {
143        Self {
144            default_dialog: None,
145            dialog: None,
146            app_process: vec![],
147            dialog_process: vec![],
148            dump_dir: Some(zng_env::cache("zng_minidump")),
149            no_crash_handler: false,
150        }
151    }
152
153    /// Set the crash dialog process handler.
154    ///
155    /// The dialog `handler` can run an app or show a native dialog, it must use the [`CrashArgs`] process
156    /// terminating methods to respond, if it returns [`CrashArgs::exit`] will run.
157    ///
158    /// Note that the handler does not need to actually show any dialog, it can just save crash info and
159    /// restart the app for example.
160    pub fn dialog(&mut self, handler: impl FnOnce(CrashArgs) + 'static) {
161        if self.dialog.is_none() {
162            self.dialog = Some(Box::new(handler));
163        }
164    }
165
166    /// Set the crash dialog-handler used if `crash_dialog` is not set.
167    ///
168    /// This is used by app libraries or themes to provide a default dialog.
169    pub fn default_dialog(&mut self, handler: impl FnOnce(CrashArgs) + 'static) {
170        self.default_dialog = Some(Box::new(handler));
171    }
172
173    /// Add a closure that is called just before the app-process is spawned.
174    pub fn app_process(
175        &mut self,
176        cfg: impl for<'a, 'b> FnMut(&'a mut std::process::Command, &'b CrashArgs) -> &'a mut std::process::Command + 'static,
177    ) {
178        self.app_process.push(Box::new(cfg));
179    }
180
181    /// Add a closure that is called just before the dialog-process is spawned.
182    pub fn dialog_process(
183        &mut self,
184        cfg: impl for<'a, 'b> FnMut(&'a mut std::process::Command, &'b CrashArgs) -> &'a mut std::process::Command + 'static,
185    ) {
186        self.dialog_process.push(Box::new(cfg));
187    }
188
189    /// Change the minidump directory.
190    ///
191    /// Is `zng::env::cache("zng_minidump")` by default.
192    pub fn minidump_dir(&mut self, dir: impl Into<PathBuf>) {
193        self.dump_dir = Some(dir.into());
194    }
195
196    /// Do not collect a minidump.
197    pub fn no_minidump(&mut self) {
198        self.dump_dir = None;
199    }
200
201    /// Does not run with crash handler.
202    ///
203    /// This is equivalent of running with `NO_ZNG_CRASH_HANDLER` env var.
204    pub fn no_crash_handler(&mut self) {
205        self.no_crash_handler = true;
206    }
207}
208
209/// Arguments for the crash handler dialog function.
210#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
211#[non_exhaustive]
212pub struct CrashArgs {
213    /// Info about the app-process crashes.
214    ///
215    /// Has at least one entry, latest is last. Includes all crashes since the start of the monitor-process.
216    pub app_crashes: Vec<CrashError>,
217
218    /// Info about a crash in the dialog-process spawned to handle the latest app-process crash.
219    ///
220    /// If set this is the last chance to show something to the end user, if the current dialog crashes too
221    /// the monitor-process will give up. If you started an `APP` to show a crash dialog try using a native
222    /// dialog directly now, or just give up, clearly things are far from ok.
223    pub dialog_crash: Option<CrashError>,
224}
225impl CrashArgs {
226    /// Latest crash.
227    pub fn latest(&self) -> &CrashError {
228        self.app_crashes.last().unwrap()
229    }
230
231    /// Restart the app-process with same argument as the latest crash.
232    pub fn restart(&self) -> ! {
233        let json_args = serde_json::to_string(&self.latest().args[..]).unwrap();
234        println!("{RESPONSE_PREFIX}restart {json_args}");
235        zng_env::exit(0)
236    }
237
238    /// Restart the app-process with custom arguments.
239    pub fn restart_with(&self, args: &[Txt]) -> ! {
240        let json_args = serde_json::to_string(&args).unwrap();
241        println!("{RESPONSE_PREFIX}restart {json_args}");
242        zng_env::exit(0)
243    }
244
245    /// Exit the monitor-process (application) with code.
246    pub fn exit(&self, code: i32) -> ! {
247        println!("{RESPONSE_PREFIX}exit {code}");
248        zng_env::exit(0)
249    }
250}
251impl fmt::Display for CrashArgs {
252    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
253        writeln!(f, "APP CRASHES:\n")?;
254
255        for c in self.app_crashes.iter() {
256            writeln!(f, "{c}")?;
257        }
258
259        if let Some(c) = &self.dialog_crash {
260            writeln!(f, "\nDIALOG CRASH:\n")?;
261            writeln!(f, "{c}")?;
262        }
263
264        Ok(())
265    }
266}
267
268/// Info about an app-process crash.
269#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
270#[non_exhaustive]
271pub struct CrashError {
272    /// Crash moment.
273    pub timestamp: SystemTime,
274    /// Process exit code.
275    pub code: Option<i32>,
276    /// Unix signal that terminated the process.
277    pub signal: Option<i32>,
278    /// Full capture of the app stdout.
279    pub stdout: Txt,
280    /// Full capture of the app stderr.
281    pub stderr: Txt,
282    /// Arguments used.
283    pub args: Box<[Txt]>,
284    /// Minidump file.
285    pub minidump: Option<PathBuf>,
286    /// Operating system.
287    ///
288    /// See [`std::env::consts::OS`] for details.
289    pub os: Txt,
290}
291/// Alternate mode `{:#}` prints plain stdout and stderr (no ANSI escape sequences).
292impl fmt::Display for CrashError {
293    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
294        writeln!(f, "timestamp: {}", self.unix_time())?;
295        if let Some(c) = self.code {
296            writeln!(f, "exit code: {c:#X}")?
297        }
298        if let Some(c) = self.signal {
299            writeln!(f, "exit signal: {c}")?
300        }
301        if let Some(p) = self.minidump.as_ref() {
302            writeln!(f, "minidump: {}", p.display())?
303        }
304        if f.alternate() {
305            write!(f, "\nSTDOUT:\n{}\nSTDERR:\n{}\n", self.stdout_plain(), self.stderr_plain())
306        } else {
307            write!(f, "\nSTDOUT:\n{}\nSTDERR:\n{}\n", self.stdout, self.stderr)
308        }
309    }
310}
311impl CrashError {
312    fn new(
313        timestamp: SystemTime,
314        code: Option<i32>,
315        signal: Option<i32>,
316        stdout: Txt,
317        stderr: Txt,
318        minidump: Option<PathBuf>,
319        args: Box<[Txt]>,
320    ) -> Self {
321        Self {
322            timestamp,
323            code,
324            signal,
325            stdout,
326            stderr,
327            args,
328            minidump,
329            os: std::env::consts::OS.into(),
330        }
331    }
332
333    /// Seconds since Unix epoch.
334    pub fn unix_time(&self) -> u64 {
335        self.timestamp.duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default().as_secs()
336    }
337
338    /// Gets if `stdout` does not contain any ANSI scape sequences.
339    pub fn is_stdout_plain(&self) -> bool {
340        !tap::contains_ansi_csi(&self.stdout)
341    }
342
343    /// Gets if `stderr` does not contain any ANSI scape sequences.
344    pub fn is_stderr_plain(&self) -> bool {
345        !tap::contains_ansi_csi(&self.stderr)
346    }
347
348    /// Get `stdout` without any ANSI escape sequences (CSI).
349    pub fn stdout_plain(&self) -> Txt {
350        if self.is_stdout_plain() {
351            self.stdout.clone()
352        } else {
353            tap::remove_ansi_csi(&self.stdout)
354        }
355    }
356
357    /// Get `stderr` without any ANSI escape sequences (CSI).
358    pub fn stderr_plain(&self) -> Txt {
359        if self.is_stderr_plain() {
360            self.stderr.clone()
361        } else {
362            tap::remove_ansi_csi(&self.stderr)
363        }
364    }
365
366    /// Gets if `stderr` contains a crash panic.
367    pub fn has_panic(&self) -> bool {
368        if self.code == Some(101) {
369            PanicInfo::contains(&self.stderr)
370        } else {
371            false
372        }
373    }
374
375    /// Gets if `stderr` contains a crash panic that traced widget/window path.
376    pub fn has_panic_widget(&self) -> bool {
377        if self.code == Some(101) {
378            PanicInfo::contains_widget(&self.stderr)
379        } else {
380            false
381        }
382    }
383
384    /// Try parse `stderr` for the crash panic.
385    ///
386    /// Only reliably works if the panic fully printed correctly and was formatted by the panic
387    /// hook installed by `crash_handler` or by the display print of [`PanicInfo`].
388    pub fn find_panic(&self) -> Option<PanicInfo> {
389        if self.code == Some(101) {
390            PanicInfo::find(&self.stderr)
391        } else {
392            None
393        }
394    }
395
396    /// Best attempt at generating a readable error message.
397    ///
398    /// Is the panic message, or the minidump exception, with the exit code and signal.
399    pub fn message(&self) -> Txt {
400        let mut msg = if let Some(msg) = self.find_panic().map(|p| p.message) {
401            msg
402        } else if let Some(msg) = self.minidump_message() {
403            msg
404        } else {
405            "".into()
406        };
407        use std::fmt::Write as _;
408
409        if let Some(c) = self.code {
410            let sep = if msg.is_empty() { "" } else { "\n" };
411            write!(&mut msg, "{sep}Code: {c:#X}").unwrap();
412        }
413        if let Some(c) = self.signal {
414            let sep = if msg.is_empty() { "" } else { "\n" };
415            write!(&mut msg, "{sep}Signal: {c}").unwrap();
416        }
417        msg.end_mut();
418        msg
419    }
420
421    fn minidump_message(&self) -> Option<Txt> {
422        use minidump::*;
423
424        let dump = match Minidump::read_path(self.minidump.as_ref()?) {
425            Ok(d) => d,
426            Err(e) => {
427                tracing::error!("error reading minidump, {e}");
428                return None;
429            }
430        };
431
432        let exception = match dump.get_stream::<MinidumpException>() {
433            Ok(s) => s,
434            Err(e) => {
435                tracing::error!("error reading minidump exception, {e}");
436                return None;
437            }
438        };
439
440        #[cfg(debug_assertions)]
441        {
442            // nice error messages, but adds >1MB of binary code
443            let system_info = match dump.get_stream::<MinidumpSystemInfo>() {
444                Ok(s) => s,
445                Err(e) => {
446                    tracing::error!("error reading minidump system info, {e}");
447                    return None;
448                }
449            };
450            let crash_reason = exception.get_crash_reason(system_info.os, system_info.cpu);
451            Some(zng_txt::formatx!("{crash_reason}"))
452        }
453
454        #[cfg(not(debug_assertions))]
455        {
456            // raw error code, only common names
457            let raw = exception.raw;
458
459            let code = raw.exception_record.exception_code;
460            let addr = raw.exception_record.exception_address;
461
462            cfg_select! {
463                windows => {
464                    let name = match code {
465                        0xC0000005 => "ACCESS_VIOLATION",
466                        0xC0000409 => "STACK_BUFFER_OVERRUN",
467                        0x80000003 => "BREAKPOINT",
468                        0xC000001D => "ILLEGAL_INSTRUCTION",
469                        0xC0000094 => "INTEGER_DIVIDE_BY_ZERO",
470                        0xC00000FD => "STACK_OVERFLOW",
471                        0xC0000096 => "PRIVILEGED_INSTRUCTION",
472                        0xC0000008 => "INVALID_HANDLE",
473                        0xC0000135 => "DLL_NOT_FOUND",
474                        _ => "",
475                    };
476                }
477                any(target_os = "linux", target_os = "android") => {
478                    let name = match code as i32 {
479                        4 => "SIGILL",
480                        5 => "SIGTRAP",
481                        6 => "SIGABRT",
482                        7 => "SIGBUS",
483                        8 => "SIGFPE",
484                        9 => "SIGKILL",
485                        11 => "SIGSEGV",
486                        13 => "SIGPIPE",
487                        _ => "",
488                    };
489                }
490                any(target_os = "macos", target_os = "ios") => {
491                    let name = match code as i32 {
492                        4 => "SIGILL",
493                        5 => "SIGTRAP",
494                        6 => "SIGABRT",
495                        8 => "SIGFPE",
496                        10 => "SIGBUS",
497                        11 => "SIGSEGV",
498                        _ => "",
499                    };
500                }
501                _ => {
502                    let name = "";
503                }
504            }
505            if name.is_empty() {
506                Some(zng_txt::formatx!("exception 0x{code:08X} at 0x{addr:X}"))
507            } else {
508                Some(zng_txt::formatx!("exception 0x{code:08X} ({name}) at 0x{addr:X}"))
509            }
510        }
511    }
512}
513
514fn crash_handler_monitor_process(
515    dump_dir: Option<PathBuf>,
516    mut cfg_app: ConfigProcess,
517    mut cfg_dialog: ConfigProcess,
518    has_dialog_handler: bool,
519) -> ! {
520    zng_env::set_process_name("crash-handler-process");
521
522    let exe = std::env::current_exe()
523        .and_then(dunce::canonicalize)
524        .expect("failed to get the current executable");
525
526    let mut args: Box<[_]> = std::env::args().skip(1).map(Txt::from).collect();
527
528    let mut dialog_args = CrashArgs {
529        app_crashes: vec![],
530        dialog_crash: None,
531    };
532    loop {
533        let mut app_process = std::process::Command::new(&exe);
534        for cfg in &mut cfg_app {
535            cfg(&mut app_process, &dialog_args);
536        }
537
538        match run_process(
539            dump_dir.as_deref(),
540            app_process
541                .env(APP_PROCESS, format!("restart-{}", dialog_args.app_crashes.len()))
542                .args(args.iter()),
543        ) {
544            Ok((status, stdout, stderr, dump_file)) => {
545                if status.success() {
546                    let code = status.code().unwrap_or(0);
547                    tracing::info!(
548                        "crash monitor-process exiting with success code ({code}), {} crashes",
549                        dialog_args.app_crashes.len()
550                    );
551                    zng_env::exit(code);
552                } else {
553                    let code = status.code();
554                    #[allow(unused_mut)] // Windows has no signal
555                    let mut signal = None::<i32>;
556
557                    #[cfg(windows)]
558                    if code == Some(1) {
559                        tracing::warn!(
560                            "app-process exit code (1), probably killed by the system, \
561                                        will exit monitor-process with the same code"
562                        );
563                        zng_env::exit(1);
564                    }
565                    #[cfg(unix)]
566                    if code.is_none() {
567                        use std::os::unix::process::ExitStatusExt as _;
568                        signal = status.signal();
569
570                        if let Some(sig) = signal
571                            && [2, 9, 17, 19, 23].contains(&sig)
572                        {
573                            tracing::warn!(
574                                "app-process exited by signal ({sig}), \
575                                                will exit monitor-process with code 1"
576                            );
577                            zng_env::exit(1);
578                        }
579                    }
580
581                    tracing::error!(
582                        "app-process crashed with exit code ({:#X}), signal ({:#?}), {} crashes previously",
583                        code.unwrap_or(0),
584                        signal.unwrap_or(0),
585                        dialog_args.app_crashes.len()
586                    );
587
588                    let timestamp = SystemTime::now();
589
590                    dialog_args.app_crashes.push(CrashError::new(
591                        timestamp,
592                        code,
593                        signal,
594                        stdout.into_txt_blocking(false),
595                        stderr.into_txt_blocking(false),
596                        dump_file,
597                        args.clone(),
598                    ));
599
600                    // show dialog, retries once if dialog crashes too.
601                    for _ in 0..2 {
602                        // serialize app-crashes to a temp JSON file
603                        let timestamp_nanos = timestamp.duration_since(SystemTime::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0);
604                        let mut timestamp = timestamp_nanos;
605                        let mut retries = 0;
606                        let crash_file = loop {
607                            let path = std::env::temp_dir().join(format!("zng-crash-errors-{timestamp:#x}"));
608                            match std::fs::File::create_new(&path) {
609                                Ok(f) => match serde_json::to_writer(std::io::BufWriter::new(f), &dialog_args) {
610                                    Ok(_) => break path,
611                                    Err(e) => {
612                                        if e.is_io() {
613                                            if retries > 20 {
614                                                panic!("error writing crash errors, {e}");
615                                            } else if retries > 5 {
616                                                timestamp += 1;
617                                            }
618                                            std::thread::sleep(100.ms());
619                                        } else {
620                                            panic!("error serializing crash errors, {e}");
621                                        }
622                                    }
623                                },
624                                Err(e) => {
625                                    if e.kind() == std::io::ErrorKind::AlreadyExists {
626                                        timestamp += 1;
627                                    } else {
628                                        if retries > 20 {
629                                            panic!("error creating crash errors file, {e}");
630                                        } else if retries > 5 {
631                                            timestamp += 1;
632                                        }
633                                        std::thread::sleep(100.ms());
634                                    }
635                                }
636                            }
637                            retries += 1;
638                        };
639
640                        let dialog_result = if has_dialog_handler {
641                            let mut dialog_process = std::process::Command::new(&exe);
642                            for cfg in &mut cfg_dialog {
643                                cfg(&mut dialog_process, &dialog_args);
644                            }
645                            run_process(dump_dir.as_deref(), dialog_process.env(DIALOG_PROCESS, &crash_file))
646                        } else {
647                            Ok((
648                                std::process::ExitStatus::default(),
649                                tap::StdoutTap::dummy(),
650                                tap::StderrTap::dummy(),
651                                None,
652                            ))
653                        };
654
655                        for _ in 0..5 {
656                            if !crash_file.exists() || std::fs::remove_file(&crash_file).is_ok() {
657                                break;
658                            }
659                            std::thread::sleep(100.ms());
660                        }
661
662                        let response = match dialog_result {
663                            Ok((dlg_status, dlg_stdout, dlg_stderr, dlg_dump_file)) => {
664                                if dlg_status.success() {
665                                    let dlg_stdout = dlg_stdout.into_string_blocking(false);
666                                    dlg_stdout
667                                        .lines()
668                                        .filter_map(|l| l.trim().strip_prefix(RESPONSE_PREFIX))
669                                        .next_back()
670                                        .unwrap_or("exit 0")
671                                        .to_owned()
672                                } else {
673                                    let code = dlg_status.code();
674                                    #[allow(unused_mut)] // Windows has no signal
675                                    let mut signal = None::<i32>;
676
677                                    #[cfg(windows)]
678                                    if code == Some(1) {
679                                        tracing::warn!(
680                                            "dialog-process exit code (1), probably killed by the system, \
681                                                        will exit monitor-process with the same code"
682                                        );
683                                        zng_env::exit(1);
684                                    }
685                                    #[cfg(unix)]
686                                    if code.is_none() {
687                                        use std::os::unix::process::ExitStatusExt as _;
688                                        signal = status.signal();
689
690                                        if let Some(sig) = signal
691                                            && [2, 9, 17, 19, 23].contains(&sig)
692                                        {
693                                            tracing::warn!(
694                                                "dialog-process exited by signal ({sig}), \
695                                                                will exit monitor-process with code 1"
696                                            );
697                                            zng_env::exit(1);
698                                        }
699                                    }
700
701                                    let dialog_crash = CrashError::new(
702                                        SystemTime::now(),
703                                        code,
704                                        signal,
705                                        dlg_stdout.into_txt_blocking(false),
706                                        dlg_stderr.into_txt_blocking(false),
707                                        dlg_dump_file,
708                                        Box::new([]),
709                                    );
710                                    tracing::error!("crash dialog-process crashed, {dialog_crash:#}");
711
712                                    if dialog_args.dialog_crash.is_none() {
713                                        dialog_args.dialog_crash = Some(dialog_crash);
714                                        continue;
715                                    } else {
716                                        let latest = dialog_args.latest();
717                                        eprintln!("{latest}");
718                                        zng_env::exit(latest.code.unwrap_or(1));
719                                    }
720                                }
721                            }
722                            Err(e) => panic!("error running dialog-process, {e}"),
723                        };
724
725                        if let Some(args_json) = response.strip_prefix("restart ") {
726                            args = serde_json::from_str(args_json).expect("crash dialog-process did not respond 'restart' correctly");
727                            break;
728                        } else if let Some(code) = response.strip_prefix("exit ") {
729                            let code: i32 = code.parse().expect("crash dialog-process did not respond 'code' correctly");
730                            zng_env::exit(code);
731                        } else {
732                            panic!("crash dialog-process did not respond correctly")
733                        }
734                    }
735                }
736            }
737            Err(e) => panic!("error running app-process, {e}"),
738        }
739    }
740}
741fn run_process(
742    dump_dir: Option<&Path>,
743    command: &mut std::process::Command,
744) -> std::io::Result<(std::process::ExitStatus, tap::StdoutTap, tap::StderrTap, Option<PathBuf>)> {
745    struct DumpServer {
746        shutdown: Arc<AtomicBool>,
747        runner: std::thread::JoinHandle<Option<PathBuf>>,
748    }
749    let mut dump_server = None;
750    if let Some(dump_dir) = dump_dir {
751        match std::fs::create_dir_all(dump_dir) {
752            Ok(_) => {
753                let uuid = uuid::Uuid::new_v4();
754                let dump_file = dump_dir.join(format!("{}.dmp", uuid.simple()));
755                let dump_channel = std::env::temp_dir().join(format!("zng-crash-{}", uuid.simple()));
756                match minidumper::Server::with_name(minidumper::SocketName::Path(&dump_channel)) {
757                    Ok(mut s) => {
758                        command.env(DUMP_CHANNEL, &dump_channel);
759                        let shutdown = Arc::new(AtomicBool::new(false));
760                        let runner = std::thread::Builder::new()
761                            .name("minidumper-server".into())
762                            .stack_size(512 * 1024)
763                            .spawn(clmv!(shutdown, || {
764                                let created_file = Arc::new(Mutex::new(None));
765                                if let Err(e) = s.run(
766                                    Box::new(MinidumpServerHandler {
767                                        dump_file,
768                                        created_file: created_file.clone(),
769                                    }),
770                                    &shutdown,
771                                    None,
772                                ) {
773                                    tracing::error!("minidump server exited with error, {e}");
774                                }
775                                created_file.lock().take()
776                            }))
777                            .expect("failed to spawn thread");
778                        dump_server = Some(DumpServer { shutdown, runner });
779                    }
780                    Err(e) => tracing::error!("failed to spawn minidump server, will not enable crash handling, {e}"),
781                }
782            }
783            Err(e) => tracing::error!("cannot create minidump dir, will not enable crash handling, {e}"),
784        }
785    }
786
787    let mut app_process = command
788        .env("RUST_BACKTRACE", "full")
789        .env("CLICOLOR_FORCE", "1")
790        .stdout(std::process::Stdio::piped())
791        .stderr(std::process::Stdio::piped())
792        .spawn()?;
793
794    let stdout = tap::StdoutTap::new_blocking(app_process.stdout.take().unwrap());
795    let stderr = tap::StderrTap::new_blocking(app_process.stderr.take().unwrap());
796
797    let status = app_process.wait()?;
798
799    let mut dump_file = None;
800    if let Some(s) = dump_server {
801        s.shutdown.store(true, atomic::Ordering::Relaxed);
802        match s.runner.join() {
803            Ok(r) => dump_file = r,
804            Err(p) => std::panic::resume_unwind(p),
805        };
806    }
807
808    Ok((status, stdout, stderr, dump_file))
809}
810struct MinidumpServerHandler {
811    dump_file: PathBuf,
812    created_file: Arc<Mutex<Option<PathBuf>>>,
813}
814impl minidumper::ServerHandler for MinidumpServerHandler {
815    fn create_minidump_file(&self) -> Result<(std::fs::File, PathBuf), std::io::Error> {
816        let file = std::fs::File::create_new(&self.dump_file)?;
817        Ok((file, self.dump_file.clone()))
818    }
819
820    fn on_minidump_created(&self, result: Result<minidumper::MinidumpBinary, minidumper::Error>) -> minidumper::LoopAction {
821        match result {
822            Ok(b) => *self.created_file.lock() = Some(b.path),
823            Err(e) => tracing::error!("failed to write minidump file, {e}"),
824        }
825        minidumper::LoopAction::Exit
826    }
827
828    fn on_message(&self, _: u32, _: Vec<u8>) {}
829
830    fn on_client_connected(&self, num_clients: usize) -> minidumper::LoopAction {
831        if num_clients > 1 {
832            tracing::error!("expected only one minidump client, {num_clients} connected, exiting server");
833            minidumper::LoopAction::Exit
834        } else {
835            minidumper::LoopAction::Continue
836        }
837    }
838
839    fn on_client_disconnected(&self, num_clients: usize) -> minidumper::LoopAction {
840        if num_clients != 0 {
841            tracing::error!("expected only one minidump client disconnect, {num_clients} still connected");
842        }
843        minidumper::LoopAction::Exit
844    }
845}
846
847fn crash_handler_app_process(dump_enabled: bool) {
848    PanicInfo::set_hook(|| crate::widget::WIDGET.trace_path());
849    if dump_enabled {
850        minidump_attach();
851    }
852
853    // app-process execution happens after this.
854}
855
856fn crash_handler_dialog_process(dump_enabled: bool, dialog: CrashDialogHandler, args_file: String) -> ! {
857    zng_env::set_process_name("crash-dialog-process");
858
859    PanicInfo::set_hook(|| crate::widget::WIDGET.trace_path());
860    if dump_enabled {
861        minidump_attach();
862    }
863
864    let mut retries = 0;
865    let args = loop {
866        match std::fs::read_to_string(&args_file) {
867            Ok(args) => break args,
868            Err(e) => {
869                if e.kind() != std::io::ErrorKind::NotFound && retries < 10 {
870                    retries += 1;
871                    continue;
872                }
873                panic!("error reading args file, {e}");
874            }
875        }
876    };
877
878    dialog(serde_json::from_str(&args).expect("error deserializing args"));
879    CrashArgs {
880        app_crashes: vec![],
881        dialog_crash: None,
882    }
883    .exit(0)
884}
885
886fn minidump_attach() {
887    let channel_name = match std::env::var(DUMP_CHANNEL) {
888        Ok(n) if !n.is_empty() => PathBuf::from(n),
889        _ => {
890            eprintln!("expected minidump channel name, this instance will not handle crashes");
891            return;
892        }
893    };
894    let client = match minidumper::Client::with_name(minidumper::SocketName::Path(&channel_name)) {
895        Ok(c) => c,
896        Err(e) => {
897            eprintln!("failed to connect minidump client, this instance will not handle crashes, {e}");
898            return;
899        }
900    };
901    struct Handler(minidumper::Client);
902    // SAFETY: on_crash does the minimal possible work
903    unsafe impl crash_handler::CrashEvent for Handler {
904        fn on_crash(&self, context: &crash_handler::CrashContext) -> crash_handler::CrashEventResult {
905            crash_handler::CrashEventResult::Handled(self.0.request_dump(context).is_ok())
906        }
907    }
908    let handler = match crash_handler::CrashHandler::attach(Box::new(Handler(client))) {
909        Ok(h) => h,
910        Err(e) => {
911            eprintln!("failed attach minidump crash handler, this instance will not handle crashes, {e}");
912            return;
913        }
914    };
915
916    *CRASH_HANDLER.lock() = Some(handler);
917}
918static CRASH_HANDLER: Mutex<Option<crash_handler::CrashHandler>> = Mutex::new(None);
zng_app/crash_handler.rs

zng_app/
crash_handler.rs
