Add minidump crash reporting (#35263)

- [x] Handle uploading minidumps from the remote_server
- [x] Associate minidumps with panics with some sort of ID (we don't use
session_id on the remote)
  - [x] Update the protobufs and client/server code to request panics
- [x] Upload minidumps with no corresponding panic
- [x] Fill in panic info when there _is_ a corresponding panic
- [x] Use an env var for the sentry endpoint instead of hardcoding it

Release Notes:

- Zed now generates minidumps for crash reporting

---------

Co-authored-by: Max Brunsfeld <maxbrunsfeld@gmail.com>

crates/client/src/telemetry.rs

@@ -74,6 +74,12 @@ static ZED_CLIENT_CHECKSUM_SEED: LazyLock<Option<Vec<u8>>> = LazyLock::new(|| {
         })
 });
 
+pub static SENTRY_MINIDUMP_ENDPOINT: LazyLock<Option<String>> = LazyLock::new(|| {
+    option_env!("SENTRY_MINIDUMP_ENDPOINT")
+        .map(|s| s.to_owned())
+        .or_else(|| env::var("SENTRY_MINIDUMP_ENDPOINT").ok())
+});
+
 static DOTNET_PROJECT_FILES_REGEX: LazyLock<Regex> = LazyLock::new(|| {
     Regex::new(r"^(global\.json|Directory\.Build\.props|.*\.(csproj|fsproj|vbproj|sln))$").unwrap()
 });

crates/crashes/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "crashes"
+version = "0.1.0"
+publish.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later"
+
+[dependencies]
+crash-handler.workspace = true
+log.workspace = true
+minidumper.workspace = true
+paths.workspace = true
+smol.workspace = true
+workspace-hack.workspace = true
+
+[lints]
+workspace = true
+
+[lib]
+path = "src/crashes.rs"

crates/crashes/LICENSE-GPL

@@ -0,0 +1 @@
+../../LICENSE-GPL

crates/crashes/src/crashes.rs

@@ -0,0 +1,172 @@
+use crash_handler::CrashHandler;
+use log::info;
+use minidumper::{Client, LoopAction, MinidumpBinary};
+
+use std::{
+    env,
+    fs::File,
+    io,
+    path::{Path, PathBuf},
+    process::{self, Command},
+    sync::{
+        OnceLock,
+        atomic::{AtomicBool, Ordering},
+    },
+    thread,
+    time::Duration,
+};
+
+// set once the crash handler has initialized and the client has connected to it
+pub static CRASH_HANDLER: AtomicBool = AtomicBool::new(false);
+// set when the first minidump request is made to avoid generating duplicate crash reports
+pub static REQUESTED_MINIDUMP: AtomicBool = AtomicBool::new(false);
+const CRASH_HANDLER_TIMEOUT: Duration = Duration::from_secs(60);
+
+pub async fn init(id: String) {
+    let exe = env::current_exe().expect("unable to find ourselves");
+    let zed_pid = process::id();
+    // TODO: we should be able to get away with using 1 crash-handler process per machine,
+    // but for now we append the PID of the current process which makes it unique per remote
+    // server or interactive zed instance. This solves an issue where occasionally the socket
+    // used by the crash handler isn't destroyed correctly which causes it to stay on the file
+    // system and block further attempts to initialize crash handlers with that socket path.
+    let socket_name = paths::temp_dir().join(format!("zed-crash-handler-{zed_pid}"));
+    #[allow(unused)]
+    let server_pid = Command::new(exe)
+        .arg("--crash-handler")
+        .arg(&socket_name)
+        .spawn()
+        .expect("unable to spawn server process")
+        .id();
+    info!("spawning crash handler process");
+
+    let mut elapsed = Duration::ZERO;
+    let retry_frequency = Duration::from_millis(100);
+    let mut maybe_client = None;
+    while maybe_client.is_none() {
+        if let Ok(client) = Client::with_name(socket_name.as_path()) {
+            maybe_client = Some(client);
+            info!("connected to crash handler process after {elapsed:?}");
+            break;
+        }
+        elapsed += retry_frequency;
+        smol::Timer::after(retry_frequency).await;
+    }
+    let client = maybe_client.unwrap();
+    client.send_message(1, id).unwrap(); // set session id on the server
+
+    let client = std::sync::Arc::new(client);
+    let handler = crash_handler::CrashHandler::attach(unsafe {
+        let client = client.clone();
+        crash_handler::make_crash_event(move |crash_context: &crash_handler::CrashContext| {
+            // only request a minidump once
+            let res = if REQUESTED_MINIDUMP
+                .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
+                .is_ok()
+            {
+                client.send_message(2, "mistakes were made").unwrap();
+                client.ping().unwrap();
+                client.request_dump(crash_context).is_ok()
+            } else {
+                true
+            };
+            crash_handler::CrashEventResult::Handled(res)
+        })
+    })
+    .expect("failed to attach signal handler");
+
+    #[cfg(target_os = "linux")]
+    {
+        handler.set_ptracer(Some(server_pid));
+    }
+    CRASH_HANDLER.store(true, Ordering::Release);
+    std::mem::forget(handler);
+    info!("crash handler registered");
+
+    loop {
+        client.ping().ok();
+        smol::Timer::after(Duration::from_secs(10)).await;
+    }
+}
+
+pub struct CrashServer {
+    session_id: OnceLock<String>,
+}
+
+impl minidumper::ServerHandler for CrashServer {
+    fn create_minidump_file(&self) -> Result<(File, PathBuf), io::Error> {
+        let err_message = "Need to send a message with the ID upon starting the crash handler";
+        let dump_path = paths::logs_dir()
+            .join(self.session_id.get().expect(err_message))
+            .with_extension("dmp");
+        let file = File::create(&dump_path)?;
+        Ok((file, dump_path))
+    }
+
+    fn on_minidump_created(&self, result: Result<MinidumpBinary, minidumper::Error>) -> LoopAction {
+        match result {
+            Ok(mut md_bin) => {
+                use io::Write;
+                let _ = md_bin.file.flush();
+                info!("wrote minidump to disk {:?}", md_bin.path);
+            }
+            Err(e) => {
+                info!("failed to write minidump: {:#}", e);
+            }
+        }
+        LoopAction::Exit
+    }
+
+    fn on_message(&self, kind: u32, buffer: Vec<u8>) {
+        let message = String::from_utf8(buffer).expect("invalid utf-8");
+        info!("kind: {kind}, message: {message}",);
+        if kind == 1 {
+            self.session_id
+                .set(message)
+                .expect("session id already initialized");
+        }
+    }
+
+    fn on_client_disconnected(&self, clients: usize) -> LoopAction {
+        info!("client disconnected, {clients} remaining");
+        if clients == 0 {
+            LoopAction::Exit
+        } else {
+            LoopAction::Continue
+        }
+    }
+}
+
+pub fn handle_panic() {
+    // wait 500ms for the crash handler process to start up
+    // if it's still not there just write panic info and no minidump
+    let retry_frequency = Duration::from_millis(100);
+    for _ in 0..5 {
+        if CRASH_HANDLER.load(Ordering::Acquire) {
+            log::error!("triggering a crash to generate a minidump...");
+            #[cfg(target_os = "linux")]
+            CrashHandler.simulate_signal(crash_handler::Signal::Trap as u32);
+            #[cfg(not(target_os = "linux"))]
+            CrashHandler.simulate_exception(None);
+            break;
+        }
+        thread::sleep(retry_frequency);
+    }
+}
+
+pub fn crash_server(socket: &Path) {
+    let Ok(mut server) = minidumper::Server::with_name(socket) else {
+        log::info!("Couldn't create socket, there may already be a running crash server");
+        return;
+    };
+    let ab = AtomicBool::new(false);
+    server
+        .run(
+            Box::new(CrashServer {
+                session_id: OnceLock::new(),
+            }),
+            &ab,
+            Some(CRASH_HANDLER_TIMEOUT),
+        )
+        .expect("failed to run server");
+}

crates/http_client/Cargo.toml

@@ -24,6 +24,7 @@ http.workspace = true
 http-body.workspace = true
 log.workspace = true
 parking_lot.workspace = true
+reqwest.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 url.workspace = true

crates/http_client/src/async_body.rs

@@ -88,6 +88,17 @@ impl From<&'static str> for AsyncBody {
     }
 }
 
+impl TryFrom<reqwest::Body> for AsyncBody {
+    type Error = anyhow::Error;
+
+    fn try_from(value: reqwest::Body) -> Result<Self, Self::Error> {
+        value
+            .as_bytes()
+            .ok_or_else(|| anyhow::anyhow!("Underlying data is a stream"))
+            .map(|bytes| Self::from_bytes(Bytes::copy_from_slice(bytes)))
+    }
+}
+
 impl<T: Into<Self>> From<Option<T>> for AsyncBody {
     fn from(body: Option<T>) -> Self {
         match body {

crates/http_client/src/http_client.rs

@@ -7,7 +7,10 @@ use derive_more::Deref;
 use http::HeaderValue;
 pub use http::{self, Method, Request, Response, StatusCode, Uri};
 
-use futures::future::BoxFuture;
+use futures::{
+    FutureExt as _,
+    future::{self, BoxFuture},
+};
 use http::request::Builder;
 use parking_lot::Mutex;
 #[cfg(feature = "test-support")]
@@ -89,6 +92,14 @@ pub trait HttpClient: 'static + Send + Sync {
     fn as_fake(&self) -> &FakeHttpClient {
         panic!("called as_fake on {}", type_name::<Self>())
     }
+
+    fn send_multipart_form<'a>(
+        &'a self,
+        _url: &str,
+        _request: reqwest::multipart::Form,
+    ) -> BoxFuture<'a, anyhow::Result<Response<AsyncBody>>> {
+        future::ready(Err(anyhow!("not implemented"))).boxed()
+    }
 }
 
 /// An [`HttpClient`] that may have a proxy.
@@ -140,31 +151,13 @@ impl HttpClient for HttpClientWithProxy {
     fn as_fake(&self) -> &FakeHttpClient {
         self.client.as_fake()
     }
-}
 
-impl HttpClient for Arc<HttpClientWithProxy> {
-    fn send(
-        &self,
-        req: Request<AsyncBody>,
-    ) -> BoxFuture<'static, anyhow::Result<Response<AsyncBody>>> {
-        self.client.send(req)
-    }
-
-    fn user_agent(&self) -> Option<&HeaderValue> {
-        self.client.user_agent()
-    }
-
-    fn proxy(&self) -> Option<&Url> {
-        self.proxy.as_ref()
-    }
-
-    fn type_name(&self) -> &'static str {
-        self.client.type_name()
-    }
-
-    #[cfg(feature = "test-support")]
-    fn as_fake(&self) -> &FakeHttpClient {
-        self.client.as_fake()
+    fn send_multipart_form<'a>(
+        &'a self,
+        url: &str,
+        form: reqwest::multipart::Form,
+    ) -> BoxFuture<'a, anyhow::Result<Response<AsyncBody>>> {
+        self.client.send_multipart_form(url, form)
     }
 }
 
@@ -275,32 +268,6 @@ impl HttpClientWithUrl {
     }
 }
 
-impl HttpClient for Arc<HttpClientWithUrl> {
-    fn send(
-        &self,
-        req: Request<AsyncBody>,
-    ) -> BoxFuture<'static, anyhow::Result<Response<AsyncBody>>> {
-        self.client.send(req)
-    }
-
-    fn user_agent(&self) -> Option<&HeaderValue> {
-        self.client.user_agent()
-    }
-
-    fn proxy(&self) -> Option<&Url> {
-        self.client.proxy.as_ref()
-    }
-
-    fn type_name(&self) -> &'static str {
-        self.client.type_name()
-    }
-
-    #[cfg(feature = "test-support")]
-    fn as_fake(&self) -> &FakeHttpClient {
-        self.client.as_fake()
-    }
-}
-
 impl HttpClient for HttpClientWithUrl {
     fn send(
         &self,
@@ -325,6 +292,14 @@ impl HttpClient for HttpClientWithUrl {
     fn as_fake(&self) -> &FakeHttpClient {
         self.client.as_fake()
     }
+
+    fn send_multipart_form<'a>(
+        &'a self,
+        url: &str,
+        request: reqwest::multipart::Form,
+    ) -> BoxFuture<'a, anyhow::Result<Response<AsyncBody>>> {
+        self.client.send_multipart_form(url, request)
+    }
 }
 
 pub fn read_proxy_from_env() -> Option<Url> {

crates/proto/proto/app.proto

@@ -79,11 +79,16 @@ message OpenServerSettings {
     uint64 project_id = 1;
 }
 
-message GetPanicFiles {
+message GetCrashFiles {
 }
 
-message GetPanicFilesResponse {
-    repeated string file_contents = 2;
+message GetCrashFilesResponse {
+    repeated CrashReport crashes = 1;
+}
+
+message CrashReport {
+    optional string panic_contents = 1;
+    optional bytes minidump_contents = 2;
 }
 
 message Extension {

crates/proto/proto/zed.proto

@@ -294,9 +294,6 @@ message Envelope {
         GetPathMetadata get_path_metadata = 278;
         GetPathMetadataResponse get_path_metadata_response = 279;
 
-        GetPanicFiles get_panic_files = 280;
-        GetPanicFilesResponse get_panic_files_response = 281;
-
         CancelLanguageServerWork cancel_language_server_work = 282;
 
         LspExtOpenDocs lsp_ext_open_docs = 283;
@@ -402,7 +399,10 @@ message Envelope {
         StashPop stash_pop = 358;
 
         GetDefaultBranch get_default_branch = 359;
-        GetDefaultBranchResponse get_default_branch_response = 360; // current max
+        GetDefaultBranchResponse get_default_branch_response = 360;
+
+        GetCrashFiles get_crash_files = 361;
+        GetCrashFilesResponse get_crash_files_response = 362; // current max
     }
 
     reserved 87 to 88;
@@ -423,6 +423,7 @@ message Envelope {
     reserved 270;
     reserved 247 to 254;
     reserved 255 to 256;
+    reserved 280 to 281;
 }
 
 message Hello {

crates/proto/src/proto.rs

@@ -99,8 +99,8 @@ messages!(
     (GetHoverResponse, Background),
     (GetNotifications, Foreground),
     (GetNotificationsResponse, Foreground),
-    (GetPanicFiles, Background),
-    (GetPanicFilesResponse, Background),
+    (GetCrashFiles, Background),
+    (GetCrashFilesResponse, Background),
     (GetPathMetadata, Background),
     (GetPathMetadataResponse, Background),
     (GetPermalinkToLine, Foreground),
@@ -462,7 +462,7 @@ request_messages!(
     (ActivateToolchain, Ack),
     (ActiveToolchain, ActiveToolchainResponse),
     (GetPathMetadata, GetPathMetadataResponse),
-    (GetPanicFiles, GetPanicFilesResponse),
+    (GetCrashFiles, GetCrashFilesResponse),
     (CancelLanguageServerWork, Ack),
     (SyncExtensions, SyncExtensionsResponse),
     (InstallExtension, Ack),

crates/remote_server/Cargo.toml

@@ -67,8 +67,11 @@ watch.workspace = true
 worktree.workspace = true
 
 [target.'cfg(not(windows))'.dependencies]
+crashes.workspace = true
+crash-handler.workspace = true
 fork.workspace = true
 libc.workspace = true
+minidumper.workspace = true
 
 [dev-dependencies]
 assistant_tool.workspace = true

crates/remote_server/src/main.rs

@@ -12,6 +12,10 @@ struct Cli {
     /// by having Zed act like netcat communicating over a Unix socket.
     #[arg(long, hide = true)]
     askpass: Option<String>,
+    /// Used for recording minidumps on crashes by having the server run a separate
+    /// process communicating over a socket.
+    #[arg(long, hide = true)]
+    crash_handler: Option<PathBuf>,
     /// Used for loading the environment from the project.
     #[arg(long, hide = true)]
     printenv: bool,
@@ -58,6 +62,11 @@ fn main() {
         return;
     }
 
+    if let Some(socket) = &cli.crash_handler {
+        crashes::crash_server(socket.as_path());
+        return;
+    }
+
     if cli.printenv {
         util::shell_env::print_env();
         return;

crates/remote_server/src/unix.rs

@@ -17,6 +17,7 @@ use node_runtime::{NodeBinaryOptions, NodeRuntime};
 use paths::logs_dir;
 use project::project_settings::ProjectSettings;
 
+use proto::CrashReport;
 use release_channel::{AppVersion, RELEASE_CHANNEL, ReleaseChannel};
 use remote::proxy::ProxyLaunchError;
 use remote::ssh_session::ChannelClient;
@@ -33,6 +34,7 @@ use smol::io::AsyncReadExt;
 
 use smol::Async;
 use smol::{net::unix::UnixListener, stream::StreamExt as _};
+use std::collections::HashMap;
 use std::ffi::OsStr;
 use std::ops::ControlFlow;
 use std::str::FromStr;
@@ -109,8 +111,9 @@ fn init_logging_server(log_file_path: PathBuf) -> Result<Receiver<Vec<u8>>> {
     Ok(rx)
 }
 
-fn init_panic_hook() {
-    std::panic::set_hook(Box::new(|info| {
+fn init_panic_hook(session_id: String) {
+    std::panic::set_hook(Box::new(move |info| {
+        crashes::handle_panic();
         let payload = info
             .payload()
             .downcast_ref::<&str>()
@@ -171,9 +174,11 @@ fn init_panic_hook() {
             architecture: env::consts::ARCH.into(),
             panicked_on: Utc::now().timestamp_millis(),
             backtrace,
-            system_id: None,            // Set on SSH client
-            installation_id: None,      // Set on SSH client
-            session_id: "".to_string(), // Set on SSH client
+            system_id: None,       // Set on SSH client
+            installation_id: None, // Set on SSH client
+
+            // used on this end to associate panics with minidumps, but will be replaced on the SSH client
+            session_id: session_id.clone(),
         };
 
         if let Some(panic_data_json) = serde_json::to_string(&panic_data).log_err() {
@@ -194,44 +199,69 @@ fn init_panic_hook() {
     }));
 }
 
-fn handle_panic_requests(project: &Entity<HeadlessProject>, client: &Arc<ChannelClient>) {
+fn handle_crash_files_requests(project: &Entity<HeadlessProject>, client: &Arc<ChannelClient>) {
     let client: AnyProtoClient = client.clone().into();
     client.add_request_handler(
         project.downgrade(),
-        |_, _: TypedEnvelope<proto::GetPanicFiles>, _cx| async move {
+        |_, _: TypedEnvelope<proto::GetCrashFiles>, _cx| async move {
+            let mut crashes = Vec::new();
+            let mut minidumps_by_session_id = HashMap::new();
             let mut children = smol::fs::read_dir(paths::logs_dir()).await?;
-            let mut panic_files = Vec::new();
             while let Some(child) = children.next().await {
                 let child = child?;
                 let child_path = child.path();
 
-                if child_path.extension() != Some(OsStr::new("panic")) {
-                    continue;
+                let extension = child_path.extension();
+                if extension == Some(OsStr::new("panic")) {
+                    let filename = if let Some(filename) = child_path.file_name() {
+                        filename.to_string_lossy()
+                    } else {
+                        continue;
+                    };
+
+                    if !filename.starts_with("zed") {
+                        continue;
+                    }
+
+                    let file_contents = smol::fs::read_to_string(&child_path)
+                        .await
+                        .context("error reading panic file")?;
+
+                    crashes.push(proto::CrashReport {
+                        panic_contents: Some(file_contents),
+                        minidump_contents: None,
+                    });
+                } else if extension == Some(OsStr::new("dmp")) {
+                    let session_id = child_path.file_stem().unwrap().to_string_lossy();
+                    minidumps_by_session_id
+                        .insert(session_id.to_string(), smol::fs::read(&child_path).await?);
                 }
-                let filename = if let Some(filename) = child_path.file_name() {
-                    filename.to_string_lossy()
-                } else {
-                    continue;
-                };
-
-                if !filename.starts_with("zed") {
-                    continue;
-                }
-
-                let file_contents = smol::fs::read_to_string(&child_path)
-                    .await
-                    .context("error reading panic file")?;
-
-                panic_files.push(file_contents);
 
                 // We've done what we can, delete the file
-                std::fs::remove_file(child_path)
+                smol::fs::remove_file(&child_path)
+                    .await
                     .context("error removing panic")
                     .log_err();
             }
-            anyhow::Ok(proto::GetPanicFilesResponse {
-                file_contents: panic_files,
-            })
+
+            for crash in &mut crashes {
+                let panic: telemetry_events::Panic =
+                    serde_json::from_str(crash.panic_contents.as_ref().unwrap())?;
+                if let dump @ Some(_) = minidumps_by_session_id.remove(&panic.session_id) {
+                    crash.minidump_contents = dump;
+                }
+            }
+
+            crashes.extend(
+                minidumps_by_session_id
+                    .into_values()
+                    .map(|dmp| CrashReport {
+                        panic_contents: None,
+                        minidump_contents: Some(dmp),
+                    }),
+            );
+
+            anyhow::Ok(proto::GetCrashFilesResponse { crashes })
         },
     );
 }
@@ -409,7 +439,12 @@ pub fn execute_run(
         ControlFlow::Continue(_) => {}
     }
 
-    init_panic_hook();
+    let app = gpui::Application::headless();
+    let id = std::process::id().to_string();
+    app.background_executor()
+        .spawn(crashes::init(id.clone()))
+        .detach();
+    init_panic_hook(id);
     let log_rx = init_logging_server(log_file)?;
     log::info!(
         "starting up. pid_file: {:?}, stdin_socket: {:?}, stdout_socket: {:?}, stderr_socket: {:?}",
@@ -425,7 +460,7 @@ pub fn execute_run(
     let listeners = ServerListeners::new(stdin_socket, stdout_socket, stderr_socket)?;
 
     let git_hosting_provider_registry = Arc::new(GitHostingProviderRegistry::new());
-    gpui::Application::headless().run(move |cx| {
+    app.run(move |cx| {
         settings::init(cx);
         let app_version = AppVersion::load(env!("ZED_PKG_VERSION"));
         release_channel::init(app_version, cx);
@@ -486,7 +521,7 @@ pub fn execute_run(
             )
         });
 
-        handle_panic_requests(&project, &session);
+        handle_crash_files_requests(&project, &session);
 
         cx.background_spawn(async move { cleanup_old_binaries() })
             .detach();
@@ -530,12 +565,15 @@ impl ServerPaths {
 
 pub fn execute_proxy(identifier: String, is_reconnecting: bool) -> Result<()> {
     init_logging_proxy();
-    init_panic_hook();
-
-    log::info!("starting proxy process. PID: {}", std::process::id());
 
     let server_paths = ServerPaths::new(&identifier)?;
 
+    let id = std::process::id().to_string();
+    smol::spawn(crashes::init(id.clone())).detach();
+    init_panic_hook(id);
+
+    log::info!("starting proxy process. PID: {}", std::process::id());
+
     let server_pid = check_pid_file(&server_paths.pid_file)?;
     let server_running = server_pid.is_some();
     if is_reconnecting {

crates/reqwest_client/src/reqwest_client.rs

@@ -4,14 +4,13 @@ use std::{any::type_name, borrow::Cow, mem, pin::Pin, task::Poll, time::Duration
 
 use anyhow::anyhow;
 use bytes::{BufMut, Bytes, BytesMut};
-use futures::{AsyncRead, TryStreamExt as _};
+use futures::{AsyncRead, FutureExt as _, TryStreamExt as _};
 use http_client::{RedirectPolicy, Url, http};
 use regex::Regex;
 use reqwest::{
     header::{HeaderMap, HeaderValue},
     redirect,
 };
-use smol::future::FutureExt;
 
 const DEFAULT_CAPACITY: usize = 4096;
 static RUNTIME: OnceLock<tokio::runtime::Runtime> = OnceLock::new();
@@ -274,6 +273,26 @@ impl http_client::HttpClient for ReqwestClient {
         }
         .boxed()
     }
+
+    fn send_multipart_form<'a>(
+        &'a self,
+        url: &str,
+        form: reqwest::multipart::Form,
+    ) -> futures::future::BoxFuture<'a, anyhow::Result<http_client::Response<http_client::AsyncBody>>>
+    {
+        let response = self.client.post(url).multipart(form).send();
+        self.handle
+            .spawn(async move {
+                let response = response.await?;
+                let mut builder = http::response::Builder::new().status(response.status());
+                for (k, v) in response.headers() {
+                    builder = builder.header(k, v)
+                }
+                Ok(builder.body(response.bytes().await?.into())?)
+            })
+            .map(|e| e?)
+            .boxed()
+    }
 }
 
 #[cfg(test)]

crates/zed/Cargo.toml

@@ -45,6 +45,7 @@ collections.workspace = true
 command_palette.workspace = true
 component.workspace = true
 copilot.workspace = true
+crashes.workspace = true
 dap_adapters.workspace = true
 db.workspace = true
 debug_adapter_extension.workspace = true
@@ -117,6 +118,7 @@ recent_projects.workspace = true
 release_channel.workspace = true
 remote.workspace = true
 repl.workspace = true
+reqwest.workspace = true
 reqwest_client.workspace = true
 rope.workspace = true
 search.workspace = true

crates/zed/src/main.rs

@@ -172,6 +172,12 @@ pub fn main() {
 
     let args = Args::parse();
 
+    // `zed --crash-handler` Makes zed operate in minidump crash handler mode
+    if let Some(socket) = &args.crash_handler {
+        crashes::crash_server(socket.as_path());
+        return;
+    }
+
     // `zed --askpass` Makes zed operate in nc/netcat mode for use with askpass
     if let Some(socket) = &args.askpass {
         askpass::main(socket);
@@ -264,6 +270,9 @@ pub fn main() {
     let session_id = Uuid::new_v4().to_string();
     let session = app.background_executor().block(Session::new());
 
+    app.background_executor()
+        .spawn(crashes::init(session_id.clone()))
+        .detach();
     reliability::init_panic_hook(
         app_version,
         app_commit_sha.clone(),
@@ -1185,6 +1194,11 @@ struct Args {
     #[arg(long, hide = true)]
     nc: Option<String>,
 
+    /// Used for recording minidumps on crashes by having Zed run a separate
+    /// process communicating over a socket.
+    #[arg(long, hide = true)]
+    crash_handler: Option<PathBuf>,
+
     /// Run zed in the foreground, only used on Windows, to match the behavior on macOS.
     #[arg(long)]
     #[cfg(target_os = "windows")]

crates/zed/src/reliability.rs

@@ -2,21 +2,32 @@ use crate::stdout_is_a_pty;
 use anyhow::{Context as _, Result};
 use backtrace::{self, Backtrace};
 use chrono::Utc;
-use client::{TelemetrySettings, telemetry};
+use client::{
+    TelemetrySettings,
+    telemetry::{self, SENTRY_MINIDUMP_ENDPOINT},
+};
 use db::kvp::KEY_VALUE_STORE;
+use futures::AsyncReadExt;
 use gpui::{App, AppContext as _, SemanticVersion};
 use http_client::{self, HttpClient, HttpClientWithUrl, HttpRequestExt, Method};
 use paths::{crashes_dir, crashes_retired_dir};
 use project::Project;
 use release_channel::{AppCommitSha, RELEASE_CHANNEL, ReleaseChannel};
+use reqwest::multipart::{Form, Part};
 use settings::Settings;
 use smol::stream::StreamExt;
 use std::{
     env,
     ffi::{OsStr, c_void},
-    sync::{Arc, atomic::Ordering},
+    fs,
+    io::Write,
+    panic,
+    sync::{
+        Arc,
+        atomic::{AtomicU32, Ordering},
+    },
+    thread,
 };
-use std::{io::Write, panic, sync::atomic::AtomicU32, thread};
 use telemetry_events::{LocationData, Panic, PanicRequest};
 use url::Url;
 use util::ResultExt;
@@ -37,9 +48,10 @@ pub fn init_panic_hook(
         if prior_panic_count > 0 {
             // Give the panic-ing thread time to write the panic file
             loop {
-                std::thread::yield_now();
+                thread::yield_now();
             }
         }
+        crashes::handle_panic();
 
         let thread = thread::current();
         let thread_name = thread.name().unwrap_or("<unnamed>");
@@ -136,9 +148,8 @@ pub fn init_panic_hook(
             if let Some(panic_data_json) = serde_json::to_string(&panic_data).log_err() {
                 let timestamp = chrono::Utc::now().format("%Y_%m_%d %H_%M_%S").to_string();
                 let panic_file_path = paths::logs_dir().join(format!("zed-{timestamp}.panic"));
-                let panic_file = std::fs::OpenOptions::new()
-                    .append(true)
-                    .create(true)
+                let panic_file = fs::OpenOptions::new()
+                    .create_new(true)
                     .open(&panic_file_path)
                     .log_err();
                 if let Some(mut panic_file) = panic_file {
@@ -205,27 +216,31 @@ pub fn init(
         if let Some(ssh_client) = project.ssh_client() {
             ssh_client.update(cx, |client, cx| {
                 if TelemetrySettings::get_global(cx).diagnostics {
-                    let request = client.proto_client().request(proto::GetPanicFiles {});
+                    let request = client.proto_client().request(proto::GetCrashFiles {});
                     cx.background_spawn(async move {
-                        let panic_files = request.await?;
-                        for file in panic_files.file_contents {
-                            let panic: Option<Panic> = serde_json::from_str(&file)
-                                .log_err()
-                                .or_else(|| {
-                                    file.lines()
-                                        .next()
-                                        .and_then(|line| serde_json::from_str(line).ok())
-                                })
-                                .unwrap_or_else(|| {
-                                    log::error!("failed to deserialize panic file {:?}", file);
-                                    None
-                                });
+                        let crash_files = request.await?;
+                        for crash in crash_files.crashes {
+                            let mut panic: Option<Panic> = crash
+                                .panic_contents
+                                .and_then(|s| serde_json::from_str(&s).log_err());
 
-                            if let Some(mut panic) = panic {
+                            if let Some(panic) = panic.as_mut() {
                                 panic.session_id = session_id.clone();
                                 panic.system_id = system_id.clone();
                                 panic.installation_id = installation_id.clone();
+                            }
 
+                            if let Some(minidump) = crash.minidump_contents {
+                                upload_minidump(
+                                    http_client.clone(),
+                                    minidump.clone(),
+                                    panic.as_ref(),
+                                )
+                                .await
+                                .log_err();
+                            }
+
+                            if let Some(panic) = panic {
                                 upload_panic(&http_client, &panic_report_url, panic, &mut None)
                                     .await?;
                             }
@@ -510,6 +525,22 @@ async fn upload_previous_panics(
                 });
 
             if let Some(panic) = panic {
+                let minidump_path = paths::logs_dir()
+                    .join(&panic.session_id)
+                    .with_extension("dmp");
+                if minidump_path.exists() {
+                    let minidump = smol::fs::read(&minidump_path)
+                        .await
+                        .context("Failed to read minidump")?;
+                    if upload_minidump(http.clone(), minidump, Some(&panic))
+                        .await
+                        .log_err()
+                        .is_some()
+                    {
+                        fs::remove_file(minidump_path).ok();
+                    }
+                }
+
                 if !upload_panic(&http, &panic_report_url, panic, &mut most_recent_panic).await? {
                     continue;
                 }
@@ -517,13 +548,75 @@ async fn upload_previous_panics(
         }
 
         // We've done what we can, delete the file
-        std::fs::remove_file(child_path)
+        fs::remove_file(child_path)
             .context("error removing panic")
             .log_err();
     }
+
+    // loop back over the directory again to upload any minidumps that are missing panics
+    let mut children = smol::fs::read_dir(paths::logs_dir()).await?;
+    while let Some(child) = children.next().await {
+        let child = child?;
+        let child_path = child.path();
+        if child_path.extension() != Some(OsStr::new("dmp")) {
+            continue;
+        }
+        if upload_minidump(
+            http.clone(),
+            smol::fs::read(&child_path)
+                .await
+                .context("Failed to read minidump")?,
+            None,
+        )
+        .await
+        .log_err()
+        .is_some()
+        {
+            fs::remove_file(child_path).ok();
+        }
+    }
+
     Ok(most_recent_panic)
 }
 
+async fn upload_minidump(
+    http: Arc<HttpClientWithUrl>,
+    minidump: Vec<u8>,
+    panic: Option<&Panic>,
+) -> Result<()> {
+    let sentry_upload_url = SENTRY_MINIDUMP_ENDPOINT
+        .to_owned()
+        .ok_or_else(|| anyhow::anyhow!("Minidump endpoint not set"))?;
+
+    let mut form = Form::new()
+        .part(
+            "upload_file_minidump",
+            Part::bytes(minidump)
+                .file_name("minidump.dmp")
+                .mime_str("application/octet-stream")?,
+        )
+        .text("platform", "rust");
+    if let Some(panic) = panic {
+        form = form.text(
+            "release",
+            format!("{}-{}", panic.release_channel, panic.app_version),
+        );
+        // TODO: tack on more fields
+    }
+
+    let mut response_text = String::new();
+    let mut response = http.send_multipart_form(&sentry_upload_url, form).await?;
+    response
+        .body_mut()
+        .read_to_string(&mut response_text)
+        .await?;
+    if !response.status().is_success() {
+        anyhow::bail!("failed to upload minidump: {response_text}");
+    }
+    log::info!("Uploaded minidump. event id: {response_text}");
+    Ok(())
+}
+
 async fn upload_panic(
     http: &Arc<HttpClientWithUrl>,
     panic_report_url: &Url,