use anyhow::{Context as _, Result}; use client::{Client, telemetry::MINIDUMP_ENDPOINT}; use futures::AsyncReadExt; use gpui::{App, AppContext as _, SerializedThreadTaskTimings}; use http_client::{self, HttpClient}; use log::info; use project::Project; use proto::{CrashReport, GetCrashFilesResponse}; use reqwest::multipart::{Form, Part}; use smol::stream::StreamExt; use std::{ffi::OsStr, fs, sync::Arc, thread::ThreadId, time::Duration}; use util::ResultExt; use crate::STARTUP_TIME; pub fn init(client: Arc, cx: &mut App) { monitor_hangs(cx); if client.telemetry().diagnostics_enabled() { let client = client.clone(); cx.background_spawn(async move { upload_previous_minidumps(client).await.warn_on_err(); }) .detach() } cx.observe_new(move |project: &mut Project, _, cx| { let client = client.clone(); let Some(remote_client) = project.remote_client() else { return; }; remote_client.update(cx, |remote_client, cx| { if !client.telemetry().diagnostics_enabled() { return; } let request = remote_client .proto_client() .request(proto::GetCrashFiles {}); cx.background_spawn(async move { let GetCrashFilesResponse { crashes } = request.await?; let Some(endpoint) = MINIDUMP_ENDPOINT.as_ref() else { return Ok(()); }; for CrashReport { metadata, minidump_contents, } in crashes { if let Some(metadata) = serde_json::from_str(&metadata).log_err() { upload_minidump(client.clone(), endpoint, minidump_contents, &metadata) .await .log_err(); } } anyhow::Ok(()) }) .detach_and_log_err(cx); }) }) .detach(); } fn monitor_hangs(cx: &App) { let main_thread_id = std::thread::current().id(); let foreground_executor = cx.foreground_executor(); let background_executor = cx.background_executor(); // 3 seconds hang let (mut tx, mut rx) = futures::channel::mpsc::channel(3); foreground_executor .spawn(async move { while (rx.next().await).is_some() {} }) .detach(); background_executor .spawn({ let background_executor = background_executor.clone(); async move { let mut hang_time = None; let mut hanging = false; loop { background_executor.timer(Duration::from_secs(1)).await; match tx.try_send(()) { Ok(_) => { hang_time = None; hanging = false; continue; } Err(e) => { let is_full = e.into_send_error().is_full(); if is_full && !hanging { hanging = true; hang_time = Some(chrono::Local::now()); } if is_full { save_hang_trace( main_thread_id, &background_executor, hang_time.unwrap(), ); } } } } } }) .detach(); } fn save_hang_trace( main_thread_id: ThreadId, background_executor: &gpui::BackgroundExecutor, hang_time: chrono::DateTime, ) { let thread_timings = background_executor.dispatcher.get_all_timings(); let thread_timings = thread_timings .into_iter() .map(|mut timings| { if timings.thread_id == main_thread_id { timings.thread_name = Some("main".to_string()); } SerializedThreadTaskTimings::convert(*STARTUP_TIME.get().unwrap(), timings) }) .collect::>(); let trace_path = paths::hang_traces_dir().join(&format!( "hang-{}.miniprof", hang_time.format("%Y-%m-%d_%H-%M-%S") )); let Some(timings) = serde_json::to_string(&thread_timings) .context("hang timings serialization") .log_err() else { return; }; std::fs::write(&trace_path, timings) .context("hang trace file writing") .log_err(); info!( "hang detected, trace file saved at: {}", trace_path.display() ); } pub async fn upload_previous_minidumps(client: Arc) -> anyhow::Result<()> { let Some(minidump_endpoint) = MINIDUMP_ENDPOINT.as_ref() else { log::warn!("Minidump endpoint not set"); return Ok(()); }; let mut children = smol::fs::read_dir(paths::logs_dir()).await?; while let Some(child) = children.next().await { let child = child?; let child_path = child.path(); if child_path.extension() != Some(OsStr::new("dmp")) { continue; } let mut json_path = child_path.clone(); json_path.set_extension("json"); if let Ok(metadata) = serde_json::from_slice(&smol::fs::read(&json_path).await?) && upload_minidump( client.clone(), minidump_endpoint, smol::fs::read(&child_path) .await .context("Failed to read minidump")?, &metadata, ) .await .log_err() .is_some() { fs::remove_file(child_path).ok(); fs::remove_file(json_path).ok(); } } Ok(()) } async fn upload_minidump( client: Arc, endpoint: &str, minidump: Vec, metadata: &crashes::CrashInfo, ) -> Result<()> { let mut form = Form::new() .part( "upload_file_minidump", Part::bytes(minidump) .file_name("minidump.dmp") .mime_str("application/octet-stream")?, ) .text( "sentry[tags][channel]", metadata.init.release_channel.clone(), ) .text("sentry[tags][version]", metadata.init.zed_version.clone()) .text("sentry[tags][binary]", metadata.init.binary.clone()) .text("sentry[release]", metadata.init.commit_sha.clone()) .text("platform", "rust"); let mut panic_message = "".to_owned(); if let Some(panic_info) = metadata.panic.as_ref() { panic_message = panic_info.message.clone(); form = form .text("sentry[logentry][formatted]", panic_info.message.clone()) .text("span", panic_info.span.clone()); } if let Some(minidump_error) = metadata.minidump_error.clone() { form = form.text("minidump_error", minidump_error); } if let Some(id) = client.telemetry().metrics_id() { form = form.text("sentry[user][id]", id.to_string()); form = form.text( "sentry[user][is_staff]", if client.telemetry().is_staff().unwrap_or_default() { "true" } else { "false" }, ); } else if let Some(id) = client.telemetry().installation_id() { form = form.text("sentry[user][id]", format!("installation-{}", id)) } ::telemetry::event!( "Minidump Uploaded", panic_message = panic_message, crashed_version = metadata.init.zed_version.clone(), commit_sha = metadata.init.commit_sha.clone(), ); let gpu_count = metadata.gpus.len(); for (index, gpu) in metadata.gpus.iter().cloned().enumerate() { let system_specs::GpuInfo { device_name, device_pci_id, vendor_name, vendor_pci_id, driver_version, driver_name, } = gpu; let num = if gpu_count == 1 && metadata.active_gpu.is_none() { String::new() } else { index.to_string() }; let name = format!("gpu{num}"); let root = format!("sentry[contexts][{name}]"); form = form .text( format!("{root}[Description]"), "A GPU found on the users system. May or may not be the GPU Zed is running on", ) .text(format!("{root}[type]"), "gpu") .text(format!("{root}[name]"), device_name.unwrap_or(name)) .text(format!("{root}[id]"), format!("{:#06x}", device_pci_id)) .text( format!("{root}[vendor_id]"), format!("{:#06x}", vendor_pci_id), ) .text_if_some(format!("{root}[vendor_name]"), vendor_name) .text_if_some(format!("{root}[driver_version]"), driver_version) .text_if_some(format!("{root}[driver_name]"), driver_name); } if let Some(active_gpu) = metadata.active_gpu.clone() { form = form .text( "sentry[contexts][Active_GPU][Description]", "The GPU Zed is running on", ) .text("sentry[contexts][Active_GPU][type]", "gpu") .text("sentry[contexts][Active_GPU][name]", active_gpu.device_name) .text( "sentry[contexts][Active_GPU][driver_version]", active_gpu.driver_info, ) .text( "sentry[contexts][Active_GPU][driver_name]", active_gpu.driver_name, ) .text( "sentry[contexts][Active_GPU][is_software_emulated]", active_gpu.is_software_emulated.to_string(), ); } // TODO: feature-flag-context, and more of device-context like screen resolution, available ram, device model, etc let mut response_text = String::new(); let mut response = client .http_client() .send_multipart_form(endpoint, form) .await?; response .body_mut() .read_to_string(&mut response_text) .await?; if !response.status().is_success() { anyhow::bail!("failed to upload minidump: {response_text}"); } log::info!("Uploaded minidump. event id: {response_text}"); Ok(()) } trait FormExt { fn text_if_some( self, label: impl Into>, value: Option>>, ) -> Self; } impl FormExt for Form { fn text_if_some( self, label: impl Into>, value: Option>>, ) -> Self { match value { Some(value) => self.text(label.into(), value.into()), None => self, } } }