use anyhow::{Context as _, Result}; use client::{TelemetrySettings, telemetry::MINIDUMP_ENDPOINT}; use futures::AsyncReadExt; use gpui::{App, AppContext as _}; use http_client::{self, HttpClient, HttpClientWithUrl}; use project::Project; use proto::{CrashReport, GetCrashFilesResponse}; use reqwest::multipart::{Form, Part}; use settings::Settings; use smol::stream::StreamExt; use std::{ffi::OsStr, fs, sync::Arc}; use util::ResultExt; pub fn init(http_client: Arc, installation_id: Option, cx: &mut App) { #[cfg(target_os = "macos")] monitor_main_thread_hangs(http_client.clone(), installation_id.clone(), cx); if client::TelemetrySettings::get_global(cx).diagnostics { let client = http_client.clone(); let id = installation_id.clone(); cx.background_spawn(async move { upload_previous_minidumps(client, id).await.warn_on_err(); }) .detach() } cx.observe_new(move |project: &mut Project, _, cx| { let http_client = http_client.clone(); let installation_id = installation_id.clone(); let Some(remote_client) = project.remote_client() else { return; }; remote_client.update(cx, |client, cx| { if !TelemetrySettings::get_global(cx).diagnostics { return; } let request = client.proto_client().request(proto::GetCrashFiles {}); cx.background_spawn(async move { let GetCrashFilesResponse { crashes } = request.await?; let Some(endpoint) = MINIDUMP_ENDPOINT.as_ref() else { return Ok(()); }; for CrashReport { metadata, minidump_contents, } in crashes { if let Some(metadata) = serde_json::from_str(&metadata).log_err() { upload_minidump( http_client.clone(), endpoint, minidump_contents, &metadata, installation_id.clone(), ) .await .log_err(); } } anyhow::Ok(()) }) .detach_and_log_err(cx); }) }) .detach(); } #[cfg(target_os = "macos")] pub fn monitor_main_thread_hangs( http_client: Arc, installation_id: Option, cx: &App, ) { // This is too noisy to ship to stable for now. if !matches!( ReleaseChannel::global(cx), ReleaseChannel::Dev | ReleaseChannel::Nightly | ReleaseChannel::Preview ) { return; } use nix::sys::signal::{ SaFlags, SigAction, SigHandler, SigSet, Signal::{self, SIGUSR2}, sigaction, }; use parking_lot::Mutex; use http_client::Method; use release_channel::ReleaseChannel; use std::{ ffi::c_int, sync::{OnceLock, mpsc}, time::Duration, }; use telemetry_events::{BacktraceFrame, HangReport}; use nix::sys::pthread; let foreground_executor = cx.foreground_executor(); let background_executor = cx.background_executor(); let telemetry_settings = *client::TelemetrySettings::get_global(cx); // Initialize SIGUSR2 handler to send a backtrace to a channel. let (backtrace_tx, backtrace_rx) = mpsc::channel(); static BACKTRACE: Mutex> = Mutex::new(Vec::new()); static BACKTRACE_SENDER: OnceLock> = OnceLock::new(); BACKTRACE_SENDER.get_or_init(|| backtrace_tx); BACKTRACE.lock().reserve(100); fn handle_backtrace_signal() { unsafe { extern "C" fn handle_sigusr2(_i: c_int) { unsafe { // ASYNC SIGNAL SAFETY: This lock is only accessed one other time, // which can only be triggered by This signal handler. In addition, // this signal handler is immediately removed by SA_RESETHAND, and this // signal handler cannot be re-entrant due to the SIGUSR2 mask defined // below let mut bt = BACKTRACE.lock(); bt.clear(); backtrace::trace_unsynchronized(|frame| { if bt.len() < bt.capacity() { bt.push(frame.clone()); true } else { false } }); } BACKTRACE_SENDER.get().unwrap().send(()).ok(); } let mut mask = SigSet::empty(); mask.add(SIGUSR2); sigaction( Signal::SIGUSR2, &SigAction::new( SigHandler::Handler(handle_sigusr2), SaFlags::SA_RESTART | SaFlags::SA_RESETHAND, mask, ), ) .log_err(); } } handle_backtrace_signal(); let main_thread = pthread::pthread_self(); let (mut tx, mut rx) = futures::channel::mpsc::channel(3); foreground_executor .spawn(async move { while (rx.next().await).is_some() {} }) .detach(); background_executor .spawn({ let background_executor = background_executor.clone(); async move { loop { background_executor.timer(Duration::from_secs(1)).await; match tx.try_send(()) { Ok(_) => continue, Err(e) => { if e.into_send_error().is_full() { pthread::pthread_kill(main_thread, SIGUSR2).log_err(); } // Only detect the first hang break; } } } } }) .detach(); let app_version = release_channel::AppVersion::global(cx); let os_name = client::telemetry::os_name(); background_executor .clone() .spawn(async move { let os_version = client::telemetry::os_version(); loop { while backtrace_rx.recv().is_ok() { if !telemetry_settings.diagnostics { return; } // ASYNC SIGNAL SAFETY: This lock is only accessed _after_ // the backtrace transmitter has fired, which itself is only done // by the signal handler. And due to SA_RESETHAND the signal handler // will not run again until `handle_backtrace_signal` is called. let raw_backtrace = BACKTRACE.lock().drain(..).collect::>(); let backtrace: Vec<_> = raw_backtrace .into_iter() .map(|frame| { let mut btf = BacktraceFrame { ip: frame.ip() as usize, symbol_addr: frame.symbol_address() as usize, base: frame.module_base_address().map(|addr| addr as usize), symbols: vec![], }; backtrace::resolve_frame(&frame, |symbol| { if let Some(name) = symbol.name() { btf.symbols.push(name.to_string()); } }); btf }) .collect(); // IMPORTANT: Don't move this to before `BACKTRACE.lock()` handle_backtrace_signal(); log::error!( "Suspected hang on main thread:\n{}", backtrace .iter() .flat_map(|bt| bt.symbols.first().as_ref().map(|s| s.as_str())) .collect::>() .join("\n") ); let report = HangReport { backtrace, app_version: Some(app_version), os_name: os_name.clone(), os_version: Some(os_version.clone()), architecture: std::env::consts::ARCH.into(), installation_id: installation_id.clone(), }; let Some(json_bytes) = serde_json::to_vec(&report).log_err() else { continue; }; let Some(checksum) = client::telemetry::calculate_json_checksum(&json_bytes) else { continue; }; let Ok(url) = http_client.build_zed_api_url("/telemetry/hangs", &[]) else { continue; }; let Ok(request) = http_client::Request::builder() .method(Method::POST) .uri(url.as_ref()) .header("x-zed-checksum", checksum) .body(json_bytes.into()) else { continue; }; if let Some(response) = http_client.send(request).await.log_err() && response.status() != 200 { log::error!("Failed to send hang report: HTTP {:?}", response.status()); } } } }) .detach() } pub async fn upload_previous_minidumps( http: Arc, installation_id: Option, ) -> anyhow::Result<()> { let Some(minidump_endpoint) = MINIDUMP_ENDPOINT.as_ref() else { log::warn!("Minidump endpoint not set"); return Ok(()); }; let mut children = smol::fs::read_dir(paths::logs_dir()).await?; while let Some(child) = children.next().await { let child = child?; let child_path = child.path(); if child_path.extension() != Some(OsStr::new("dmp")) { continue; } let mut json_path = child_path.clone(); json_path.set_extension("json"); if let Ok(metadata) = serde_json::from_slice(&smol::fs::read(&json_path).await?) && upload_minidump( http.clone(), minidump_endpoint, smol::fs::read(&child_path) .await .context("Failed to read minidump")?, &metadata, installation_id.clone(), ) .await .log_err() .is_some() { fs::remove_file(child_path).ok(); fs::remove_file(json_path).ok(); } } Ok(()) } async fn upload_minidump( http: Arc, endpoint: &str, minidump: Vec, metadata: &crashes::CrashInfo, installation_id: Option, ) -> Result<()> { let mut form = Form::new() .part( "upload_file_minidump", Part::bytes(minidump) .file_name("minidump.dmp") .mime_str("application/octet-stream")?, ) .text( "sentry[tags][channel]", metadata.init.release_channel.clone(), ) .text("sentry[tags][version]", metadata.init.zed_version.clone()) .text("sentry[release]", metadata.init.commit_sha.clone()) .text("platform", "rust"); let mut panic_message = "".to_owned(); if let Some(panic_info) = metadata.panic.as_ref() { panic_message = panic_info.message.clone(); form = form .text("sentry[logentry][formatted]", panic_info.message.clone()) .text("span", panic_info.span.clone()); } if let Some(minidump_error) = metadata.minidump_error.clone() { form = form.text("minidump_error", minidump_error); } if let Some(id) = installation_id.clone() { form = form.text("sentry[user][id]", id) } ::telemetry::event!( "Minidump Uploaded", panic_message = panic_message, crashed_version = metadata.init.zed_version.clone(), commit_sha = metadata.init.commit_sha.clone(), ); let gpu_count = metadata.gpus.len(); for (index, gpu) in metadata.gpus.iter().cloned().enumerate() { let system_specs::GpuInfo { device_name, device_pci_id, vendor_name, vendor_pci_id, driver_version, driver_name, } = gpu; let num = if gpu_count == 1 && metadata.active_gpu.is_none() { String::new() } else { index.to_string() }; let name = format!("gpu{num}"); let root = format!("sentry[contexts][{name}]"); form = form .text( format!("{root}[Description]"), "A GPU found on the users system. May or may not be the GPU Zed is running on", ) .text(format!("{root}[type]"), "gpu") .text(format!("{root}[name]"), device_name.unwrap_or(name)) .text(format!("{root}[id]"), format!("{:#06x}", device_pci_id)) .text( format!("{root}[vendor_id]"), format!("{:#06x}", vendor_pci_id), ) .text_if_some(format!("{root}[vendor_name]"), vendor_name) .text_if_some(format!("{root}[driver_version]"), driver_version) .text_if_some(format!("{root}[driver_name]"), driver_name); } if let Some(active_gpu) = metadata.active_gpu.clone() { form = form .text( "sentry[contexts][Active_GPU][Description]", "The GPU Zed is running on", ) .text("sentry[contexts][Active_GPU][type]", "gpu") .text("sentry[contexts][Active_GPU][name]", active_gpu.device_name) .text( "sentry[contexts][Active_GPU][driver_version]", active_gpu.driver_info, ) .text( "sentry[contexts][Active_GPU][driver_name]", active_gpu.driver_name, ) .text( "sentry[contexts][Active_GPU][is_software_emulated]", active_gpu.is_software_emulated.to_string(), ); } // TODO: feature-flag-context, and more of device-context like screen resolution, available ram, device model, etc let mut response_text = String::new(); let mut response = http.send_multipart_form(endpoint, form).await?; response .body_mut() .read_to_string(&mut response_text) .await?; if !response.status().is_success() { anyhow::bail!("failed to upload minidump: {response_text}"); } log::info!("Uploaded minidump. event id: {response_text}"); Ok(()) } trait FormExt { fn text_if_some( self, label: impl Into>, value: Option>>, ) -> Self; } impl FormExt for Form { fn text_if_some( self, label: impl Into>, value: Option>>, ) -> Self { match value { Some(value) => self.text(label.into(), value.into()), None => self, } } }