tests: Add an automatic perf profiler (#38543)

Add an auto-profiler for our tests, to hopefully allow better triage of
performance impacts resulting from code changes. Comprehensive usage
docs are in the code.

Currently, it uses hyperfine under the hood and prints markdown to the
command line for all crates with relevant tests enabled. We may want to
expand this to allow outputting json in the future to allow e.g.
automatically comparing the difference between two runs on different
commits, and in general a lot of functionality could be added (maybe
measuring memory usage?).

It's enabled (mostly as an example) on two tests inside `gpui` and a
bunch of those inside `vim`. I'd have happily used `cargo bench`, but that's nightly-only.

Release Notes:

- N/A

crates/gpui/Cargo.toml

@@ -110,6 +110,7 @@ resvg = { version = "0.45.0", default-features = false, features = [
     "memmap-fonts",
 ] }
 usvg = { version = "0.45.0", default-features = false }
+util_macros.workspace = true
 schemars.workspace = true
 seahash = "4.1"
 semantic_version.workspace = true

crates/gpui/src/style.rs

@@ -1300,7 +1300,9 @@ mod tests {
 
     use super::*;
 
-    #[test]
+    use util_macros::perf;
+
+    #[perf]
     fn test_basic_highlight_style_combination() {
         let style_a = HighlightStyle::default();
         let style_b = HighlightStyle::default();
@@ -1385,7 +1387,7 @@ mod tests {
         );
     }
 
-    #[test]
+    #[perf]
     fn test_combine_highlights() {
         assert_eq!(
             combine_highlights(

crates/util_macros/Cargo.toml

@@ -17,3 +17,6 @@ doctest = false
 quote.workspace = true
 syn.workspace = true
 workspace-hack.workspace = true
+
+[features]
+perf-enabled = []

crates/util_macros/src/util_macros.rs

@@ -1,8 +1,9 @@
 #![cfg_attr(not(target_os = "windows"), allow(unused))]
+#![allow(clippy::test_attr_in_doctest)]
 
 use proc_macro::TokenStream;
-use quote::quote;
-use syn::{LitStr, parse_macro_input};
+use quote::{ToTokens, quote};
+use syn::{ItemFn, LitStr, parse_macro_input, parse_quote};
 
 /// A macro used in tests for cross-platform path string literals in tests. On Windows it replaces
 /// `/` with `\\` and adds `C:` to the beginning of absolute paths. On other platforms, the path is
@@ -87,3 +88,145 @@ pub fn line_endings(input: TokenStream) -> TokenStream {
         #text
     })
 }
+
+/// Inner data for the perf macro.
+struct PerfArgs {
+    /// How many times to loop a test before rerunning the test binary.
+    /// If left empty, the test harness will auto-determine this value.
+    iterations: Option<syn::Expr>,
+}
+
+impl syn::parse::Parse for PerfArgs {
+    fn parse(input: syn::parse::ParseStream) -> syn::Result<Self> {
+        if input.is_empty() {
+            return Ok(PerfArgs { iterations: None });
+        }
+
+        let mut iterations = None;
+        // In principle we only have one possible argument, but leave this as
+        // a loop in case we expand this in the future.
+        for meta in
+            syn::punctuated::Punctuated::<syn::Meta, syn::Token![,]>::parse_terminated(input)?
+        {
+            match &meta {
+                syn::Meta::NameValue(meta_name_value) => {
+                    if meta_name_value.path.is_ident("iterations") {
+                        iterations = Some(meta_name_value.value.clone());
+                    } else {
+                        return Err(syn::Error::new_spanned(
+                            &meta_name_value.path,
+                            "unexpected argument, expected 'iterations'",
+                        ));
+                    }
+                }
+                _ => {
+                    return Err(syn::Error::new_spanned(
+                        meta,
+                        "expected name-value argument like 'iterations = 1'",
+                    ));
+                }
+            }
+        }
+
+        Ok(PerfArgs { iterations })
+    }
+}
+
+/// Marks a test as perf-sensitive, to be triaged when checking the performance
+/// of a build. This also automatically applies `#[test]`.
+///
+/// By default, the number of iterations when profiling this test is auto-determined.
+/// If this needs to be overwritten, pass the desired iteration count to the macro
+/// as a parameter (`#[perf(iterations = n)]`). Note that the actual profiler may still
+/// run the test an arbitrary number times; this flag just sets the number of executions
+/// before the process is restarted and global state is reset.
+///
+/// # Usage notes
+/// This should probably not be applied to tests that do any significant fs IO, as
+/// locks on files may not be released in time when repeating a test many times. This
+/// might lead to spurious failures.
+///
+/// # Examples
+/// ```rust
+/// use util_macros::perf;
+///
+/// #[perf]
+/// fn expensive_computation_test() {
+///     // Test goes here.
+/// }
+/// ```
+///
+/// This also works with `#[gpui::test]`s, though in most cases it shouldn't
+/// be used with automatic iterations.
+/// ```rust,ignore
+/// use util_macros::perf;
+///
+/// #[perf(iterations = 1)]
+/// #[gpui::test]
+/// fn oneshot_test(_cx: &mut gpui::TestAppContext) {
+///     // Test goes here.
+/// }
+/// ```
+#[proc_macro_attribute]
+pub fn perf(our_attr: TokenStream, input: TokenStream) -> TokenStream {
+    // If any of the below constants are changed, make sure to also update the perf
+    // profiler to match!
+
+    /// The suffix on tests marked with `#[perf]`.
+    const SUF_NORMAL: &str = "__ZED_PERF";
+    /// The suffix on tests marked with `#[perf(iterations = n)]`.
+    const SUF_FIXED: &str = "__ZED_PERF_FIXEDITER";
+    /// The env var in which we pass the iteration count to our tests.
+    const ITER_ENV_VAR: &str = "ZED_PERF_ITER";
+
+    let iter_count = parse_macro_input!(our_attr as PerfArgs).iterations;
+
+    let ItemFn {
+        mut attrs,
+        vis,
+        mut sig,
+        block,
+    } = parse_macro_input!(input as ItemFn);
+    attrs.push(parse_quote!(#[test]));
+    attrs.push(parse_quote!(#[allow(non_snake_case)]));
+
+    let block: Box<syn::Block> = if cfg!(perf_enabled) {
+        // Make the ident obvious when calling, for the test parser.
+        let mut new_ident = sig.ident.to_string();
+        if iter_count.is_some() {
+            new_ident.push_str(SUF_FIXED);
+        } else {
+            new_ident.push_str(SUF_NORMAL);
+        }
+
+        let new_ident = syn::Ident::new(&new_ident, sig.ident.span());
+        sig.ident = new_ident;
+        // If we have a preset iteration count, just use that.
+        if let Some(iter_count) = iter_count {
+            parse_quote!({
+               for _ in 0..#iter_count {
+                   #block
+               }
+            })
+        } else {
+            // Otherwise, the perf harness will pass us the value in an env var.
+            parse_quote!({
+                let iter_count = std::env::var(#ITER_ENV_VAR).unwrap().parse::<usize>().unwrap();
+                for _ in 0..iter_count {
+                    #block
+                }
+            })
+        }
+    } else {
+        block
+    };
+
+    ItemFn {
+        attrs,
+        vis,
+        sig,
+        block,
+    }
+    .into_token_stream()
+    .into()
+}

crates/vim/Cargo.toml

@@ -46,6 +46,7 @@ theme.workspace = true
 tokio = { version = "1.15", features = ["full"], optional = true }
 ui.workspace = true
 util.workspace = true
+util_macros.workspace = true
 vim_mode_setting.workspace = true
 workspace.workspace = true
 zed_actions.workspace = true

crates/vim/src/test.rs

@@ -25,6 +25,9 @@ use search::BufferSearchBar;
 
 use crate::{PushSneak, PushSneakBackward, insert::NormalBefore, motion, state::Mode};
 
+use util_macros::perf;
+
+#[perf]
 #[gpui::test]
 async fn test_initially_disabled(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, false).await;
@@ -44,6 +47,7 @@ async fn test_neovim(cx: &mut gpui::TestAppContext) {
     cx.assert_editor_state("ˇtest");
 }
 
+#[perf]
 #[gpui::test]
 async fn test_toggle_through_settings(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -80,6 +84,7 @@ async fn test_toggle_through_settings(cx: &mut gpui::TestAppContext) {
     assert_eq!(cx.mode(), Mode::Normal);
 }
 
+#[perf]
 #[gpui::test]
 async fn test_cancel_selection(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -104,6 +109,7 @@ async fn test_cancel_selection(cx: &mut gpui::TestAppContext) {
     cx.assert_editor_state("The quick brown fox juˇmps over the lazy dog");
 }
 
+#[perf]
 #[gpui::test]
 async fn test_buffer_search(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -132,6 +138,7 @@ async fn test_buffer_search(cx: &mut gpui::TestAppContext) {
     })
 }
 
+#[perf]
 #[gpui::test]
 async fn test_count_down(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -143,6 +150,7 @@ async fn test_count_down(cx: &mut gpui::TestAppContext) {
     cx.assert_editor_state("aa\nbb\ncc\ndd\neˇe");
 }
 
+#[perf]
 #[gpui::test]
 async fn test_end_of_document_710(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -157,6 +165,7 @@ async fn test_end_of_document_710(cx: &mut gpui::TestAppContext) {
     cx.assert_editor_state("aˇa\nbb\ncc");
 }
 
+#[perf]
 #[gpui::test]
 async fn test_end_of_line_with_times(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -175,6 +184,7 @@ async fn test_end_of_line_with_times(cx: &mut gpui::TestAppContext) {
     cx.assert_editor_state("aa\nbb\ncˇc");
 }
 
+#[perf]
 #[gpui::test]
 async fn test_indent_outdent(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -241,6 +251,7 @@ async fn test_escape_command_palette(cx: &mut gpui::TestAppContext) {
     cx.assert_state("aˇbc\n", Mode::Insert);
 }
 
+#[perf]
 #[gpui::test]
 async fn test_escape_cancels(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -251,6 +262,7 @@ async fn test_escape_cancels(cx: &mut gpui::TestAppContext) {
     cx.assert_state("aˇbc", Mode::Normal);
 }
 
+#[perf]
 #[gpui::test]
 async fn test_selection_on_search(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -289,6 +301,7 @@ async fn test_selection_on_search(cx: &mut gpui::TestAppContext) {
     cx.assert_state(indoc! {"aa\nbb\nˇcc\ncc\ncc\n"}, Mode::Normal);
 }
 
+#[perf]
 #[gpui::test]
 async fn test_word_characters(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new_typescript(cx).await;
@@ -315,6 +328,7 @@ async fn test_word_characters(cx: &mut gpui::TestAppContext) {
     )
 }
 
+#[perf]
 #[gpui::test]
 async fn test_kebab_case(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new_html(cx).await;
@@ -821,6 +835,7 @@ async fn test_paragraphs_dont_wrap(cx: &mut gpui::TestAppContext) {
         two"});
 }
 
+#[perf]
 #[gpui::test]
 async fn test_select_all_issue_2170(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -881,6 +896,7 @@ fn assert_pending_input(cx: &mut VimTestContext, expected: &str) {
     });
 }
 
+#[perf]
 #[gpui::test]
 async fn test_jk_multi(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -972,6 +988,7 @@ async fn test_comma_w(cx: &mut gpui::TestAppContext) {
         .assert_eq("hellˇo hello\nhello hello");
 }
 
+#[perf]
 #[gpui::test]
 async fn test_completion_menu_scroll_aside(cx: &mut TestAppContext) {
     let mut cx = VimTestContext::new_typescript(cx).await;
@@ -1053,6 +1070,7 @@ async fn test_completion_menu_scroll_aside(cx: &mut TestAppContext) {
     });
 }
 
+#[perf]
 #[gpui::test]
 async fn test_rename(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new_typescript(cx).await;
@@ -1088,6 +1106,7 @@ async fn test_rename(cx: &mut gpui::TestAppContext) {
     cx.assert_state("const afterˇ = 2; console.log(after)", Mode::Normal)
 }
 
+#[perf(iterations = 1)]
 #[gpui::test]
 async fn test_remap(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -1210,6 +1229,7 @@ async fn test_undo(cx: &mut gpui::TestAppContext) {
         3"});
 }
 
+#[perf]
 #[gpui::test]
 async fn test_mouse_selection(cx: &mut TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -1385,6 +1405,7 @@ async fn test_dw_eol(cx: &mut gpui::TestAppContext) {
         .assert_eq("twelve ˇtwelve char\ntwelve char");
 }
 
+#[perf]
 #[gpui::test]
 async fn test_toggle_comments(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -1476,6 +1497,7 @@ async fn test_find_multibyte(cx: &mut gpui::TestAppContext) {
         .assert_eq(r#"<label for="guests">ˇo</label>"#);
 }
 
+#[perf]
 #[gpui::test]
 async fn test_sneak(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -1695,6 +1717,7 @@ async fn test_ctrl_w_override(cx: &mut gpui::TestAppContext) {
     cx.shared_state().await.assert_eq("ˇ");
 }
 
+#[perf]
 #[gpui::test]
 async fn test_visual_indent_count(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;
@@ -1850,6 +1873,7 @@ async fn test_ctrl_o_dot(cx: &mut gpui::TestAppContext) {
     cx.shared_state().await.assert_eq("hellˇllo world.");
 }
 
+#[perf(iterations = 1)]
 #[gpui::test]
 async fn test_folded_multibuffer_excerpts(cx: &mut gpui::TestAppContext) {
     VimTestContext::init(cx);
@@ -2150,6 +2174,7 @@ async fn test_paragraph_multi_delete(cx: &mut gpui::TestAppContext) {
     cx.shared_state().await.assert_eq(indoc! {"ˇ"});
 }
 
+#[perf]
 #[gpui::test]
 async fn test_multi_cursor_replay(cx: &mut gpui::TestAppContext) {
     let mut cx = VimTestContext::new(cx, true).await;