Add more eval examples + filtering examples by language + fix git concurrent usage (#28719)

Release Notes:

- N/A

---------

Co-authored-by: michael <michael@zed.dev>
Co-authored-by: agus <agus@zed.dev>

crates/eval/src/eval.rs

@@ -39,6 +39,9 @@ struct Args {
     /// Model to use (default: "claude-3-7-sonnet-latest")
     #[arg(long, default_value = "claude-3-7-sonnet-latest")]
     model: String,
+    /// Languages to run (comma-separated, e.g. "js,ts,py"). If unspecified, only Rust examples are run.
+    #[arg(long, value_delimiter = ',')]
+    languages: Option<Vec<String>>,
 }
 
 fn main() {
@@ -46,6 +49,8 @@ fn main() {
 
     let args = Args::parse();
     let all_available_examples = list_all_examples().unwrap();
+    let languages = args.languages.unwrap_or_else(|| vec!["rs".to_string()]);
+
     let example_paths = all_available_examples
         .iter()
         .filter_map(|example_path| {
@@ -94,6 +99,17 @@ fn main() {
             let mut examples = Vec::new();
             for example_path in example_paths {
                 let example = Example::load_from_directory(&example_path, &run_dir)?;
+
+                if !example
+                    .base
+                    .language_extension
+                    .as_ref()
+                    .map_or(false, |lang| languages.contains(lang))
+                {
+                    println!("Skipping {}", example.name);
+                    continue;
+                }
+
                 examples.push((example_path, example));
             }
             let mut repo_urls = HashSet::new();
@@ -133,6 +149,10 @@ fn main() {
 
             future::join_all(clone_tasks).await;
 
+            for (_, example) in examples.iter() {
+                example.setup().await?;
+            }
+
             let tasks = examples
                 .into_iter()
                 .map(|(example_path, example)| {
@@ -197,7 +217,6 @@ async fn run_example(
     app_state: Arc<AgentAppState>,
     cx: &mut AsyncApp,
 ) -> Result<JudgeOutput> {
-    example.setup().await?;
     cx.update(|cx| example.run(model.clone(), app_state, cx))?
         .await?;
     let diff = example.repository_diff().await?;

crates/eval/src/example.rs

@@ -115,6 +115,8 @@ impl Example {
     pub async fn setup(&self) -> Result<()> {
         let repo_path = repo_path_for_url(&self.base.url);
 
+        println!("{}> Fetching", self.name);
+
         run_git(
             &repo_path,
             &["fetch", "--depth", "1", "origin", &self.base.revision],

crates/eval/src/judge_prompt.hbs

@@ -11,6 +11,7 @@ Use the following criteria to score the above changes.
 </criteria>
 
 Based on these criteria, give the test output a score between 0 and 5.
+The output score should ONLY INCLUDE whole numbers. DO NOT return decimals or floats.
 
 - 5 means: changes meet all criteria
 - 0 means: changes don't meet any criteria