zeta eval: --repeat flag (#42569)

Adds a `--repeat` flag to the zeta eval that runs each example as many
times as specified. Also makes the output nicer in a few ways.

Release Notes:

- N/A

---------

Co-authored-by: Ben Kunkle <ben@zed.dev>
Co-authored-by: Michael <michael@zed.dev>
This commit is contained in:
Agus Zubiaga
2025-11-12 16:58:22 -05:00
committed by GitHub
co-authored by Ben Kunkle Michael
parent f2a1eb9963
commit b0700a4625
7 changed files with 342 additions and 145 deletions
+81 -3
View File
@@ -6,9 +6,10 @@ use std::{
io::Write,
mem,
path::{Path, PathBuf},
sync::Arc,
sync::{Arc, OnceLock},
};
use crate::headless::ZetaCliAppState;
use anyhow::{Context as _, Result, anyhow};
use clap::ValueEnum;
use cloud_zeta2_prompt::CURSOR_MARKER;
@@ -18,13 +19,14 @@ use futures::{
AsyncWriteExt as _,
lock::{Mutex, OwnedMutexGuard},
};
use gpui::{AsyncApp, Entity, http_client::Url};
use futures::{FutureExt as _, future::Shared};
use gpui::{AppContext as _, AsyncApp, Entity, Task, http_client::Url};
use language::{Anchor, Buffer};
use project::{Project, ProjectPath};
use pulldown_cmark::CowStr;
use serde::{Deserialize, Serialize};
use util::{paths::PathStyle, rel_path::RelPath};
use zeta2::udiff::OpenedBuffers;
use zeta2::{Zeta, udiff::OpenedBuffers};
use crate::paths::{REPOS_DIR, WORKTREES_DIR};
@@ -311,6 +313,82 @@ impl NamedExample {
}
}
pub async fn setup_project<'a>(
&'a self,
app_state: &Arc<ZetaCliAppState>,
repetitions: u16,
cx: &mut AsyncApp,
) -> Result<(Entity<Project>, Vec<Entity<Zeta>>, OpenedBuffers<'a>)> {
let worktree_path = self.setup_worktree().await?;
static AUTHENTICATED: OnceLock<Shared<Task<()>>> = OnceLock::new();
AUTHENTICATED
.get_or_init(|| {
let client = app_state.client.clone();
cx.spawn(async move |cx| {
client
.sign_in_with_optional_connect(true, cx)
.await
.unwrap();
})
.shared()
})
.clone()
.await;
let project = cx.update(|cx| {
Project::local(
app_state.client.clone(),
app_state.node_runtime.clone(),
app_state.user_store.clone(),
app_state.languages.clone(),
app_state.fs.clone(),
None,
cx,
)
})?;
let worktree = project
.update(cx, |project, cx| {
project.create_worktree(&worktree_path, true, cx)
})?
.await?;
worktree
.read_with(cx, |worktree, _cx| {
worktree.as_local().unwrap().scan_complete()
})?
.await;
let buffer_store = project.read_with(cx, |project, _| project.buffer_store().clone())?;
let zetas = (0..repetitions)
.map(|_| {
let zeta = cx.new(|cx| {
zeta2::Zeta::new(app_state.client.clone(), app_state.user_store.clone(), cx)
})?;
cx.subscribe(&buffer_store, {
let project = project.clone();
let zeta = zeta.clone();
move |_, event, cx| match event {
project::buffer_store::BufferStoreEvent::BufferAdded(buffer) => {
zeta.update(cx, |zeta, cx| zeta.register_buffer(&buffer, &project, cx));
}
_ => {}
}
})?
.detach();
anyhow::Ok(zeta)
})
.collect::<Result<Vec<_>>>()?;
let edited_buffers = self.apply_edit_history(&project, cx).await?;
anyhow::Ok((project, zetas, edited_buffers))
}
pub async fn setup_worktree(&self) -> Result<PathBuf> {
let (repo_owner, repo_name) = self.repo_name()?;
let file_name = self.file_name();