zeta2: Improve error reporting and eval purity (#42470)

Closes #ISSUE

Improves error reporting for various failure modes of zeta2, including
failing to parse the `<old_text>`/`<new_text>` pattern, and the contents
of `<old_text>` failing to match.

Additionally, makes it so that evals are checked out into a worktree
with the _repo_ name instead of the _example_ name, in order to make
sure that the eval name has no influence on the models prediction. The
repo name worktrees are still namespaced by the example name like
`{example_name}/{repo_name}` to ensure evals pointing to the same repo
do not conflict.

Release Notes:

- N/A *or* Added/Fixed/Improved ...

---------

Co-authored-by: Agus <agus@zed.dev>
This commit is contained in:
Ben Kunkle
2025-11-12 12:52:11 -05:00
committed by GitHub
co-authored by Agus
parent c8930e07a3
commit 6c0069ca98
5 changed files with 89 additions and 29 deletions
+10 -6
View File
@@ -315,9 +315,6 @@ impl NamedExample {
let (repo_owner, repo_name) = self.repo_name()?;
let file_name = self.file_name();
fs::create_dir_all(&*REPOS_DIR)?;
fs::create_dir_all(&*WORKTREES_DIR)?;
let repo_dir = REPOS_DIR.join(repo_owner.as_ref()).join(repo_name.as_ref());
let repo_lock = lock_repo(&repo_dir).await;
@@ -332,7 +329,14 @@ impl NamedExample {
}
// Resolve the example to a revision, fetching it if needed.
let revision = run_git(&repo_dir, &["rev-parse", &self.example.revision]).await;
let revision = run_git(
&repo_dir,
&[
"rev-parse",
&format!("{}^{{commit}}", self.example.revision),
],
)
.await;
let revision = if let Ok(revision) = revision {
revision
} else {
@@ -349,7 +353,7 @@ impl NamedExample {
};
// Create the worktree for this example if needed.
let worktree_path = WORKTREES_DIR.join(&file_name);
let worktree_path = WORKTREES_DIR.join(&file_name).join(repo_name.as_ref());
if worktree_path.is_dir() {
run_git(&worktree_path, &["clean", "--force", "-d"]).await?;
run_git(&worktree_path, &["reset", "--hard", "HEAD"]).await?;
@@ -477,7 +481,7 @@ impl NamedExample {
let mut matches = text.match_indices(&cursor_excerpt);
let Some((excerpt_offset, _)) = matches.next() else {
anyhow::bail!(
"Cursor excerpt did not exist in buffer.\nExcerpt:\n\n{cursor_excerpt}\nBuffer text:\n{text}\n"
"\nExcerpt:\n\n{cursor_excerpt}\nBuffer text:\n{text}\n.Cursor excerpt did not exist in buffer."
);
};
assert!(matches.next().is_none());