Evaluate zeta2 context retrieval and edit predictions (#41921)
This PR implements the `zeta-cli eval` command. It will: - Run the edit prediction model if there are no cached results - Compute precision/recall/F1 for context retrieval at the line level: every retrieved line of context is counted as a true positive (correct retrieval), false positive (retrieved something that was not expected), or false negative (didn't retrieve an expected line) - Compute similar metrics for edit predictions - Pretty-print results, highlighting the difference between actual and expected when printing to tty Other changes: - `zeta-cli predict` accepts a `--format` argument with options `md`, `json`, `diff` - Code restructure Release Notes: - N/A --------- Co-authored-by: Piotr Osiewicz <24362066+osiewicz@users.noreply.github.com> Co-authored-by: Agus Zubiaga <agus@zed.dev>
This commit is contained in:
co-authored by
Piotr Osiewicz
Agus Zubiaga
parent
054d2e1524
commit
91d631c229
@@ -1,4 +1,4 @@
|
||||
use std::borrow::Cow;
|
||||
use std::{borrow::Cow, fmt::Display};
|
||||
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub enum DiffLine<'a> {
|
||||
@@ -8,7 +8,7 @@ pub enum DiffLine<'a> {
|
||||
Context(&'a str),
|
||||
Deletion(&'a str),
|
||||
Addition(&'a str),
|
||||
Garbage,
|
||||
Garbage(&'a str),
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq)]
|
||||
@@ -21,7 +21,7 @@ pub struct HunkLocation {
|
||||
|
||||
impl<'a> DiffLine<'a> {
|
||||
pub fn parse(line: &'a str) -> Self {
|
||||
Self::try_parse(line).unwrap_or(Self::Garbage)
|
||||
Self::try_parse(line).unwrap_or(Self::Garbage(line))
|
||||
}
|
||||
|
||||
fn try_parse(line: &'a str) -> Option<Self> {
|
||||
@@ -60,6 +60,30 @@ impl<'a> DiffLine<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Display for DiffLine<'a> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
DiffLine::OldPath { path } => write!(f, "--- {path}"),
|
||||
DiffLine::NewPath { path } => write!(f, "+++ {path}"),
|
||||
DiffLine::HunkHeader(Some(hunk_location)) => {
|
||||
write!(
|
||||
f,
|
||||
"@@ -{},{} +{},{} @@",
|
||||
hunk_location.start_line_old + 1,
|
||||
hunk_location.count_old,
|
||||
hunk_location.start_line_new + 1,
|
||||
hunk_location.count_new
|
||||
)
|
||||
}
|
||||
DiffLine::HunkHeader(None) => write!(f, "@@ ... @@"),
|
||||
DiffLine::Context(content) => write!(f, " {content}"),
|
||||
DiffLine::Deletion(content) => write!(f, "-{content}"),
|
||||
DiffLine::Addition(content) => write!(f, "+{content}"),
|
||||
DiffLine::Garbage(line) => write!(f, "{line}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_header_path<'a>(strip_prefix: &'static str, header: &'a str) -> Cow<'a, str> {
|
||||
if !header.contains(['"', '\\']) {
|
||||
let path = header.split_ascii_whitespace().next().unwrap_or(header);
|
||||
@@ -134,8 +158,8 @@ mod tests {
|
||||
pretty_assertions::assert_eq!(
|
||||
lines,
|
||||
&[
|
||||
DiffLine::Garbage,
|
||||
DiffLine::Garbage,
|
||||
DiffLine::Garbage("diff --git a/text.txt b/text.txt"),
|
||||
DiffLine::Garbage("index 86c770d..a1fd855 100644"),
|
||||
DiffLine::OldPath {
|
||||
path: "file.txt".into()
|
||||
},
|
||||
@@ -151,7 +175,7 @@ mod tests {
|
||||
DiffLine::Context("context"),
|
||||
DiffLine::Deletion("deleted"),
|
||||
DiffLine::Addition("inserted"),
|
||||
DiffLine::Garbage,
|
||||
DiffLine::Garbage("garbage"),
|
||||
DiffLine::Context(""),
|
||||
DiffLine::OldPath {
|
||||
path: "b/file.txt".into()
|
||||
|
||||
Reference in New Issue
Block a user