eval: Fine-grained assertions (#29246)
- Support programmatic examples ([example](https://github.com/zed-industries/zed/blob/17feb260a0919e102ae4e220669c467885bf4b57/crates/eval/src/examples/file_search.rs)) - Combine data-driven example declarations into a single `.toml` file ([example](https://github.com/zed-industries/zed/blob/17feb260a0919e102ae4e220669c467885bf4b57/crates/eval/src/examples/find_and_replace_diff_card.toml)) - Run judge on individual assertions (previously called "criteria") - Report judge and programmatic assertions in one combined table Note: We still need to work on concept naming <img width=400 src="https://github.com/user-attachments/assets/fc719c93-467f-412b-8d47-68821bd8a5f5"> Release Notes: - N/A --------- Co-authored-by: Richard Feldman <oss@rtfeldman.com> Co-authored-by: Max Brunsfeld <maxbrunsfeld@gmail.com> Co-authored-by: Thomas Mickley-Doyle <tmickleydoyle@gmail.com>
This commit is contained in:
co-authored by
Richard Feldman
Max Brunsfeld
Thomas Mickley-Doyle
parent
0d3fe474db
commit
ce1a674eba
@@ -0,0 +1,53 @@
|
||||
use anyhow::Result;
|
||||
use assistant_tools::PathSearchToolInput;
|
||||
use async_trait::async_trait;
|
||||
use regex::Regex;
|
||||
|
||||
use crate::example::{Example, ExampleContext, ExampleMetadata};
|
||||
|
||||
pub struct FileSearchExample;
|
||||
|
||||
#[async_trait(?Send)]
|
||||
impl Example for FileSearchExample {
|
||||
fn meta(&self) -> ExampleMetadata {
|
||||
ExampleMetadata {
|
||||
name: "file_search".to_string(),
|
||||
url: "https://github.com/zed-industries/zed.git".to_string(),
|
||||
revision: "03ecb88fe30794873f191ddb728f597935b3101c".to_string(),
|
||||
language_server: None,
|
||||
max_assertions: Some(4),
|
||||
}
|
||||
}
|
||||
|
||||
async fn conversation(&self, cx: &mut ExampleContext) -> Result<()> {
|
||||
const FILENAME: &str = "find_replace_file_tool.rs";
|
||||
cx.push_user_message(format!(
|
||||
r#"
|
||||
Look at the `{FILENAME}`. I want to implement a card for it. The card should implement the `Render` trait.
|
||||
|
||||
The card should show a diff. It should be a beautifully presented diff. The card "box" should look like what we show for
|
||||
markdown codeblocks (look at `MarkdownElement`). I want to see a red background for lines that were deleted and a green
|
||||
background for lines that were added. We should have a div per diff line.
|
||||
"#
|
||||
));
|
||||
|
||||
let response = cx.run_turn().await?;
|
||||
let tool_use = response.expect_tool("path_search", cx)?;
|
||||
let input = tool_use.expect_input::<PathSearchToolInput>(cx)?;
|
||||
|
||||
let glob = input.glob;
|
||||
cx.assert(
|
||||
glob.ends_with(FILENAME),
|
||||
format!("glob ends with `{FILENAME}`"),
|
||||
)?;
|
||||
|
||||
let without_filename = glob.replace(FILENAME, "");
|
||||
let matches = Regex::new("(\\*\\*|zed)/(\\*\\*?/)?")
|
||||
.unwrap()
|
||||
.is_match(&without_filename);
|
||||
|
||||
cx.assert(matches, "glob starts with either `**` or `zed`")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user