Oleksiy SyvokonandBen Brandt
3884de937b
This problem seems to be specific to Opus 4. Eval shows improvement from 89% to 97%. Closes: https://github.com/zed-industries/zed/issues/32060 Release Notes: - N/A Co-authored-by: Ben Brandt <benjamin.j.brandt@gmail.com>
60 lines
2.1 KiB
Rust
60 lines
2.1 KiB
Rust
use agent_settings::AgentProfileId;
|
|
use anyhow::Result;
|
|
use assistant_tools::GrepToolInput;
|
|
use async_trait::async_trait;
|
|
|
|
use crate::example::{Example, ExampleContext, ExampleMetadata};
|
|
|
|
pub struct GrepParamsEscapementExample;
|
|
|
|
/*
|
|
|
|
This eval checks that the model doesn't use HTML escapement for characters like `<` and
|
|
`>` in tool parameters.
|
|
|
|
original +system_prompt change +tool description
|
|
claude-opus-4 89% 92% 97%+
|
|
claude-sonnet-4 100%
|
|
gpt-4.1-mini 100%
|
|
gemini-2.5-pro 98%
|
|
|
|
*/
|
|
|
|
#[async_trait(?Send)]
|
|
impl Example for GrepParamsEscapementExample {
|
|
fn meta(&self) -> ExampleMetadata {
|
|
ExampleMetadata {
|
|
name: "grep_params_escapement".to_string(),
|
|
url: "https://github.com/octocat/hello-world".to_string(),
|
|
revision: "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d".to_string(),
|
|
language_server: None,
|
|
max_assertions: Some(1),
|
|
profile_id: AgentProfileId::default(),
|
|
existing_thread_json: None,
|
|
max_turns: Some(2),
|
|
}
|
|
}
|
|
|
|
async fn conversation(&self, cx: &mut ExampleContext) -> Result<()> {
|
|
// cx.push_user_message("How does the precedence/specificity work with Keymap contexts? I am seeing that `MessageEditor > Editor` is lower precendence than `Editor` which is surprising to me, but might be how it works");
|
|
cx.push_user_message("Search for files containing the characters `>` or `<`");
|
|
let response = cx.run_turns(2).await?;
|
|
let grep_input = response
|
|
.find_tool_call("grep")
|
|
.and_then(|tool_use| tool_use.parse_input::<GrepToolInput>().ok());
|
|
|
|
cx.assert_some(grep_input.as_ref(), "`grep` tool should be called")?;
|
|
|
|
cx.assert(
|
|
!contains_html_entities(&grep_input.unwrap().regex),
|
|
"Tool parameters should not be escaped",
|
|
)
|
|
}
|
|
}
|
|
|
|
fn contains_html_entities(pattern: &str) -> bool {
|
|
regex::Regex::new(r"&[a-zA-Z]+;|&#[0-9]+;|&#x[0-9a-fA-F]+;")
|
|
.unwrap()
|
|
.is_match(pattern)
|
|
}
|