agent: Overwrite files more cautiously (#30649)

1. The `edit_file` tool tended to use `create_or_overwrite` a bit too
often, leading to corruption of long files. This change replaces the
boolean flag with an `EditFileMode` enum, which helps Agent make a more
deliberate choice when overwriting files.

With this change, the pass rate of the new eval increased from 10% to
100%.

2. eval: Added ability to run eval on top of an existing thread. Threads
can now be loaded from JSON files in the `SerializedThread` format,
which makes it easy to use real threads as starting points for
tests/evals.

3. Don't try to restore tool cards when running in headless or eval mode
-- we don't have a window to properly do this.

Release Notes:

- N/A
This commit is contained in:
Oleksiy Syvokon
2025-05-14 10:40:44 +03:00
committed by GitHub
parent 22f76ac1a7
commit 255d8f7cf8
18 changed files with 425 additions and 37 deletions
+14
View File
@@ -16,6 +16,7 @@ mod add_arg_to_trait_method;
mod code_block_citations;
mod comment_translation;
mod file_search;
mod overwrite_file;
mod planets;
pub fn all(examples_dir: &Path) -> Vec<Rc<dyn Example>> {
@@ -25,6 +26,7 @@ pub fn all(examples_dir: &Path) -> Vec<Rc<dyn Example>> {
Rc::new(code_block_citations::CodeBlockCitations),
Rc::new(planets::Planets),
Rc::new(comment_translation::CommentTranslation),
Rc::new(overwrite_file::FileOverwriteExample),
];
for example_path in list_declarative_examples(examples_dir).unwrap() {
@@ -45,6 +47,7 @@ impl DeclarativeExample {
pub fn load(example_path: &Path) -> Result<Self> {
let name = Self::name_from_path(example_path);
let base: ExampleToml = toml::from_str(&fs::read_to_string(&example_path)?)?;
let example_dir = example_path.parent().unwrap();
let language_server = if base.require_lsp {
Some(crate::example::LanguageServer {
@@ -63,6 +66,14 @@ impl DeclarativeExample {
AgentProfileId::default()
};
let existing_thread_json = if let Some(path) = base.existing_thread_path {
let content = fs::read_to_string(example_dir.join(&path))
.unwrap_or_else(|_| panic!("Failed to read existing thread file: {}", path));
Some(content)
} else {
None
};
let metadata = ExampleMetadata {
name,
url: base.url,
@@ -70,6 +81,7 @@ impl DeclarativeExample {
language_server,
max_assertions: None,
profile_id,
existing_thread_json,
};
Ok(DeclarativeExample {
@@ -110,6 +122,8 @@ pub struct ExampleToml {
pub diff_assertions: BTreeMap<String, String>,
#[serde(default)]
pub thread_assertions: BTreeMap<String, String>,
#[serde(default)]
pub existing_thread_path: Option<String>,
}
#[async_trait(?Send)]