agent: Overwrite files more cautiously (#30649)
1. The `edit_file` tool tended to use `create_or_overwrite` a bit too often, leading to corruption of long files. This change replaces the boolean flag with an `EditFileMode` enum, which helps Agent make a more deliberate choice when overwriting files. With this change, the pass rate of the new eval increased from 10% to 100%. 2. eval: Added ability to run eval on top of an existing thread. Threads can now be loaded from JSON files in the `SerializedThread` format, which makes it easy to use real threads as starting points for tests/evals. 3. Don't try to restore tool cards when running in headless or eval mode -- we don't have a window to properly do this. Release Notes: - N/A
This commit is contained in:
@@ -16,6 +16,7 @@ mod add_arg_to_trait_method;
|
||||
mod code_block_citations;
|
||||
mod comment_translation;
|
||||
mod file_search;
|
||||
mod overwrite_file;
|
||||
mod planets;
|
||||
|
||||
pub fn all(examples_dir: &Path) -> Vec<Rc<dyn Example>> {
|
||||
@@ -25,6 +26,7 @@ pub fn all(examples_dir: &Path) -> Vec<Rc<dyn Example>> {
|
||||
Rc::new(code_block_citations::CodeBlockCitations),
|
||||
Rc::new(planets::Planets),
|
||||
Rc::new(comment_translation::CommentTranslation),
|
||||
Rc::new(overwrite_file::FileOverwriteExample),
|
||||
];
|
||||
|
||||
for example_path in list_declarative_examples(examples_dir).unwrap() {
|
||||
@@ -45,6 +47,7 @@ impl DeclarativeExample {
|
||||
pub fn load(example_path: &Path) -> Result<Self> {
|
||||
let name = Self::name_from_path(example_path);
|
||||
let base: ExampleToml = toml::from_str(&fs::read_to_string(&example_path)?)?;
|
||||
let example_dir = example_path.parent().unwrap();
|
||||
|
||||
let language_server = if base.require_lsp {
|
||||
Some(crate::example::LanguageServer {
|
||||
@@ -63,6 +66,14 @@ impl DeclarativeExample {
|
||||
AgentProfileId::default()
|
||||
};
|
||||
|
||||
let existing_thread_json = if let Some(path) = base.existing_thread_path {
|
||||
let content = fs::read_to_string(example_dir.join(&path))
|
||||
.unwrap_or_else(|_| panic!("Failed to read existing thread file: {}", path));
|
||||
Some(content)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let metadata = ExampleMetadata {
|
||||
name,
|
||||
url: base.url,
|
||||
@@ -70,6 +81,7 @@ impl DeclarativeExample {
|
||||
language_server,
|
||||
max_assertions: None,
|
||||
profile_id,
|
||||
existing_thread_json,
|
||||
};
|
||||
|
||||
Ok(DeclarativeExample {
|
||||
@@ -110,6 +122,8 @@ pub struct ExampleToml {
|
||||
pub diff_assertions: BTreeMap<String, String>,
|
||||
#[serde(default)]
|
||||
pub thread_assertions: BTreeMap<String, String>,
|
||||
#[serde(default)]
|
||||
pub existing_thread_path: Option<String>,
|
||||
}
|
||||
|
||||
#[async_trait(?Send)]
|
||||
|
||||
Reference in New Issue
Block a user