oak-team/oak-gpui
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ai: Add an eval for the inline assistant (#43291)

Browse Source 
Release Notes:

- N/A

---------

Co-authored-by: Mikayla Maki <mikayla.c.maki@gmail.com>
This commit is contained in:
Michael Benfield
2025-12-03 20:32:25 +00:00
committed by GitHub
co-authored by Mikayla Maki
parent 92dcfdef76
commit 290a1550aa
19 changed files with 832 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/agent_ui/src/agent_panel.rs
+3 -2
View File
@@ -2685,16 +2685,17 @@ impl rules_library::InlineAssistDelegate for PromptLibraryInlineAssist {
return;
};
let project = workspace.read(cx).project().downgrade();
let thread_store = panel.read(cx).thread_store().clone();
assistant.assist(
prompt_editor,
self.workspace.clone(),
project,
panel.read(cx).thread_store().clone(),
thread_store,
None,
initial_prompt,
window,
cx,
)
);
})
}
crates/agent_ui/src/agent_ui.rs
+2
View File
@@ -7,6 +7,8 @@ mod buffer_codegen;
mod completion_provider;
mod context;
mod context_server_configuration;
#[cfg(test)]
mod evals;
mod inline_assistant;
mod inline_prompt_editor;
mod language_model_selector;
crates/agent_ui/src/buffer_codegen.rs
+1
View File
@@ -719,6 +719,7 @@ impl CodegenAlternative {
output_tokens = usage.output_tokens,
)
}
cx.emit(CodegenEvent::Finished);
cx.notify();
})
crates/agent_ui/src/evals.rs
+89
View File
@@ -0,0 +1,89 @@
use std::str::FromStr;
use crate::inline_assistant::test::run_inline_assistant_test;
use eval_utils::{EvalOutput, NoProcessor};
use gpui::TestAppContext;
use language_model::{LanguageModelRegistry, SelectedModel};
use rand::{SeedableRng as _, rngs::StdRng};
#[test]
#[cfg_attr(not(feature = "unit-eval"), ignore)]
fn eval_single_cursor_edit() {
eval_utils::eval(20, 1.0, NoProcessor, move || {
run_eval(
&EvalInput {
prompt: "Rename this variable to buffer_text".to_string(),
buffer: indoc::indoc! {"
struct EvalExampleStruct {
text: Strˇing,
prompt: String,
}
"}
.to_string(),
},
&|_, output| {
let expected = indoc::indoc! {"
struct EvalExampleStruct {
buffer_text: String,
prompt: String,
}
"};
if output == expected {
EvalOutput {
outcome: eval_utils::OutcomeKind::Passed,
data: "Passed!".to_string(),
metadata: (),
}
} else {
EvalOutput {
outcome: eval_utils::OutcomeKind::Failed,
data: format!("Failed to rename variable, output: {}", output),
metadata: (),
}
}
},
)
});
}
struct EvalInput {
buffer: String,
prompt: String,
}
fn run_eval(
input: &EvalInput,
judge: &dyn Fn(&EvalInput, &str) -> eval_utils::EvalOutput<()>,
) -> eval_utils::EvalOutput<()> {
let dispatcher = gpui::TestDispatcher::new(StdRng::from_os_rng());
let mut cx = TestAppContext::build(dispatcher, None);
cx.skip_drawing();
let buffer_text = run_inline_assistant_test(
input.buffer.clone(),
input.prompt.clone(),
|cx| {
// Reconfigure to use a real model instead of the fake one
let model_name = std::env::var("ZED_AGENT_MODEL")
.unwrap_or("anthropic/claude-sonnet-4-latest".into());
let selected_model = SelectedModel::from_str(&model_name)
.expect("Invalid model format. Use 'provider/model-id'");
log::info!("Selected model: {selected_model:?}");
cx.update(|_, cx| {
LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
registry.select_inline_assistant_model(Some(&selected_model), cx);
});
});
},
|_cx| {
log::info!("Waiting for actual response from the LLM...");
},
&mut cx,
);
judge(input, &buffer_text)
}
crates/agent_ui/src/inline_assistant.rs
+190 -6
View File
@@ -32,7 +32,7 @@ use editor::{
},
};
use fs::Fs;
use futures::FutureExt;
use futures::{FutureExt, channel::mpsc};
use gpui::{
App, Context, Entity, Focusable, Global, HighlightStyle, Subscription, Task, UpdateGlobal,
WeakEntity, Window, point,
@@ -102,6 +102,7 @@ pub struct InlineAssistant {
prompt_builder: Arc<PromptBuilder>,
telemetry: Arc<Telemetry>,
fs: Arc<dyn Fs>,
_inline_assistant_completions: Option<mpsc::UnboundedSender<anyhow::Result<InlineAssistId>>>,
}
impl Global for InlineAssistant {}
@@ -123,9 +124,18 @@ impl InlineAssistant {
prompt_builder,
telemetry,
fs,
_inline_assistant_completions: None,
}
}
#[cfg(any(test, feature = "test-support"))]
pub fn set_completion_receiver(
&mut self,
sender: mpsc::UnboundedSender<anyhow::Result<InlineAssistId>>,
) {
self._inline_assistant_completions = Some(sender);
}
pub fn register_workspace(
&mut self,
workspace: &Entity<Workspace>,
@@ -287,7 +297,7 @@ impl InlineAssistant {
action.prompt.clone(),
window,
cx,
)
);
})
}
InlineAssistTarget::Terminal(active_terminal) => {
@@ -301,8 +311,8 @@ impl InlineAssistant {
action.prompt.clone(),
window,
cx,
)
})
);
});
}
};
@@ -598,13 +608,13 @@ impl InlineAssistant {
initial_prompt: Option<String>,
window: &mut Window,
cx: &mut App,
) {
) -> Option<InlineAssistId> {
let snapshot = editor.update(cx, |editor, cx| editor.snapshot(window, cx));
let Some((codegen_ranges, newest_selection)) =
self.codegen_ranges(editor, &snapshot, window, cx)
else {
return;
return None;
};
let assist_to_focus = self.batch_assist(
@@ -624,6 +634,8 @@ impl InlineAssistant {
if let Some(assist_id) = assist_to_focus {
self.focus_assist(assist_id, window, cx);
}
assist_to_focus
}
pub fn suggest_assist(
@@ -1740,6 +1752,16 @@ impl InlineAssist {
&& assist.decorations.is_none()
&& let Some(workspace) = assist.workspace.upgrade()
{
#[cfg(any(test, feature = "test-support"))]
if let Some(sender) = &mut this._inline_assistant_completions {
sender
.unbounded_send(Err(anyhow::anyhow!(
"Inline assistant error: {}",
error
)))
.ok();
}
let error = format!("Inline assistant error: {}", error);
workspace.update(cx, |workspace, cx| {
struct InlineAssistantError;
@@ -1750,6 +1772,11 @@ impl InlineAssist {
workspace.show_toast(Toast::new(id, error), cx);
})
} else {
#[cfg(any(test, feature = "test-support"))]
if let Some(sender) = &mut this._inline_assistant_completions {
sender.unbounded_send(Ok(assist_id)).ok();
}
}
if assist.decorations.is_none() {
@@ -1943,3 +1970,160 @@ fn merge_ranges(ranges: &mut Vec<Range<Anchor>>, buffer: &MultiBufferSnapshot) {
}
}
}
#[cfg(any(test, feature = "test-support"))]
pub mod test {
use std::sync::Arc;
use agent::HistoryStore;
use assistant_text_thread::TextThreadStore;
use client::{Client, UserStore};
use editor::{Editor, MultiBuffer, MultiBufferOffset};
use fs::FakeFs;
use futures::channel::mpsc;
use gpui::{AppContext, TestAppContext, UpdateGlobal as _};
use language::Buffer;
use language_model::LanguageModelRegistry;
use project::Project;
use prompt_store::PromptBuilder;
use smol::stream::StreamExt as _;
use util::test::marked_text_ranges;
use workspace::Workspace;
use crate::InlineAssistant;
pub fn run_inline_assistant_test<SetupF, TestF>(
base_buffer: String,
prompt: String,
setup: SetupF,
test: TestF,
cx: &mut TestAppContext,
) -> String
where
SetupF: FnOnce(&mut gpui::VisualTestContext),
TestF: FnOnce(&mut gpui::VisualTestContext),
{
let fs = FakeFs::new(cx.executor());
let app_state = cx.update(|cx| workspace::AppState::test(cx));
let prompt_builder = Arc::new(PromptBuilder::new(None).unwrap());
let http = Arc::new(reqwest_client::ReqwestClient::user_agent("agent tests").unwrap());
let client = cx.update(|cx| {
cx.set_http_client(http);
Client::production(cx)
});
let mut inline_assistant =
InlineAssistant::new(fs.clone(), prompt_builder, client.telemetry().clone());
let (tx, mut completion_rx) = mpsc::unbounded();
inline_assistant.set_completion_receiver(tx);
// Initialize settings and client
cx.update(|cx| {
gpui_tokio::init(cx);
settings::init(cx);
client::init(&client, cx);
workspace::init(app_state.clone(), cx);
let user_store = cx.new(|cx| UserStore::new(client.clone(), cx));
language_model::init(client.clone(), cx);
language_models::init(user_store, client.clone(), cx);
cx.set_global(inline_assistant);
});
let project = cx
.executor()
.block_test(async { Project::test(fs.clone(), [], cx).await });
// Create workspace with window
let (workspace, cx) = cx.add_window_view(|window, cx| {
window.activate_window();
Workspace::new(None, project.clone(), app_state.clone(), window, cx)
});
setup(cx);
let (_editor, buffer) = cx.update(|window, cx| {
let buffer = cx.new(|cx| Buffer::local("", cx));
let multibuffer = cx.new(|cx| MultiBuffer::singleton(buffer.clone(), cx));
let editor = cx.new(|cx| Editor::for_multibuffer(multibuffer, None, window, cx));
editor.update(cx, |editor, cx| {
let (unmarked_text, selection_ranges) = marked_text_ranges(&base_buffer, true);
editor.set_text(unmarked_text, window, cx);
editor.change_selections(Default::default(), window, cx, |s| {
s.select_ranges(
selection_ranges.into_iter().map(|range| {
MultiBufferOffset(range.start)..MultiBufferOffset(range.end)
}),
)
})
});
let text_thread_store = cx.new(|cx| TextThreadStore::fake(project.clone(), cx));
let history_store = cx.new(|cx| HistoryStore::new(text_thread_store, cx));
// Add editor to workspace
workspace.update(cx, |workspace, cx| {
workspace.add_item_to_active_pane(Box::new(editor.clone()), None, true, window, cx);
});
// Call assist method
InlineAssistant::update_global(cx, |inline_assistant, cx| {
let assist_id = inline_assistant
.assist(
&editor,
workspace.downgrade(),
project.downgrade(),
history_store, // thread_store
None, // prompt_store
Some(prompt),
window,
cx,
)
.unwrap();
inline_assistant.start_assist(assist_id, window, cx);
});
(editor, buffer)
});
cx.run_until_parked();
test(cx);
cx.executor()
.block_test(async { completion_rx.next().await });
buffer.read_with(cx, |buffer, _| buffer.text())
}
#[allow(unused)]
pub fn test_inline_assistant(
base_buffer: &'static str,
llm_output: &'static str,
cx: &mut TestAppContext,
) -> String {
run_inline_assistant_test(
base_buffer.to_string(),
"Prompt doesn't matter because we're using a fake model".to_string(),
|cx| {
cx.update(|_, cx| LanguageModelRegistry::test(cx));
},
|cx| {
let fake_model = cx.update(|_, cx| {
LanguageModelRegistry::global(cx)
.update(cx, |registry, _| registry.fake_model())
});
let fake = fake_model.as_fake();
// let fake = fake_model;
fake.send_last_completion_stream_text_chunk(llm_output.to_string());
fake.end_last_completion_stream();
// Run again to process the model's response
cx.run_until_parked();
},
cx,
)
}
}