Have read_file support images (#30435)

This is very basic support for them. There are a number of other TODOs
before this is really a first-class supported feature, so not adding any
release notes for it; for now, this PR just makes it so that if
read_file tries to read a PNG (which has come up in practice), it at
least correctly sends it to Anthropic instead of messing up.

This also lays the groundwork for future PRs for more first-class
support for images in tool calls across more image file formats and LLM
providers.

Release Notes:

- N/A

---------

Co-authored-by: Agus Zubiaga <hi@aguz.me>
Co-authored-by: Agus Zubiaga <agus@zed.dev>
This commit is contained in:
Richard Feldman
2025-05-13 10:58:00 +02:00
committed by GitHub
co-authored by Agus Zubiaga Agus Zubiaga
parent f01af006e1
commit 8fdf309a4a
30 changed files with 557 additions and 194 deletions
+44 -44
View File
@@ -33,7 +33,9 @@ use language_model::{
LanguageModelRequestMessage, LanguageModelToolUseId, MessageContent, Role, StopReason,
};
use markdown::parser::{CodeBlockKind, CodeBlockMetadata};
use markdown::{HeadingLevelStyles, Markdown, MarkdownElement, MarkdownStyle, ParsedMarkdown};
use markdown::{
HeadingLevelStyles, Markdown, MarkdownElement, MarkdownStyle, ParsedMarkdown, PathWithRange,
};
use project::{ProjectEntryId, ProjectItem as _};
use rope::Point;
use settings::{Settings as _, SettingsStore, update_settings_file};
@@ -430,49 +432,8 @@ fn render_markdown_code_block(
let path_range = path_range.clone();
move |_, window, cx| {
workspace
.update(cx, {
|workspace, cx| {
let Some(project_path) = workspace
.project()
.read(cx)
.find_project_path(&path_range.path, cx)
else {
return;
};
let Some(target) = path_range.range.as_ref().map(|range| {
Point::new(
// Line number is 1-based
range.start.line.saturating_sub(1),
range.start.col.unwrap_or(0),
)
}) else {
return;
};
let open_task = workspace.open_path(
project_path,
None,
true,
window,
cx,
);
window
.spawn(cx, async move |cx| {
let item = open_task.await?;
if let Some(active_editor) =
item.downcast::<Editor>()
{
active_editor
.update_in(cx, |editor, window, cx| {
editor.go_to_singleton_buffer_point(
target, window, cx,
);
})
.ok();
}
anyhow::Ok(())
})
.detach_and_log_err(cx);
}
.update(cx, |workspace, cx| {
open_path(&path_range, window, workspace, cx)
})
.ok();
}
@@ -598,6 +559,45 @@ fn render_markdown_code_block(
.when(can_expand && !is_expanded, |this| this.max_h_80())
}
fn open_path(
path_range: &PathWithRange,
window: &mut Window,
workspace: &mut Workspace,
cx: &mut Context<'_, Workspace>,
) {
let Some(project_path) = workspace
.project()
.read(cx)
.find_project_path(&path_range.path, cx)
else {
return; // TODO instead of just bailing out, open that path in a buffer.
};
let Some(target) = path_range.range.as_ref().map(|range| {
Point::new(
// Line number is 1-based
range.start.line.saturating_sub(1),
range.start.col.unwrap_or(0),
)
}) else {
return;
};
let open_task = workspace.open_path(project_path, None, true, window, cx);
window
.spawn(cx, async move |cx| {
let item = open_task.await?;
if let Some(active_editor) = item.downcast::<Editor>() {
active_editor
.update_in(cx, |editor, window, cx| {
editor.go_to_singleton_buffer_point(target, window, cx);
})
.ok();
}
anyhow::Ok(())
})
.detach_and_log_err(cx);
}
fn render_code_language(
language: Option<&Arc<Language>>,
name_fallback: SharedString,
+19 -5
View File
@@ -22,9 +22,9 @@ use language_model::{
ConfiguredModel, LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
LanguageModelId, LanguageModelKnownError, LanguageModelRegistry, LanguageModelRequest,
LanguageModelRequestMessage, LanguageModelRequestTool, LanguageModelToolResult,
LanguageModelToolUseId, MaxMonthlySpendReachedError, MessageContent,
ModelRequestLimitReachedError, PaymentRequiredError, RequestUsage, Role, SelectedModel,
StopReason, TokenUsage,
LanguageModelToolResultContent, LanguageModelToolUseId, MaxMonthlySpendReachedError,
MessageContent, ModelRequestLimitReachedError, PaymentRequiredError, RequestUsage, Role,
SelectedModel, StopReason, TokenUsage,
};
use postage::stream::Stream as _;
use project::Project;
@@ -880,7 +880,13 @@ impl Thread {
}
pub fn output_for_tool(&self, id: &LanguageModelToolUseId) -> Option<&Arc<str>> {
Some(&self.tool_use.tool_result(id)?.content)
match &self.tool_use.tool_result(id)?.content {
LanguageModelToolResultContent::Text(str) => Some(str),
LanguageModelToolResultContent::Image(_) => {
// TODO: We should display image
None
}
}
}
pub fn card_for_tool(&self, id: &LanguageModelToolUseId) -> Option<AnyToolCard> {
@@ -2502,7 +2508,15 @@ impl Thread {
}
writeln!(markdown, "**\n")?;
writeln!(markdown, "{}", tool_result.content)?;
match &tool_result.content {
LanguageModelToolResultContent::Text(str) => {
writeln!(markdown, "{}", str)?;
}
LanguageModelToolResultContent::Image(image) => {
writeln!(markdown, "![Image](data:base64,{})", image.source)?;
}
}
if let Some(output) = tool_result.output.as_ref() {
writeln!(
markdown,
+2 -2
View File
@@ -19,7 +19,7 @@ use gpui::{
};
use heed::Database;
use heed::types::SerdeBincode;
use language_model::{LanguageModelToolUseId, Role, TokenUsage};
use language_model::{LanguageModelToolResultContent, LanguageModelToolUseId, Role, TokenUsage};
use project::context_server_store::{ContextServerStatus, ContextServerStore};
use project::{Project, ProjectItem, ProjectPath, Worktree};
use prompt_store::{
@@ -775,7 +775,7 @@ pub struct SerializedToolUse {
pub struct SerializedToolResult {
pub tool_use_id: LanguageModelToolUseId,
pub is_error: bool,
pub content: Arc<str>,
pub content: LanguageModelToolResultContent,
pub output: Option<serde_json::Value>,
}
+49 -17
View File
@@ -1,14 +1,16 @@
use std::sync::Arc;
use anyhow::Result;
use assistant_tool::{AnyToolCard, Tool, ToolResultOutput, ToolUseStatus, ToolWorkingSet};
use assistant_tool::{
AnyToolCard, Tool, ToolResultContent, ToolResultOutput, ToolUseStatus, ToolWorkingSet,
};
use collections::HashMap;
use futures::FutureExt as _;
use futures::future::Shared;
use gpui::{App, Entity, SharedString, Task};
use language_model::{
ConfiguredModel, LanguageModel, LanguageModelRequest, LanguageModelToolResult,
LanguageModelToolUse, LanguageModelToolUseId, Role,
LanguageModelToolResultContent, LanguageModelToolUse, LanguageModelToolUseId, Role,
};
use project::Project;
use ui::{IconName, Window};
@@ -165,10 +167,16 @@ impl ToolUseState {
let status = (|| {
if let Some(tool_result) = tool_result {
let content = tool_result
.content
.to_str()
.map(|str| str.to_owned().into())
.unwrap_or_default();
return if tool_result.is_error {
ToolUseStatus::Error(tool_result.content.clone().into())
ToolUseStatus::Error(content)
} else {
ToolUseStatus::Finished(tool_result.content.clone().into())
ToolUseStatus::Finished(content)
};
}
@@ -399,21 +407,44 @@ impl ToolUseState {
let tool_result = output.content;
const BYTES_PER_TOKEN_ESTIMATE: usize = 3;
// Protect from clearly large output
let old_use = self.pending_tool_uses_by_id.remove(&tool_use_id);
// Protect from overly large output
let tool_output_limit = configured_model
.map(|model| model.model.max_token_count() * BYTES_PER_TOKEN_ESTIMATE)
.unwrap_or(usize::MAX);
let tool_result = if tool_result.len() <= tool_output_limit {
tool_result
} else {
let truncated = truncate_lines_to_byte_limit(&tool_result, tool_output_limit);
let content = match tool_result {
ToolResultContent::Text(text) => {
let truncated = truncate_lines_to_byte_limit(&text, tool_output_limit);
format!(
"Tool result too long. The first {} bytes:\n\n{}",
truncated.len(),
truncated
)
LanguageModelToolResultContent::Text(
format!(
"Tool result too long. The first {} bytes:\n\n{}",
truncated.len(),
truncated
)
.into(),
)
}
ToolResultContent::Image(language_model_image) => {
if language_model_image.estimate_tokens() < tool_output_limit {
LanguageModelToolResultContent::Image(language_model_image)
} else {
self.tool_results.insert(
tool_use_id.clone(),
LanguageModelToolResult {
tool_use_id: tool_use_id.clone(),
tool_name,
content: "Tool responded with an image that would exceeded the remaining tokens".into(),
is_error: true,
output: None,
},
);
return old_use;
}
}
};
self.tool_results.insert(
@@ -421,12 +452,13 @@ impl ToolUseState {
LanguageModelToolResult {
tool_use_id: tool_use_id.clone(),
tool_name,
content: tool_result.into(),
content,
is_error: false,
output: output.output,
},
);
self.pending_tool_uses_by_id.remove(&tool_use_id)
old_use
}
Err(err) => {
self.tool_results.insert(
@@ -434,7 +466,7 @@ impl ToolUseState {
LanguageModelToolResult {
tool_use_id: tool_use_id.clone(),
tool_name,
content: err.to_string().into(),
content: LanguageModelToolResultContent::Text(err.to_string().into()),
is_error: true,
output: None,
},