Have read_file support images (#30435)
This is very basic support for them. There are a number of other TODOs before this is really a first-class supported feature, so not adding any release notes for it; for now, this PR just makes it so that if read_file tries to read a PNG (which has come up in practice), it at least correctly sends it to Anthropic instead of messing up. This also lays the groundwork for future PRs for more first-class support for images in tool calls across more image file formats and LLM providers. Release Notes: - N/A --------- Co-authored-by: Agus Zubiaga <hi@aguz.me> Co-authored-by: Agus Zubiaga <agus@zed.dev>
This commit is contained in:
co-authored by
Agus Zubiaga
Agus Zubiaga
parent
f01af006e1
commit
8fdf309a4a
@@ -33,7 +33,9 @@ use language_model::{
|
||||
LanguageModelRequestMessage, LanguageModelToolUseId, MessageContent, Role, StopReason,
|
||||
};
|
||||
use markdown::parser::{CodeBlockKind, CodeBlockMetadata};
|
||||
use markdown::{HeadingLevelStyles, Markdown, MarkdownElement, MarkdownStyle, ParsedMarkdown};
|
||||
use markdown::{
|
||||
HeadingLevelStyles, Markdown, MarkdownElement, MarkdownStyle, ParsedMarkdown, PathWithRange,
|
||||
};
|
||||
use project::{ProjectEntryId, ProjectItem as _};
|
||||
use rope::Point;
|
||||
use settings::{Settings as _, SettingsStore, update_settings_file};
|
||||
@@ -430,49 +432,8 @@ fn render_markdown_code_block(
|
||||
let path_range = path_range.clone();
|
||||
move |_, window, cx| {
|
||||
workspace
|
||||
.update(cx, {
|
||||
|workspace, cx| {
|
||||
let Some(project_path) = workspace
|
||||
.project()
|
||||
.read(cx)
|
||||
.find_project_path(&path_range.path, cx)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let Some(target) = path_range.range.as_ref().map(|range| {
|
||||
Point::new(
|
||||
// Line number is 1-based
|
||||
range.start.line.saturating_sub(1),
|
||||
range.start.col.unwrap_or(0),
|
||||
)
|
||||
}) else {
|
||||
return;
|
||||
};
|
||||
let open_task = workspace.open_path(
|
||||
project_path,
|
||||
None,
|
||||
true,
|
||||
window,
|
||||
cx,
|
||||
);
|
||||
window
|
||||
.spawn(cx, async move |cx| {
|
||||
let item = open_task.await?;
|
||||
if let Some(active_editor) =
|
||||
item.downcast::<Editor>()
|
||||
{
|
||||
active_editor
|
||||
.update_in(cx, |editor, window, cx| {
|
||||
editor.go_to_singleton_buffer_point(
|
||||
target, window, cx,
|
||||
);
|
||||
})
|
||||
.ok();
|
||||
}
|
||||
anyhow::Ok(())
|
||||
})
|
||||
.detach_and_log_err(cx);
|
||||
}
|
||||
.update(cx, |workspace, cx| {
|
||||
open_path(&path_range, window, workspace, cx)
|
||||
})
|
||||
.ok();
|
||||
}
|
||||
@@ -598,6 +559,45 @@ fn render_markdown_code_block(
|
||||
.when(can_expand && !is_expanded, |this| this.max_h_80())
|
||||
}
|
||||
|
||||
fn open_path(
|
||||
path_range: &PathWithRange,
|
||||
window: &mut Window,
|
||||
workspace: &mut Workspace,
|
||||
cx: &mut Context<'_, Workspace>,
|
||||
) {
|
||||
let Some(project_path) = workspace
|
||||
.project()
|
||||
.read(cx)
|
||||
.find_project_path(&path_range.path, cx)
|
||||
else {
|
||||
return; // TODO instead of just bailing out, open that path in a buffer.
|
||||
};
|
||||
|
||||
let Some(target) = path_range.range.as_ref().map(|range| {
|
||||
Point::new(
|
||||
// Line number is 1-based
|
||||
range.start.line.saturating_sub(1),
|
||||
range.start.col.unwrap_or(0),
|
||||
)
|
||||
}) else {
|
||||
return;
|
||||
};
|
||||
let open_task = workspace.open_path(project_path, None, true, window, cx);
|
||||
window
|
||||
.spawn(cx, async move |cx| {
|
||||
let item = open_task.await?;
|
||||
if let Some(active_editor) = item.downcast::<Editor>() {
|
||||
active_editor
|
||||
.update_in(cx, |editor, window, cx| {
|
||||
editor.go_to_singleton_buffer_point(target, window, cx);
|
||||
})
|
||||
.ok();
|
||||
}
|
||||
anyhow::Ok(())
|
||||
})
|
||||
.detach_and_log_err(cx);
|
||||
}
|
||||
|
||||
fn render_code_language(
|
||||
language: Option<&Arc<Language>>,
|
||||
name_fallback: SharedString,
|
||||
|
||||
@@ -22,9 +22,9 @@ use language_model::{
|
||||
ConfiguredModel, LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
|
||||
LanguageModelId, LanguageModelKnownError, LanguageModelRegistry, LanguageModelRequest,
|
||||
LanguageModelRequestMessage, LanguageModelRequestTool, LanguageModelToolResult,
|
||||
LanguageModelToolUseId, MaxMonthlySpendReachedError, MessageContent,
|
||||
ModelRequestLimitReachedError, PaymentRequiredError, RequestUsage, Role, SelectedModel,
|
||||
StopReason, TokenUsage,
|
||||
LanguageModelToolResultContent, LanguageModelToolUseId, MaxMonthlySpendReachedError,
|
||||
MessageContent, ModelRequestLimitReachedError, PaymentRequiredError, RequestUsage, Role,
|
||||
SelectedModel, StopReason, TokenUsage,
|
||||
};
|
||||
use postage::stream::Stream as _;
|
||||
use project::Project;
|
||||
@@ -880,7 +880,13 @@ impl Thread {
|
||||
}
|
||||
|
||||
pub fn output_for_tool(&self, id: &LanguageModelToolUseId) -> Option<&Arc<str>> {
|
||||
Some(&self.tool_use.tool_result(id)?.content)
|
||||
match &self.tool_use.tool_result(id)?.content {
|
||||
LanguageModelToolResultContent::Text(str) => Some(str),
|
||||
LanguageModelToolResultContent::Image(_) => {
|
||||
// TODO: We should display image
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn card_for_tool(&self, id: &LanguageModelToolUseId) -> Option<AnyToolCard> {
|
||||
@@ -2502,7 +2508,15 @@ impl Thread {
|
||||
}
|
||||
|
||||
writeln!(markdown, "**\n")?;
|
||||
writeln!(markdown, "{}", tool_result.content)?;
|
||||
match &tool_result.content {
|
||||
LanguageModelToolResultContent::Text(str) => {
|
||||
writeln!(markdown, "{}", str)?;
|
||||
}
|
||||
LanguageModelToolResultContent::Image(image) => {
|
||||
writeln!(markdown, "", image.source)?;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(output) = tool_result.output.as_ref() {
|
||||
writeln!(
|
||||
markdown,
|
||||
|
||||
@@ -19,7 +19,7 @@ use gpui::{
|
||||
};
|
||||
use heed::Database;
|
||||
use heed::types::SerdeBincode;
|
||||
use language_model::{LanguageModelToolUseId, Role, TokenUsage};
|
||||
use language_model::{LanguageModelToolResultContent, LanguageModelToolUseId, Role, TokenUsage};
|
||||
use project::context_server_store::{ContextServerStatus, ContextServerStore};
|
||||
use project::{Project, ProjectItem, ProjectPath, Worktree};
|
||||
use prompt_store::{
|
||||
@@ -775,7 +775,7 @@ pub struct SerializedToolUse {
|
||||
pub struct SerializedToolResult {
|
||||
pub tool_use_id: LanguageModelToolUseId,
|
||||
pub is_error: bool,
|
||||
pub content: Arc<str>,
|
||||
pub content: LanguageModelToolResultContent,
|
||||
pub output: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Result;
|
||||
use assistant_tool::{AnyToolCard, Tool, ToolResultOutput, ToolUseStatus, ToolWorkingSet};
|
||||
use assistant_tool::{
|
||||
AnyToolCard, Tool, ToolResultContent, ToolResultOutput, ToolUseStatus, ToolWorkingSet,
|
||||
};
|
||||
use collections::HashMap;
|
||||
use futures::FutureExt as _;
|
||||
use futures::future::Shared;
|
||||
use gpui::{App, Entity, SharedString, Task};
|
||||
use language_model::{
|
||||
ConfiguredModel, LanguageModel, LanguageModelRequest, LanguageModelToolResult,
|
||||
LanguageModelToolUse, LanguageModelToolUseId, Role,
|
||||
LanguageModelToolResultContent, LanguageModelToolUse, LanguageModelToolUseId, Role,
|
||||
};
|
||||
use project::Project;
|
||||
use ui::{IconName, Window};
|
||||
@@ -165,10 +167,16 @@ impl ToolUseState {
|
||||
|
||||
let status = (|| {
|
||||
if let Some(tool_result) = tool_result {
|
||||
let content = tool_result
|
||||
.content
|
||||
.to_str()
|
||||
.map(|str| str.to_owned().into())
|
||||
.unwrap_or_default();
|
||||
|
||||
return if tool_result.is_error {
|
||||
ToolUseStatus::Error(tool_result.content.clone().into())
|
||||
ToolUseStatus::Error(content)
|
||||
} else {
|
||||
ToolUseStatus::Finished(tool_result.content.clone().into())
|
||||
ToolUseStatus::Finished(content)
|
||||
};
|
||||
}
|
||||
|
||||
@@ -399,21 +407,44 @@ impl ToolUseState {
|
||||
let tool_result = output.content;
|
||||
const BYTES_PER_TOKEN_ESTIMATE: usize = 3;
|
||||
|
||||
// Protect from clearly large output
|
||||
let old_use = self.pending_tool_uses_by_id.remove(&tool_use_id);
|
||||
|
||||
// Protect from overly large output
|
||||
let tool_output_limit = configured_model
|
||||
.map(|model| model.model.max_token_count() * BYTES_PER_TOKEN_ESTIMATE)
|
||||
.unwrap_or(usize::MAX);
|
||||
|
||||
let tool_result = if tool_result.len() <= tool_output_limit {
|
||||
tool_result
|
||||
} else {
|
||||
let truncated = truncate_lines_to_byte_limit(&tool_result, tool_output_limit);
|
||||
let content = match tool_result {
|
||||
ToolResultContent::Text(text) => {
|
||||
let truncated = truncate_lines_to_byte_limit(&text, tool_output_limit);
|
||||
|
||||
format!(
|
||||
"Tool result too long. The first {} bytes:\n\n{}",
|
||||
truncated.len(),
|
||||
truncated
|
||||
)
|
||||
LanguageModelToolResultContent::Text(
|
||||
format!(
|
||||
"Tool result too long. The first {} bytes:\n\n{}",
|
||||
truncated.len(),
|
||||
truncated
|
||||
)
|
||||
.into(),
|
||||
)
|
||||
}
|
||||
ToolResultContent::Image(language_model_image) => {
|
||||
if language_model_image.estimate_tokens() < tool_output_limit {
|
||||
LanguageModelToolResultContent::Image(language_model_image)
|
||||
} else {
|
||||
self.tool_results.insert(
|
||||
tool_use_id.clone(),
|
||||
LanguageModelToolResult {
|
||||
tool_use_id: tool_use_id.clone(),
|
||||
tool_name,
|
||||
content: "Tool responded with an image that would exceeded the remaining tokens".into(),
|
||||
is_error: true,
|
||||
output: None,
|
||||
},
|
||||
);
|
||||
|
||||
return old_use;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
self.tool_results.insert(
|
||||
@@ -421,12 +452,13 @@ impl ToolUseState {
|
||||
LanguageModelToolResult {
|
||||
tool_use_id: tool_use_id.clone(),
|
||||
tool_name,
|
||||
content: tool_result.into(),
|
||||
content,
|
||||
is_error: false,
|
||||
output: output.output,
|
||||
},
|
||||
);
|
||||
self.pending_tool_uses_by_id.remove(&tool_use_id)
|
||||
|
||||
old_use
|
||||
}
|
||||
Err(err) => {
|
||||
self.tool_results.insert(
|
||||
@@ -434,7 +466,7 @@ impl ToolUseState {
|
||||
LanguageModelToolResult {
|
||||
tool_use_id: tool_use_id.clone(),
|
||||
tool_name,
|
||||
content: err.to_string().into(),
|
||||
content: LanguageModelToolResultContent::Text(err.to_string().into()),
|
||||
is_error: true,
|
||||
output: None,
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user