oak-team/oak-gpui
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

search: New old search implementation (#39956)

Browse Source 
This is an in-progress work on changing how task scheduler affects
performance of project search. Instead of relying on tasks being
executed at a discretion of the task scheduler, we want to experiment
with having a set of "agents" that prioritize driving in-progress
project search matches to completion over pushing the whole thing to
completion. This should hopefully significantly improve throughput &
latency of project search.

Release Notes:

- Improved project search performance

---------

Co-authored-by: Smit Barmase <smit@zed.dev>
Co-authored-by: Smit Barmase <heysmitbarmase@gmail.com>
This commit is contained in:
Piotr Osiewicz
2025-10-20 16:40:02 +02:00
committed by GitHub
co-authored by Smit Barmase Smit Barmase
parent 85c2aa7325
commit 7c4fb5a899
11 changed files with 1023 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/project/src/project.rs
+57 -178
View File
@@ -11,6 +11,7 @@ pub mod lsp_command;
pub mod lsp_store;
mod manifest_tree;
pub mod prettier_store;
mod project_search;
pub mod project_settings;
pub mod search;
mod task_inventory;
@@ -39,6 +40,7 @@ use crate::{
agent_server_store::AllAgentServersSettings,
git_store::GitStore,
lsp_store::{SymbolLocation, log_store::LogKind},
project_search::SearchResultsHandle,
};
pub use agent_server_store::{AgentServerStore, AgentServersUpdated};
pub use git_store::{
@@ -46,6 +48,7 @@ pub use git_store::{
git_traversal::{ChildEntriesGitIter, GitEntry, GitEntryRef, GitTraversal},
};
pub use manifest_tree::ManifestTree;
pub use project_search::Search;
use anyhow::{Context as _, Result, anyhow};
use buffer_store::{BufferStore, BufferStoreEvent};
@@ -109,7 +112,7 @@ use snippet_provider::SnippetProvider;
use std::{
borrow::Cow,
collections::BTreeMap,
ops::Range,
ops::{Not as _, Range},
path::{Path, PathBuf},
pin::pin,
str,
@@ -123,7 +126,7 @@ use text::{Anchor, BufferId, OffsetRangeExt, Point, Rope};
use toolchain_store::EmptyToolchainStore;
use util::{
ResultExt as _, maybe,
paths::{PathStyle, SanitizedPath, compare_paths, is_absolute},
paths::{PathStyle, SanitizedPath, is_absolute},
rel_path::RelPath,
};
use worktree::{CreatedEntry, Snapshot, Traversal};
@@ -150,8 +153,6 @@ pub use lsp_store::{
};
pub use toolchain_store::{ToolchainStore, Toolchains};
const MAX_PROJECT_SEARCH_HISTORY_SIZE: usize = 500;
const MAX_SEARCH_RESULT_FILES: usize = 5_000;
const MAX_SEARCH_RESULT_RANGES: usize = 10_000;
pub trait ProjectItem: 'static {
fn try_open(
@@ -3935,179 +3936,44 @@ impl Project {
})
}
pub fn search(&mut self, query: SearchQuery, cx: &mut Context<Self>) -> Receiver<SearchResult> {
let (result_tx, result_rx) = smol::channel::unbounded();
let matching_buffers_rx = if query.is_opened_only() {
self.sort_search_candidates(&query, cx)
} else {
self.find_search_candidate_buffers(&query, MAX_SEARCH_RESULT_FILES + 1, cx)
};
cx.spawn(async move |_, cx| {
let mut range_count = 0;
let mut buffer_count = 0;
let mut limit_reached = false;
let query = Arc::new(query);
let chunks = matching_buffers_rx.ready_chunks(64);
// Now that we know what paths match the query, we will load at most
// 64 buffers at a time to avoid overwhelming the main thread. For each
// opened buffer, we will spawn a background task that retrieves all the
// ranges in the buffer matched by the query.
let mut chunks = pin!(chunks);
'outer: while let Some(matching_buffer_chunk) = chunks.next().await {
let mut chunk_results = Vec::with_capacity(matching_buffer_chunk.len());
for buffer in matching_buffer_chunk {
let query = query.clone();
let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot())?;
chunk_results.push(cx.background_spawn(async move {
let ranges = query
.search(&snapshot, None)
.await
.iter()
.map(|range| {
snapshot.anchor_before(range.start)
..snapshot.anchor_after(range.end)
})
.collect::<Vec<_>>();
anyhow::Ok((buffer, ranges))
}));
}
let chunk_results = futures::future::join_all(chunk_results).await;
for result in chunk_results {
if let Some((buffer, ranges)) = result.log_err() {
range_count += ranges.len();
buffer_count += 1;
result_tx
.send(SearchResult::Buffer { buffer, ranges })
.await?;
if buffer_count > MAX_SEARCH_RESULT_FILES
|| range_count > MAX_SEARCH_RESULT_RANGES
{
limit_reached = true;
break 'outer;
}
}
}
}
if limit_reached {
result_tx.send(SearchResult::LimitReached).await?;
}
anyhow::Ok(())
})
.detach();
result_rx
}
fn find_search_candidate_buffers(
&mut self,
query: &SearchQuery,
limit: usize,
cx: &mut Context<Project>,
) -> Receiver<Entity<Buffer>> {
if self.is_local() {
let fs = self.fs.clone();
self.buffer_store.update(cx, |buffer_store, cx| {
buffer_store.find_search_candidates(query, limit, fs, cx)
})
} else {
self.find_search_candidates_remote(query, limit, cx)
}
}
fn sort_search_candidates(
&mut self,
search_query: &SearchQuery,
cx: &mut Context<Project>,
) -> Receiver<Entity<Buffer>> {
let worktree_store = self.worktree_store.read(cx);
let mut buffers = search_query
.buffers()
.into_iter()
.flatten()
.filter(|buffer| {
let b = buffer.read(cx);
if let Some(file) = b.file() {
if !search_query.match_path(file.path().as_std_path()) {
return false;
}
if let Some(entry) = b
.entry_id(cx)
.and_then(|entry_id| worktree_store.entry_for_id(entry_id, cx))
&& entry.is_ignored
&& !search_query.include_ignored()
{
return false;
}
}
true
})
.collect::<Vec<_>>();
let (tx, rx) = smol::channel::unbounded();
buffers.sort_by(|a, b| match (a.read(cx).file(), b.read(cx).file()) {
(None, None) => a.read(cx).remote_id().cmp(&b.read(cx).remote_id()),
(None, Some(_)) => std::cmp::Ordering::Less,
(Some(_), None) => std::cmp::Ordering::Greater,
(Some(a), Some(b)) => compare_paths(
(a.path().as_std_path(), true),
(b.path().as_std_path(), true),
),
});
for buffer in buffers {
tx.send_blocking(buffer.clone()).unwrap()
}
rx
}
fn find_search_candidates_remote(
&mut self,
query: &SearchQuery,
limit: usize,
cx: &mut Context<Project>,
) -> Receiver<Entity<Buffer>> {
let (tx, rx) = smol::channel::unbounded();
let (client, remote_id): (AnyProtoClient, _) = if let Some(ssh_client) = &self.remote_client
{
(ssh_client.read(cx).proto_client(), 0)
fn search_impl(&mut self, query: SearchQuery, cx: &mut Context<Self>) -> SearchResultsHandle {
let client: Option<(AnyProtoClient, _)> = if let Some(ssh_client) = &self.remote_client {
Some((ssh_client.read(cx).proto_client(), 0))
} else if let Some(remote_id) = self.remote_id() {
(self.collab_client.clone().into(), remote_id)
self.is_local()
.not()
.then(|| (self.collab_client.clone().into(), remote_id))
} else {
return rx;
None
};
let request = client.request(proto::FindSearchCandidates {
project_id: remote_id,
query: Some(query.to_proto()),
limit: limit as _,
});
let guard = self.retain_remotely_created_models(cx);
cx.spawn(async move |project, cx| {
let response = request.await?;
for buffer_id in response.buffer_ids {
let buffer_id = BufferId::new(buffer_id)?;
let buffer = project
.update(cx, |project, cx| {
project.buffer_store.update(cx, |buffer_store, cx| {
buffer_store.wait_for_remote_buffer(buffer_id, cx)
})
})?
.await?;
let _ = tx.send(buffer).await;
let searcher = if query.is_opened_only() {
project_search::Search::open_buffers_only(
self.buffer_store.clone(),
self.worktree_store.clone(),
project_search::Search::MAX_SEARCH_RESULT_FILES + 1,
)
} else {
match client {
Some((client, remote_id)) => project_search::Search::remote(
self.buffer_store.clone(),
self.worktree_store.clone(),
project_search::Search::MAX_SEARCH_RESULT_FILES + 1,
(client, remote_id, self.remotely_created_models.clone()),
),
None => project_search::Search::local(
self.fs.clone(),
self.buffer_store.clone(),
self.worktree_store.clone(),
project_search::Search::MAX_SEARCH_RESULT_FILES + 1,
cx,
),
}
};
searcher.into_handle(query, cx)
}
drop(guard);
anyhow::Ok(())
})
.detach_and_log_err(cx);
rx
pub fn search(&mut self, query: SearchQuery, cx: &mut Context<Self>) -> Receiver<SearchResult> {
self.search_impl(query, cx).results(cx)
}
pub fn request_lsp<R: LspCommand>(
@@ -4832,18 +4698,31 @@ impl Project {
fn retain_remotely_created_models(
&mut self,
cx: &mut Context<Self>,
) -> RemotelyCreatedModelGuard {
Self::retain_remotely_created_models_impl(
&self.remotely_created_models,
&self.buffer_store,
&self.worktree_store,
cx,
)
}
fn retain_remotely_created_models_impl(
models: &Arc<Mutex<RemotelyCreatedModels>>,
buffer_store: &Entity<BufferStore>,
worktree_store: &Entity<WorktreeStore>,
cx: &mut App,
) -> RemotelyCreatedModelGuard {
{
let mut remotely_create_models = self.remotely_created_models.lock();
let mut remotely_create_models = models.lock();
if remotely_create_models.retain_count == 0 {
remotely_create_models.buffers = self.buffer_store.read(cx).buffers().collect();
remotely_create_models.worktrees =
self.worktree_store.read(cx).worktrees().collect();
remotely_create_models.buffers = buffer_store.read(cx).buffers().collect();
remotely_create_models.worktrees = worktree_store.read(cx).worktrees().collect();
}
remotely_create_models.retain_count += 1;
}
RemotelyCreatedModelGuard {
remote_models: Arc::downgrade(&self.remotely_created_models),
remote_models: Arc::downgrade(&models),
}
}
@@ -4913,7 +4792,7 @@ impl Project {
let query =
SearchQuery::from_proto(message.query.context("missing query field")?, path_style)?;
let results = this.update(&mut cx, |this, cx| {
this.find_search_candidate_buffers(&query, message.limit as _, cx)
this.search_impl(query, cx).matching_buffers(cx)
})?;
let mut response = proto::FindSearchCandidatesResponse {