oak-team/oak-gpui
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

More fixes to the semantic index's chunking (#11376)

Browse Source 
This fixes a tricky intermittent issue I was seeing, where failed to
chunk certain files correctly because of the way we reuse Tree-sitter
`Parser` instances across parses.

I've also accounted for leading comments in chunk boundaries, so that
items are grouped with their leading comments whenever possible when
chunking.

Finally, we've changed the `debug project index` action so that it opens
a simple debug view in a pane, instead of printing paths to the console.
This lets you click into a path and see how it was chunked.

Release Notes:

- N/A

---------

Co-authored-by: Marshall <marshall@zed.dev>
This commit is contained in:
Max Brunsfeld
2024-05-03 19:00:18 -07:00
committed by GitHub
co-authored by Marshall
parent 335c307b93
commit 6964302d89
11 changed files with 532 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/language/src/buffer_tests.rs
+7 -18
View File
@@ -1818,7 +1818,7 @@ fn test_language_scope_at_with_javascript(cx: &mut AppContext) {
let snapshot = buffer.snapshot();
let config = snapshot.language_scope_at(0).unwrap();
assert_eq!(config.line_comment_prefixes().unwrap(), &[Arc::from("// ")]);
assert_eq!(config.line_comment_prefixes(), &[Arc::from("// ")]);
// Both bracket pairs are enabled
assert_eq!(
config.brackets().map(|e| e.1).collect::<Vec<_>>(),
@@ -1828,10 +1828,7 @@ fn test_language_scope_at_with_javascript(cx: &mut AppContext) {
let string_config = snapshot
.language_scope_at(text.find("b\"").unwrap())
.unwrap();
assert_eq!(
string_config.line_comment_prefixes().unwrap(),
&[Arc::from("// ")]
);
assert_eq!(string_config.line_comment_prefixes(), &[Arc::from("// ")]);
// Second bracket pair is disabled
assert_eq!(
string_config.brackets().map(|e| e.1).collect::<Vec<_>>(),
@@ -1842,7 +1839,7 @@ fn test_language_scope_at_with_javascript(cx: &mut AppContext) {
let element_config = snapshot
.language_scope_at(text.find("<F>").unwrap())
.unwrap();
assert_eq!(element_config.line_comment_prefixes(), None);
assert_eq!(element_config.line_comment_prefixes(), &[]);
assert_eq!(
element_config.block_comment_delimiters(),
Some((&"{/*".into(), &"*/}".into()))
@@ -1856,10 +1853,7 @@ fn test_language_scope_at_with_javascript(cx: &mut AppContext) {
let tag_config = snapshot
.language_scope_at(text.find(" d=").unwrap() + 1)
.unwrap();
assert_eq!(
tag_config.line_comment_prefixes().unwrap(),
&[Arc::from("// ")]
);
assert_eq!(tag_config.line_comment_prefixes(), &[Arc::from("// ")]);
assert_eq!(
tag_config.brackets().map(|e| e.1).collect::<Vec<_>>(),
&[true, true]
@@ -1870,9 +1864,7 @@ fn test_language_scope_at_with_javascript(cx: &mut AppContext) {
.language_scope_at(text.find('{').unwrap() + 1)
.unwrap();
assert_eq!(
expression_in_element_config
.line_comment_prefixes()
.unwrap(),
expression_in_element_config.line_comment_prefixes(),
&[Arc::from("// ")]
);
assert_eq!(
@@ -1988,17 +1980,14 @@ fn test_language_scope_at_with_combined_injections(cx: &mut AppContext) {
let snapshot = buffer.snapshot();
let html_config = snapshot.language_scope_at(Point::new(2, 4)).unwrap();
assert_eq!(html_config.line_comment_prefixes(), Some(&vec![]));
assert_eq!(html_config.line_comment_prefixes(), &[]);
assert_eq!(
html_config.block_comment_delimiters(),
Some((&"<!--".into(), &"-->".into()))
);
let ruby_config = snapshot.language_scope_at(Point::new(3, 12)).unwrap();
assert_eq!(
ruby_config.line_comment_prefixes().unwrap(),
&[Arc::from("# ")]
);
assert_eq!(ruby_config.line_comment_prefixes(), &[Arc::from("# ")]);
assert_eq!(ruby_config.block_comment_delimiters(), None);
buffer
crates/language/src/language.rs
+18 -19
View File
@@ -20,6 +20,7 @@ mod task_context;
mod buffer_tests;
pub mod markdown;
use crate::language_settings::SoftWrap;
use anyhow::{anyhow, Context, Result};
use async_trait::async_trait;
use collections::{HashMap, HashSet};
@@ -41,12 +42,11 @@ use smol::future::FutureExt as _;
use std::num::NonZeroU32;
use std::{
any::Any,
cell::RefCell,
ffi::OsStr,
fmt::Debug,
hash::Hash,
mem,
ops::Range,
ops::{DerefMut, Range},
path::{Path, PathBuf},
pin::Pin,
str,
@@ -74,8 +74,6 @@ pub use syntax_map::{OwnedSyntaxLayer, SyntaxLayer};
pub use text::LineEnding;
pub use tree_sitter::{Node, Parser, Tree, TreeCursor};
use crate::language_settings::SoftWrap;
/// Initializes the `language` crate.
///
/// This should be called before making use of items from the create.
@@ -83,29 +81,30 @@ pub fn init(cx: &mut AppContext) {
language_settings::init(cx);
}
thread_local! {
static PARSER: RefCell<Parser> = {
let mut parser = Parser::new();
parser.set_wasm_store(WasmStore::new(WASM_ENGINE.clone()).unwrap()).unwrap();
RefCell::new(parser)
};
}
static QUERY_CURSORS: Mutex<Vec<QueryCursor>> = Mutex::new(vec![]);
static PARSERS: Mutex<Vec<Parser>> = Mutex::new(vec![]);
pub fn with_parser<F, R>(func: F) -> R
where
F: FnOnce(&mut Parser) -> R,
{
PARSER.with(|parser| {
let mut parser = parser.borrow_mut();
func(&mut parser)
})
let mut parser = PARSERS.lock().pop().unwrap_or_else(|| {
let mut parser = Parser::new();
parser
.set_wasm_store(WasmStore::new(WASM_ENGINE.clone()).unwrap())
.unwrap();
parser
});
parser.set_included_ranges(&[]).unwrap();
let result = func(&mut parser);
PARSERS.lock().push(parser);
result
}
pub fn with_query_cursor<F, R>(func: F) -> R
where
F: FnOnce(&mut QueryCursor) -> R,
{
use std::ops::DerefMut;
let mut cursor = QueryCursorHandle::new();
func(cursor.deref_mut())
}
@@ -1340,11 +1339,12 @@ impl LanguageScope {
/// Returns line prefix that is inserted in e.g. line continuations or
/// in `toggle comments` action.
pub fn line_comment_prefixes(&self) -> Option<&Vec<Arc<str>>> {
pub fn line_comment_prefixes(&self) -> &[Arc<str>] {
Override::as_option(
self.config_override().map(|o| &o.line_comments),
Some(&self.language.config.line_comments),
)
.map_or(&[] as &[_], |e| e.as_slice())
}
pub fn block_comment_delimiters(&self) -> Option<(&Arc<str>, &Arc<str>)> {
@@ -1445,8 +1445,7 @@ impl Grammar {
}
fn parse_text(&self, text: &Rope, old_tree: Option<Tree>) -> Tree {
PARSER.with(|parser| {
let mut parser = parser.borrow_mut();
with_parser(|parser| {
parser
.set_language(&self.ts_language)
.expect("incompatible grammar");
crates/language/src/language_registry.rs
+3 -4
View File
@@ -3,8 +3,8 @@ use crate::{
all_language_settings, AllLanguageSettingsContent, LanguageSettingsContent,
},
task_context::ContextProvider,
CachedLspAdapter, File, Language, LanguageConfig, LanguageId, LanguageMatcher,
LanguageServerName, LspAdapter, LspAdapterDelegate, PARSER, PLAIN_TEXT,
with_parser, CachedLspAdapter, File, Language, LanguageConfig, LanguageId, LanguageMatcher,
LanguageServerName, LspAdapter, LspAdapterDelegate, PLAIN_TEXT,
};
use anyhow::{anyhow, Context as _, Result};
use collections::{hash_map, HashMap};
@@ -668,8 +668,7 @@ impl LanguageRegistry {
.file_stem()
.and_then(OsStr::to_str)
.ok_or_else(|| anyhow!("invalid grammar filename"))?;
anyhow::Ok(PARSER.with(|parser| {
let mut parser = parser.borrow_mut();
anyhow::Ok(with_parser(|parser| {
let mut store = parser.take_wasm_store().unwrap();
let grammar = store.load_language(&grammar_name, &wasm_bytes);
parser.set_wasm_store(store).unwrap();
crates/language/src/syntax_map.rs
+4 -8
View File
@@ -1,10 +1,11 @@
#[cfg(test)]
mod syntax_map_tests;
use crate::{Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry};
use crate::{
with_parser, Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry, QUERY_CURSORS,
};
use collections::HashMap;
use futures::FutureExt;
use parking_lot::Mutex;
use std::{
borrow::Cow,
cmp::{self, Ordering, Reverse},
@@ -17,10 +18,6 @@ use sum_tree::{Bias, SeekTarget, SumTree};
use text::{Anchor, BufferSnapshot, OffsetRangeExt, Point, Rope, ToOffset, ToPoint};
use tree_sitter::{Node, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree};
use super::PARSER;
static QUERY_CURSORS: Mutex<Vec<QueryCursor>> = Mutex::new(vec![]);
#[derive(Default)]
pub struct SyntaxMap {
snapshot: SyntaxSnapshot,
@@ -1177,8 +1174,7 @@ fn parse_text(
ranges: Vec<tree_sitter::Range>,
old_tree: Option<Tree>,
) -> anyhow::Result<Tree> {
PARSER.with(|parser| {
let mut parser = parser.borrow_mut();
with_parser(|parser| {
let mut chunks = text.chunks_in_range(start_byte..text.len());
parser.set_included_ranges(&ranges)?;
parser.set_language(&grammar.ts_language)?;