oak-team/oak-gpui
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bump tree-sitter and related core language parser libraries (#14986)

Browse Source 
Closes https://github.com/zed-industries/zed/issues/4565

To fix issues with code blocks' parsing in Markdown, a
tree-sitter-markdown library update is needed.
But `tree_sitter::language` is used in many places within core Zed,
which forced more library updates.

Release Notes:

- Updated tree-sitter parsers for core languages

---------

Co-authored-by: Max Brunsfeld <max@zed.dev>
Co-authored-by: Piotr Osiewicz <piotr@zed.dev>
This commit is contained in:
Kirill Bulatov
2024-07-24 23:38:21 +03:00
committed by GitHub
co-authored by Max Brunsfeld Piotr Osiewicz
parent fd4a4127eb
commit 596ee58be8
24 changed files with 282 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/collab_ui/Cargo.toml
+1 -1
View File
@@ -78,7 +78,7 @@ pretty_assertions.workspace = true
project = { workspace = true, features = ["test-support"] }
rpc = { workspace = true, features = ["test-support"] }
settings = { workspace = true, features = ["test-support"] }
tree-sitter-markdown.workspace = true
tree-sitter-md.workspace = true
util = { workspace = true, features = ["test-support"] }
http_client = { workspace = true, features = ["test-support"] }
workspace = { workspace = true, features = ["test-support"] }
crates/editor/src/editor_tests.rs
+112 -72
View File
@@ -4716,12 +4716,13 @@ async fn test_select_larger_smaller_syntax_node(cx: &mut gpui::TestAppContext) {
let buffer = cx.new_model(|cx| Buffer::local(text, cx).with_language(language, cx));
let buffer = cx.new_model(|cx| MultiBuffer::singleton(buffer, cx));
let (view, cx) = cx.add_window_view(|cx| build_editor(buffer, cx));
let (editor, cx) = cx.add_window_view(|cx| build_editor(buffer, cx));
view.condition::<crate::EditorEvent>(&cx, |view, cx| !view.buffer.read(cx).is_parsing(cx))
editor
.condition::<crate::EditorEvent>(&cx, |view, cx| !view.buffer.read(cx).is_parsing(cx))
.await;
_ = view.update(cx, |view, cx| {
editor.update(cx, |view, cx| {
view.change_selections(None, cx, |s| {
s.select_display_ranges([
DisplayPoint::new(DisplayRow(0), 25)..DisplayPoint::new(DisplayRow(0), 25),
@@ -4731,94 +4732,126 @@ async fn test_select_larger_smaller_syntax_node(cx: &mut gpui::TestAppContext) {
});
view.select_larger_syntax_node(&SelectLargerSyntaxNode, cx);
});
assert_eq!(
view.update(cx, |view, cx| { view.selections.display_ranges(cx) }),
&[
DisplayPoint::new(DisplayRow(0), 23)..DisplayPoint::new(DisplayRow(0), 27),
DisplayPoint::new(DisplayRow(2), 35)..DisplayPoint::new(DisplayRow(2), 7),
DisplayPoint::new(DisplayRow(3), 15)..DisplayPoint::new(DisplayRow(3), 21),
]
);
editor.update(cx, |editor, cx| {
assert_text_with_selections(
editor,
indoc! {r#"
use mod1::mod2::{mod3, «mod4ˇ»};
_ = view.update(cx, |view, cx| {
fn fn_1«ˇ(param1: bool, param2: &str)» {
let var1 = "«textˇ»";
}
"#},
cx,
);
});
editor.update(cx, |view, cx| {
view.select_larger_syntax_node(&SelectLargerSyntaxNode, cx);
});
editor.update(cx, |editor, cx| {
assert_text_with_selections(
editor,
indoc! {r#"
use mod1::mod2::«{mod3, mod4}ˇ»;
«ˇfn fn_1(param1: bool, param2: &str) {
let var1 = "text";
}»
"#},
cx,
);
});
editor.update(cx, |view, cx| {
view.select_larger_syntax_node(&SelectLargerSyntaxNode, cx);
});
assert_eq!(
view.update(cx, |view, cx| view.selections.display_ranges(cx)),
&[
DisplayPoint::new(DisplayRow(0), 16)..DisplayPoint::new(DisplayRow(0), 28),
DisplayPoint::new(DisplayRow(4), 1)..DisplayPoint::new(DisplayRow(2), 0),
]
);
_ = view.update(cx, |view, cx| {
view.select_larger_syntax_node(&SelectLargerSyntaxNode, cx);
});
assert_eq!(
view.update(cx, |view, cx| view.selections.display_ranges(cx)),
editor.update(cx, |view, cx| view.selections.display_ranges(cx)),
&[DisplayPoint::new(DisplayRow(5), 0)..DisplayPoint::new(DisplayRow(0), 0)]
);
// Trying to expand the selected syntax node one more time has no effect.
_ = view.update(cx, |view, cx| {
editor.update(cx, |view, cx| {
view.select_larger_syntax_node(&SelectLargerSyntaxNode, cx);
});
assert_eq!(
view.update(cx, |view, cx| view.selections.display_ranges(cx)),
editor.update(cx, |view, cx| view.selections.display_ranges(cx)),
&[DisplayPoint::new(DisplayRow(5), 0)..DisplayPoint::new(DisplayRow(0), 0)]
);
_ = view.update(cx, |view, cx| {
editor.update(cx, |view, cx| {
view.select_smaller_syntax_node(&SelectSmallerSyntaxNode, cx);
});
assert_eq!(
view.update(cx, |view, cx| view.selections.display_ranges(cx)),
&[
DisplayPoint::new(DisplayRow(0), 16)..DisplayPoint::new(DisplayRow(0), 28),
DisplayPoint::new(DisplayRow(4), 1)..DisplayPoint::new(DisplayRow(2), 0),
]
);
editor.update(cx, |editor, cx| {
assert_text_with_selections(
editor,
indoc! {r#"
use mod1::mod2::«{mod3, mod4}ˇ»;
_ = view.update(cx, |view, cx| {
view.select_smaller_syntax_node(&SelectSmallerSyntaxNode, cx);
«ˇfn fn_1(param1: bool, param2: &str) {
let var1 = "text";
}»
"#},
cx,
);
});
assert_eq!(
view.update(cx, |view, cx| view.selections.display_ranges(cx)),
&[
DisplayPoint::new(DisplayRow(0), 23)..DisplayPoint::new(DisplayRow(0), 27),
DisplayPoint::new(DisplayRow(2), 35)..DisplayPoint::new(DisplayRow(2), 7),
DisplayPoint::new(DisplayRow(3), 15)..DisplayPoint::new(DisplayRow(3), 21),
]
);
_ = view.update(cx, |view, cx| {
editor.update(cx, |view, cx| {
view.select_smaller_syntax_node(&SelectSmallerSyntaxNode, cx);
});
assert_eq!(
view.update(cx, |view, cx| view.selections.display_ranges(cx)),
&[
DisplayPoint::new(DisplayRow(0), 25)..DisplayPoint::new(DisplayRow(0), 25),
DisplayPoint::new(DisplayRow(2), 24)..DisplayPoint::new(DisplayRow(2), 12),
DisplayPoint::new(DisplayRow(3), 18)..DisplayPoint::new(DisplayRow(3), 18),
]
);
editor.update(cx, |editor, cx| {
assert_text_with_selections(
editor,
indoc! {r#"
use mod1::mod2::{mod3, «mod4ˇ»};
fn fn_1«ˇ(param1: bool, param2: &str)» {
let var1 = "«textˇ»";
}
"#},
cx,
);
});
editor.update(cx, |view, cx| {
view.select_smaller_syntax_node(&SelectSmallerSyntaxNode, cx);
});
editor.update(cx, |editor, cx| {
assert_text_with_selections(
editor,
indoc! {r#"
use mod1::mod2::{mod3, mo«ˇ»d4};
fn fn_1(para«ˇm1: bool, pa»ram2: &str) {
let var1 = "te«ˇ»xt";
}
"#},
cx,
);
});
// Trying to shrink the selected syntax node one more time has no effect.
_ = view.update(cx, |view, cx| {
editor.update(cx, |view, cx| {
view.select_smaller_syntax_node(&SelectSmallerSyntaxNode, cx);
});
assert_eq!(
view.update(cx, |view, cx| view.selections.display_ranges(cx)),
&[
DisplayPoint::new(DisplayRow(0), 25)..DisplayPoint::new(DisplayRow(0), 25),
DisplayPoint::new(DisplayRow(2), 24)..DisplayPoint::new(DisplayRow(2), 12),
DisplayPoint::new(DisplayRow(3), 18)..DisplayPoint::new(DisplayRow(3), 18),
]
);
editor.update(cx, |editor, cx| {
assert_text_with_selections(
editor,
indoc! {r#"
use mod1::mod2::{mod3, mo«ˇ»d4};
fn fn_1(para«ˇm1: bool, pa»ram2: &str) {
let var1 = "te«ˇ»xt";
}
"#},
cx,
);
});
// Ensure that we keep expanding the selection if the larger selection starts or ends within
// a fold.
_ = view.update(cx, |view, cx| {
editor.update(cx, |view, cx| {
view.fold_ranges(
vec![
(
@@ -4835,14 +4868,19 @@ async fn test_select_larger_smaller_syntax_node(cx: &mut gpui::TestAppContext) {
);
view.select_larger_syntax_node(&SelectLargerSyntaxNode, cx);
});
assert_eq!(
view.update(cx, |view, cx| view.selections.display_ranges(cx)),
&[
DisplayPoint::new(DisplayRow(0), 16)..DisplayPoint::new(DisplayRow(0), 28),
DisplayPoint::new(DisplayRow(2), 35)..DisplayPoint::new(DisplayRow(2), 7),
DisplayPoint::new(DisplayRow(3), 4)..DisplayPoint::new(DisplayRow(3), 23),
]
);
editor.update(cx, |editor, cx| {
assert_text_with_selections(
editor,
indoc! {r#"
use mod1::mod2::«{mod3, mod4}ˇ»;
fn fn_1«ˇ(param1: bool, param2: &str)» {
«let var1 = "text";ˇ»
}
"#},
cx,
);
});
}
#[gpui::test]
@@ -8173,11 +8211,13 @@ async fn test_toggle_block_comment(cx: &mut gpui::TestAppContext) {
);
cx.executor().run_until_parked();
cx.update_editor(|editor, cx| editor.toggle_comments(&ToggleComments::default(), cx));
// TODO this is how it actually worked in Zed Stable, which is not very ergonomic.
// Uncommenting and commenting from this position brings in even more wrong artifacts.
cx.assert_editor_state(
&r#"
<!-- ˇ<script> -->
// ˇvar x = new Y();
<!-- ˇ</script> -->
// ˇ</script>
"#
.unindent(),
);
crates/editor/src/test.rs
+1
View File
@@ -62,6 +62,7 @@ pub fn select_ranges(editor: &mut Editor, marked_text: &str, cx: &mut ViewContex
editor.change_selections(None, cx, |s| s.select_ranges(text_ranges));
}
#[track_caller]
pub fn assert_text_with_selections(
editor: &mut Editor,
marked_text: &str,
crates/language/Cargo.toml
+1 -1
View File
@@ -78,7 +78,7 @@ tree-sitter-embedded-template.workspace = true
tree-sitter-heex.workspace = true
tree-sitter-html.workspace = true
tree-sitter-json.workspace = true
tree-sitter-markdown.workspace = true
tree-sitter-md.workspace = true
tree-sitter-ruby.workspace = true
tree-sitter-rust.workspace = true
tree-sitter-typescript.workspace = true
crates/language/src/buffer_tests.rs
+7 -5
View File
@@ -1858,11 +1858,13 @@ fn test_language_scope_at_with_javascript(cx: &mut AppContext) {
let element_config = snapshot
.language_scope_at(text.find("<F>").unwrap())
.unwrap();
assert_eq!(element_config.line_comment_prefixes(), &[]);
assert_eq!(
element_config.block_comment_delimiters(),
Some((&"{/*".into(), &"*/}".into()))
);
// TODO nested blocks after newlines are captured with all whitespaces
// https://github.com/tree-sitter/tree-sitter-typescript/issues/306
// assert_eq!(element_config.line_comment_prefixes(), &[]);
// assert_eq!(
// element_config.block_comment_delimiters(),
// Some((&"{/*".into(), &"*/}".into()))
// );
assert_eq!(
element_config.brackets().map(|e| e.1).collect::<Vec<_>>(),
&[true, true]
crates/language/src/syntax_map/syntax_map_tests.rs
+29 -8
View File
@@ -180,7 +180,9 @@ fn test_syntax_map_layers_for_range(cx: &mut AppContext) {
fn test_dynamic_language_injection(cx: &mut AppContext) {
let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone()));
let markdown = Arc::new(markdown_lang());
let markdown_inline = Arc::new(markdown_inline_lang());
registry.add(markdown.clone());
registry.add(markdown_inline.clone());
registry.add(Arc::new(rust_lang()));
registry.add(Arc::new(ruby_lang()));
@@ -200,12 +202,14 @@ fn test_dynamic_language_injection(cx: &mut AppContext) {
let mut syntax_map = SyntaxMap::new();
syntax_map.set_language_registry(registry.clone());
syntax_map.reparse(markdown.clone(), &buffer);
syntax_map.reparse(markdown_inline.clone(), &buffer);
assert_layers_for_range(
&syntax_map,
&buffer,
Point::new(3, 0)..Point::new(3, 0),
&[
"...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
"(document (section (paragraph (inline)) (fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (block_continuation) (code_fence_content (block_continuation)) (fenced_code_block_delimiter))))",
"(inline (code_span (code_span_delimiter) (code_span_delimiter)))",
"...(function_item name: (identifier) parameters: (parameters) body: (block)...",
],
);
@@ -215,12 +219,14 @@ fn test_dynamic_language_injection(cx: &mut AppContext) {
buffer.edit([(macro_name_range, "ruby")]);
syntax_map.interpolate(&buffer);
syntax_map.reparse(markdown.clone(), &buffer);
syntax_map.reparse(markdown_inline.clone(), &buffer);
assert_layers_for_range(
&syntax_map,
&buffer,
Point::new(3, 0)..Point::new(3, 0),
&[
"...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
"(document (section (paragraph (inline)) (fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (block_continuation) (code_fence_content (block_continuation)) (fenced_code_block_delimiter))))",
"(inline (code_span (code_span_delimiter) (code_span_delimiter)))",
"...(call method: (identifier) arguments: (argument_list (call method: (identifier) arguments: (argument_list) block: (block)...",
],
);
@@ -230,25 +236,29 @@ fn test_dynamic_language_injection(cx: &mut AppContext) {
buffer.edit([(macro_name_range, "html")]);
syntax_map.interpolate(&buffer);
syntax_map.reparse(markdown.clone(), &buffer);
syntax_map.reparse(markdown_inline.clone(), &buffer);
assert_layers_for_range(
&syntax_map,
&buffer,
Point::new(3, 0)..Point::new(3, 0),
&[
"...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter..."
"(document (section (paragraph (inline)) (fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (block_continuation) (code_fence_content (block_continuation)) (fenced_code_block_delimiter))))",
"(inline (code_span (code_span_delimiter) (code_span_delimiter)))",
],
);
assert!(syntax_map.contains_unknown_injections());
registry.add(Arc::new(html_lang()));
syntax_map.reparse(markdown.clone(), &buffer);
syntax_map.reparse(markdown_inline.clone(), &buffer);
assert_layers_for_range(
&syntax_map,
&buffer,
Point::new(3, 0)..Point::new(3, 0),
&[
"...(fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (code_fence_content) (fenced_code_block_delimiter...",
"(fragment (text))",
"(document (section (paragraph (inline)) (fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (block_continuation) (code_fence_content (block_continuation)) (fenced_code_block_delimiter))))",
"(inline (code_span (code_span_delimiter) (code_span_delimiter)))",
"(document (text))",
],
);
assert!(!syntax_map.contains_unknown_injections());
@@ -769,9 +779,9 @@ fn test_empty_combined_injections_inside_injections(cx: &mut AppContext) {
&buffer,
Point::new(0, 0)..Point::new(5, 0),
&[
"...(paragraph)...",
"(document (section (fenced_code_block (fenced_code_block_delimiter) (info_string (language)) (block_continuation) (code_fence_content (block_continuation)) (fenced_code_block_delimiter)) (paragraph (inline))))",
"(template...",
"(fragment...",
"(document (text))",
// The ruby syntax tree should be empty, since there are
// no interpolations in the ERB template.
"(program)",
@@ -1055,6 +1065,7 @@ fn test_edit_sequence(
registry.add(Arc::new(html_lang()));
registry.add(Arc::new(erb_lang()));
registry.add(Arc::new(markdown_lang()));
registry.add(Arc::new(markdown_inline_lang()));
let language = registry
.language_for_name(language_name)
@@ -1228,7 +1239,7 @@ fn markdown_lang() -> Language {
},
..Default::default()
},
Some(tree_sitter_markdown::language()),
Some(tree_sitter_md::language()),
)
.with_injection_query(
r#"
@@ -1241,6 +1252,16 @@ fn markdown_lang() -> Language {
.unwrap()
}
fn markdown_inline_lang() -> Language {
Language::new(
LanguageConfig {
name: "Markdown-Inline".into(),
..LanguageConfig::default()
},
Some(tree_sitter_md::inline_language()),
)
}
fn elixir_lang() -> Language {
Language::new(
LanguageConfig {
crates/languages/Cargo.toml
+3 -3
View File
@@ -42,12 +42,12 @@ tree-sitter-c.workspace = true
tree-sitter-cpp.workspace = true
tree-sitter-css.workspace = true
tree-sitter-go.workspace = true
tree-sitter-gomod.workspace = true
tree-sitter-go-mod.workspace = true
tree-sitter-gowork.workspace = true
tree-sitter-jsdoc.workspace = true
tree-sitter-json.workspace = true
tree-sitter-markdown.workspace = true
tree-sitter-proto.workspace = true
tree-sitter-md.workspace = true
protols-tree-sitter-proto.workspace = true
tree-sitter-python.workspace = true
tree-sitter-regex.workspace = true
tree-sitter-rust.workspace = true
crates/languages/src/cpp/highlights.scm
+1 -1
View File
@@ -87,12 +87,12 @@
"typename"
"union"
"using"
"virtual"
"volatile"
"while"
(primitive_type)
(sized_type_specifier)
(type_qualifier)
(virtual)
] @keyword
[
crates/languages/src/go/outline.scm
+1 -1
View File
@@ -34,7 +34,7 @@
(var_spec
name: (identifier) @name) @item))
(method_spec
(method_elem
name: (_) @name
parameters: (parameter_list
"(" @context
crates/languages/src/lib.rs
+5 -3
View File
@@ -41,13 +41,14 @@ pub fn init(
("cpp", tree_sitter_cpp::language()),
("css", tree_sitter_css::language()),
("go", tree_sitter_go::language()),
("gomod", tree_sitter_gomod::language()),
("gomod", tree_sitter_go_mod::language()),
("gowork", tree_sitter_gowork::language()),
("jsdoc", tree_sitter_jsdoc::language()),
("json", tree_sitter_json::language()),
("jsonc", tree_sitter_json::language()),
("markdown", tree_sitter_markdown::language()),
("proto", tree_sitter_proto::language()),
("markdown", tree_sitter_md::language()),
("markdown-inline", tree_sitter_md::inline_language()),
("proto", protols_tree_sitter_proto::language()),
("python", tree_sitter_python::language()),
("regex", tree_sitter_regex::language()),
("rust", tree_sitter_rust::language()),
@@ -136,6 +137,7 @@ pub fn init(
json_task_context()
);
language!("markdown");
language!("markdown-inline");
language!(
"python",
vec![Arc::new(python::PythonLspAdapter::new(
crates/languages/src/markdown-inline/config.toml
+15
View File
@@ -0,0 +1,15 @@
name = "Markdown-Inline"
grammar = "markdown-inline"
path_suffixes = []
word_characters = ["-"]
brackets = [
{ start = "{", end = "}", close = true, newline = true },
{ start = "[", end = "]", close = true, newline = true },
{ start = "(", end = ")", close = true, newline = true },
{ start = "<", end = ">", close = true, newline = true },
{ start = "\"", end = "\"", close = false, newline = false },
{ start = "'", end = "'", close = false, newline = false },
{ start = "`", end = "`", close = false, newline = false },
]
tab_size = 2
crates/languages/src/markdown-inline/highlights.scm
+6
View File
@@ -0,0 +1,6 @@
(emphasis) @emphasis
(strong_emphasis) @emphasis.strong
(code_span) @text.literal
(link_text) @link_text
(link_label) @link_text
(link_destination) @link_uri
crates/languages/src/markdown/highlights.scm
-8
View File
@@ -1,6 +1,3 @@
(emphasis) @emphasis
(strong_emphasis) @emphasis.strong
[
(atx_heading)
(setext_heading)
@@ -14,11 +11,6 @@
(list_marker_parenthesis)
] @punctuation.list_marker
(code_span) @text.literal
(fenced_code_block
(info_string
(language) @text.literal))
(link_destination) @link_uri
(link_text) @link_text
crates/languages/src/markdown/injections.scm
+3
View File
@@ -2,3 +2,6 @@
(info_string
(language) @language)
(code_fence_content) @content)
((inline) @content
(#set! "language" "markdown-inline"))
crates/settings/Cargo.toml
+1 -1
View File
@@ -31,7 +31,7 @@ serde_derive.workspace = true
serde_json.workspace = true
serde_json_lenient.workspace = true
smallvec.workspace = true
tree-sitter-json = "*"
tree-sitter-json.workspace = true
tree-sitter.workspace = true
util.workspace = true
crates/title_bar/Cargo.toml
+1 -1
View File
@@ -67,6 +67,6 @@ pretty_assertions.workspace = true
project = { workspace = true, features = ["test-support"] }
rpc = { workspace = true, features = ["test-support"] }
settings = { workspace = true, features = ["test-support"] }
tree-sitter-markdown.workspace = true
tree-sitter-md.workspace = true
util = { workspace = true, features = ["test-support"] }
workspace = { workspace = true, features = ["test-support"] }
crates/zed/Cargo.toml
+1 -1
View File
@@ -122,7 +122,7 @@ editor = { workspace = true, features = ["test-support"] }
gpui = { workspace = true, features = ["test-support"] }
language = { workspace = true, features = ["test-support"] }
project = { workspace = true, features = ["test-support"] }
tree-sitter-markdown.workspace = true
tree-sitter-md.workspace = true
tree-sitter-rust.workspace = true
workspace = { workspace = true, features = ["test-support"] }
crates/zed/src/zed.rs
+2 -1
View File
@@ -3326,6 +3326,7 @@ mod tests {
#[gpui::test]
async fn test_bundled_languages(cx: &mut TestAppContext) {
env_logger::builder().is_test(true).try_init().ok();
let settings = cx.update(|cx| SettingsStore::test(cx));
cx.set_global(settings);
let languages = LanguageRegistry::test(cx.executor());
@@ -3490,7 +3491,7 @@ mod tests {
},
..Default::default()
},
Some(tree_sitter_markdown::language()),
Some(tree_sitter_md::language()),
))
}