From a5dd8dd0a9c14e85bc95a8b84989366f5ff590fa Mon Sep 17 00:00:00 2001
From: KCaverly <kyle@zed.dev>
Date: Mon, 31 Jul 2023 10:02:28 -0400
Subject: [PATCH 1/6] add lua embedding query for semantic search

---
 Cargo.lock                                    |  39 +++----
 crates/semantic_index/Cargo.toml              |   1 +
 .../src/semantic_index_tests.rs               | 102 ++++++++++++++++++
 crates/zed/src/languages/lua/config.toml      |   1 +
 crates/zed/src/languages/lua/embedding.scm    |  10 ++
 5 files changed, 134 insertions(+), 19 deletions(-)
 create mode 100644 crates/zed/src/languages/lua/embedding.scm

diff --git a/Cargo.lock b/Cargo.lock
index 7c6a213d0d..6c558cbb09 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2042,9 +2042,9 @@ dependencies = [
 
 [[package]]
 name = "curl-sys"
-version = "0.4.64+curl-8.2.0"
+version = "0.4.65+curl-8.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f96069f0b1cb1241c838740659a771ef143363f52772a9ce1bd9c04c75eee0dc"
+checksum = "961ba061c9ef2fe34bbd12b807152d96f0badd2bebe7b90ce6c8c8b7572a0986"
 dependencies = [
  "cc",
  "libc",
@@ -3033,9 +3033,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
 [[package]]
 name = "globset"
-version = "0.4.11"
+version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1391ab1f92ffcc08911957149833e682aa3fe252b9f45f966d2ef972274c97df"
+checksum = "aca8bbd8e0707c1887a8bbb7e6b40e228f251ff5d62c8220a4a7a53c73aff006"
 dependencies = [
  "aho-corasick 1.0.2",
  "bstr",
@@ -6688,6 +6688,7 @@ dependencies = [
  "tree-sitter-cpp 0.20.2",
  "tree-sitter-elixir 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tree-sitter-json 0.19.0",
+ "tree-sitter-lua",
  "tree-sitter-rust",
  "tree-sitter-toml 0.20.0",
  "tree-sitter-typescript 0.20.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -6722,18 +6723,18 @@ checksum = "5a9f47faea3cad316faa914d013d24f471cd90bfca1a0c70f05a3f42c6441e99"
 
 [[package]]
 name = "serde"
-version = "1.0.175"
+version = "1.0.177"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b"
+checksum = "63ba2516aa6bf82e0b19ca8b50019d52df58455d3cf9bdaf6315225fdd0c560a"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.175"
+version = "1.0.177"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4"
+checksum = "401797fe7833d72109fedec6bfcbe67c0eed9b99772f26eb8afd261f0abc6fd3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -6762,9 +6763,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.103"
+version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b"
+checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
 dependencies = [
  "indexmap 2.0.0",
  "itoa 1.0.9",
@@ -6786,9 +6787,9 @@ dependencies = [
 
 [[package]]
 name = "serde_repr"
-version = "0.1.15"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e168eaaf71e8f9bd6037feb05190485708e019f4fd87d161b3c0a0d37daf85e5"
+checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -9031,9 +9032,9 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
 
 [[package]]
 name = "wasm-encoder"
-version = "0.31.0"
+version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06a3d1b4a575ffb873679402b2aedb3117555eb65c27b1b86c8a91e574bc2a2a"
+checksum = "41763f20eafed1399fff1afb466496d3a959f58241436cfdc17e3f5ca954de16"
 dependencies = [
  "leb128",
 ]
@@ -9255,9 +9256,9 @@ dependencies = [
 
 [[package]]
 name = "wast"
-version = "62.0.0"
+version = "62.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7f7ee878019d69436895f019b65f62c33da63595d8e857cbdc87c13ecb29a32"
+checksum = "b8ae06f09dbe377b889fbd620ff8fa21e1d49d1d9d364983c0cdbf9870cb9f1f"
 dependencies = [
  "leb128",
  "memchr",
@@ -9267,11 +9268,11 @@ dependencies = [
 
 [[package]]
 name = "wat"
-version = "1.0.68"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "295572bf24aa5b685a971a83ad3e8b6e684aaad8a9be24bc7bf59bed84cc1c08"
+checksum = "842e15861d203fb4a96d314b0751cdeaf0f6f8b35e8d81d2953af2af5e44e637"
 dependencies = [
- "wast 62.0.0",
+ "wast 62.0.1",
 ]
 
 [[package]]
diff --git a/crates/semantic_index/Cargo.toml b/crates/semantic_index/Cargo.toml
index a1f126bfb8..942f61d298 100644
--- a/crates/semantic_index/Cargo.toml
+++ b/crates/semantic_index/Cargo.toml
@@ -60,3 +60,4 @@ tree-sitter-rust = "*"
 tree-sitter-toml = "*"
 tree-sitter-cpp = "*"
 tree-sitter-elixir = "*"
+tree-sitter-lua = "*"
diff --git a/crates/semantic_index/src/semantic_index_tests.rs b/crates/semantic_index/src/semantic_index_tests.rs
index acf5a9d72b..4eedade69d 100644
--- a/crates/semantic_index/src/semantic_index_tests.rs
+++ b/crates/semantic_index/src/semantic_index_tests.rs
@@ -486,6 +486,79 @@ async fn test_code_context_retrieval_javascript() {
     )
 }
 
+#[gpui::test]
+async fn test_code_context_retrieval_lua() {
+    let language = lua_lang();
+    let mut retriever = CodeContextRetriever::new();
+
+    let text = r#"
+        -- Creates a new class
+        -- @param baseclass The Baseclass of this class, or nil.
+        -- @return A new class reference.
+        function classes.class(baseclass)
+            -- Create the class definition and metatable.
+            local classdef = {}
+            -- Find the super class, either Object or user-defined.
+            baseclass = baseclass or classes.Object
+            -- If this class definition does not know of a function, it will 'look up' to the Baseclass via the __index of the metatable.
+            setmetatable(classdef, { __index = baseclass })
+            -- All class instances have a reference to the class object.
+            classdef.class = classdef
+            --- Recursivly allocates the inheritance tree of the instance.
+            -- @param mastertable The 'root' of the inheritance tree.
+            -- @return Returns the instance with the allocated inheritance tree.
+            function classdef.alloc(mastertable)
+                -- All class instances have a reference to a superclass object.
+                local instance = { super = baseclass.alloc(mastertable) }
+                -- Any functions this instance does not know of will 'look up' to the superclass definition.
+                setmetatable(instance, { __index = classdef, __newindex = mastertable })
+                return instance
+            end
+        end
+        "#.unindent();
+
+    let documents = retriever.parse_file(&text, language.clone()).unwrap();
+
+    assert_documents_eq(
+        &documents,
+        &[
+            (r#"
+                -- Creates a new class
+                -- @param baseclass The Baseclass of this class, or nil.
+                -- @return A new class reference.
+                function classes.class(baseclass)
+                    -- Create the class definition and metatable.
+                    local classdef = {}
+                    -- Find the super class, either Object or user-defined.
+                    baseclass = baseclass or classes.Object
+                    -- If this class definition does not know of a function, it will 'look up' to the Baseclass via the __index of the metatable.
+                    setmetatable(classdef, { __index = baseclass })
+                    -- All class instances have a reference to the class object.
+                    classdef.class = classdef
+                    --- Recursivly allocates the inheritance tree of the instance.
+                    -- @param mastertable The 'root' of the inheritance tree.
+                    -- @return Returns the instance with the allocated inheritance tree.
+                    function classdef.alloc(mastertable)
+                        --[ ... ]--
+                        --[ ... ]--
+                    end
+                end"#.unindent(),
+            114),
+            (r#"
+            --- Recursivly allocates the inheritance tree of the instance.
+            -- @param mastertable The 'root' of the inheritance tree.
+            -- @return Returns the instance with the allocated inheritance tree.
+            function classdef.alloc(mastertable)
+                -- All class instances have a reference to a superclass object.
+                local instance = { super = baseclass.alloc(mastertable) }
+                -- Any functions this instance does not know of will 'look up' to the superclass definition.
+                setmetatable(instance, { __index = classdef, __newindex = mastertable })
+                return instance
+            end"#.unindent(), 809),
+        ]
+    );
+}
+
 #[gpui::test]
 async fn test_code_context_retrieval_elixir() {
     let language = elixir_lang();
@@ -1084,6 +1157,35 @@ fn cpp_lang() -> Arc<Language> {
     )
 }
 
+fn lua_lang() -> Arc<Language> {
+    Arc::new(
+        Language::new(
+            LanguageConfig {
+                name: "Lua".into(),
+                path_suffixes: vec!["lua".into()],
+                collapsed_placeholder: "--[ ... ]--".to_string(),
+                ..Default::default()
+            },
+            Some(tree_sitter_lua::language()),
+        )
+        .with_embedding_query(
+            r#"
+            (
+                (comment)* @context
+                .
+                (function_declaration
+                    "function" @name
+                    name: (_) @name
+                    (comment)* @collapse
+                    body: (block) @collapse
+                ) @item
+            )
+        "#,
+        )
+        .unwrap(),
+    )
+}
+
 fn elixir_lang() -> Arc<Language> {
     Arc::new(
         Language::new(
diff --git a/crates/zed/src/languages/lua/config.toml b/crates/zed/src/languages/lua/config.toml
index fe44a3d2aa..d3e44edfe9 100644
--- a/crates/zed/src/languages/lua/config.toml
+++ b/crates/zed/src/languages/lua/config.toml
@@ -7,3 +7,4 @@ brackets = [
     { start = "[", end = "]", close = true, newline = true },
     { start = "\"", end = "\"", close = true, newline = false, not_in = ["string"] },
 ]
+collapsed_placeholder = "--[ ... ]--"
diff --git a/crates/zed/src/languages/lua/embedding.scm b/crates/zed/src/languages/lua/embedding.scm
new file mode 100644
index 0000000000..0d1065089f
--- /dev/null
+++ b/crates/zed/src/languages/lua/embedding.scm
@@ -0,0 +1,10 @@
+(
+    (comment)* @context
+    .
+    (function_declaration
+        "function" @name
+        name: (_) @name
+        (comment)* @collapse
+        body: (block) @collapse
+    ) @item
+)

From ca4e21881efc5d0293e545a1059971cab356bc54 Mon Sep 17 00:00:00 2001
From: KCaverly <kyle@zed.dev>
Date: Mon, 31 Jul 2023 10:54:30 -0400
Subject: [PATCH 2/6] add ruby support for semantic search

---
 Cargo.lock                                    |   1 +
 crates/semantic_index/Cargo.toml              |   1 +
 .../src/semantic_index_tests.rs               | 231 ++++++++++++++++++
 crates/zed/src/languages/ruby/config.toml     |   1 +
 crates/zed/src/languages/ruby/embedding.scm   |  22 ++
 5 files changed, 256 insertions(+)
 create mode 100644 crates/zed/src/languages/ruby/embedding.scm

diff --git a/Cargo.lock b/Cargo.lock
index 6c558cbb09..b4a8f13bea 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6689,6 +6689,7 @@ dependencies = [
  "tree-sitter-elixir 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tree-sitter-json 0.19.0",
  "tree-sitter-lua",
+ "tree-sitter-ruby",
  "tree-sitter-rust",
  "tree-sitter-toml 0.20.0",
  "tree-sitter-typescript 0.20.2 (registry+https://github.com/rust-lang/crates.io-index)",
diff --git a/crates/semantic_index/Cargo.toml b/crates/semantic_index/Cargo.toml
index 942f61d298..637f3f4487 100644
--- a/crates/semantic_index/Cargo.toml
+++ b/crates/semantic_index/Cargo.toml
@@ -61,3 +61,4 @@ tree-sitter-toml = "*"
 tree-sitter-cpp = "*"
 tree-sitter-elixir = "*"
 tree-sitter-lua = "*"
+tree-sitter-ruby = "*"
diff --git a/crates/semantic_index/src/semantic_index_tests.rs b/crates/semantic_index/src/semantic_index_tests.rs
index 4eedade69d..58d34649c7 100644
--- a/crates/semantic_index/src/semantic_index_tests.rs
+++ b/crates/semantic_index/src/semantic_index_tests.rs
@@ -827,6 +827,196 @@ async fn test_code_context_retrieval_cpp() {
     );
 }
 
+#[gpui::test]
+async fn test_code_context_retrieval_ruby() {
+    let language = ruby_lang();
+    let mut retriever = CodeContextRetriever::new();
+
+    let text = r#"
+        # This concern is inspired by "sudo mode" on GitHub. It
+        # is a way to re-authenticate a user before allowing them
+        # to see or perform an action.
+        #
+        # Add `before_action :require_challenge!` to actions you
+        # want to protect.
+        #
+        # The user will be shown a page to enter the challenge (which
+        # is either the password, or just the username when no
+        # password exists). Upon passing, there is a grace period
+        # during which no challenge will be asked from the user.
+        #
+        # Accessing challenge-protected resources during the grace
+        # period will refresh the grace period.
+        module ChallengableConcern
+            extend ActiveSupport::Concern
+
+            CHALLENGE_TIMEOUT = 1.hour.freeze
+
+            def require_challenge!
+                return if skip_challenge?
+
+                if challenge_passed_recently?
+                    session[:challenge_passed_at] = Time.now.utc
+                    return
+                end
+
+                @challenge = Form::Challenge.new(return_to: request.url)
+
+                if params.key?(:form_challenge)
+                    if challenge_passed?
+                        session[:challenge_passed_at] = Time.now.utc
+                    else
+                        flash.now[:alert] = I18n.t('challenge.invalid_password')
+                        render_challenge
+                    end
+                else
+                    render_challenge
+                end
+            end
+
+            def challenge_passed?
+                current_user.valid_password?(challenge_params[:current_password])
+            end
+        end
+
+        class Animal
+            include Comparable
+
+            attr_reader :legs
+
+            def initialize(name, legs)
+                @name, @legs = name, legs
+            end
+
+            def <=>(other)
+                legs <=> other.legs
+            end
+        end
+
+        # Singleton method for car object
+        def car.wheels
+            puts "There are four wheels"
+        end"#
+        .unindent();
+
+    let documents = retriever.parse_file(&text, language.clone()).unwrap();
+
+    assert_documents_eq(
+        &documents,
+        &[
+            (
+                r#"
+        # This concern is inspired by "sudo mode" on GitHub. It
+        # is a way to re-authenticate a user before allowing them
+        # to see or perform an action.
+        #
+        # Add `before_action :require_challenge!` to actions you
+        # want to protect.
+        #
+        # The user will be shown a page to enter the challenge (which
+        # is either the password, or just the username when no
+        # password exists). Upon passing, there is a grace period
+        # during which no challenge will be asked from the user.
+        #
+        # Accessing challenge-protected resources during the grace
+        # period will refresh the grace period.
+        module ChallengableConcern
+            extend ActiveSupport::Concern
+
+            CHALLENGE_TIMEOUT = 1.hour.freeze
+
+            def require_challenge!
+                # ...
+            end
+
+            def challenge_passed?
+                # ...
+            end
+        end"#
+                    .unindent(),
+                558,
+            ),
+            (
+                r#"
+            def require_challenge!
+                return if skip_challenge?
+
+                if challenge_passed_recently?
+                    session[:challenge_passed_at] = Time.now.utc
+                    return
+                end
+
+                @challenge = Form::Challenge.new(return_to: request.url)
+
+                if params.key?(:form_challenge)
+                    if challenge_passed?
+                        session[:challenge_passed_at] = Time.now.utc
+                    else
+                        flash.now[:alert] = I18n.t('challenge.invalid_password')
+                        render_challenge
+                    end
+                else
+                    render_challenge
+                end
+            end"#
+                    .unindent(),
+                663,
+            ),
+            (
+                r#"
+                def challenge_passed?
+                    current_user.valid_password?(challenge_params[:current_password])
+                end"#
+                    .unindent(),
+                1254,
+            ),
+            (
+                r#"
+                class Animal
+                    include Comparable
+
+                    attr_reader :legs
+
+                    def initialize(name, legs)
+                        # ...
+                    end
+
+                    def <=>(other)
+                        # ...
+                    end
+                end"#
+                    .unindent(),
+                1363,
+            ),
+            (
+                r#"
+                def initialize(name, legs)
+                    @name, @legs = name, legs
+                end"#
+                    .unindent(),
+                1427,
+            ),
+            (
+                r#"
+                def <=>(other)
+                    legs <=> other.legs
+                end"#
+                    .unindent(),
+                1501,
+            ),
+            (
+                r#"
+                # Singleton method for car object
+                def car.wheels
+                    puts "There are four wheels"
+                end"#
+                    .unindent(),
+                1591,
+            ),
+        ],
+    );
+}
+
 #[gpui::test]
 fn test_dot_product(mut rng: StdRng) {
     assert_eq!(dot(&[1., 0., 0., 0., 0.], &[0., 1., 0., 0., 0.]), 0.);
@@ -1186,6 +1376,47 @@ fn lua_lang() -> Arc<Language> {
     )
 }
 
+fn ruby_lang() -> Arc<Language> {
+    Arc::new(
+        Language::new(
+            LanguageConfig {
+                name: "Ruby".into(),
+                path_suffixes: vec!["rb".into()],
+                collapsed_placeholder: "# ...".to_string(),
+                ..Default::default()
+            },
+            Some(tree_sitter_ruby::language()),
+        )
+        .with_embedding_query(
+            r#"
+            (
+                (comment)* @context
+                .
+                [
+                (module
+                    "module" @name
+                    name: (_) @name)
+                (method
+                    "def" @name
+                    name: (_) @name
+                    body: (body_statement) @collapse)
+                (class
+                    "class" @name
+                    name: (_) @name)
+                (singleton_method
+                    "def" @name
+                    object: (_) @name
+                    "." @name
+                    name: (_) @name
+                    body: (body_statement) @collapse)
+                ] @item
+            )
+            "#,
+        )
+        .unwrap(),
+    )
+}
+
 fn elixir_lang() -> Arc<Language> {
     Arc::new(
         Language::new(
diff --git a/crates/zed/src/languages/ruby/config.toml b/crates/zed/src/languages/ruby/config.toml
index a0b26bff92..6c8c615015 100644
--- a/crates/zed/src/languages/ruby/config.toml
+++ b/crates/zed/src/languages/ruby/config.toml
@@ -10,3 +10,4 @@ brackets = [
   { start = "\"", end = "\"", close = true, newline = false, not_in = ["comment", "string"] },
   { start = "'", end = "'", close = true, newline = false, not_in = ["comment", "string"] },
 ]
+collapsed_placeholder = "# ..."
diff --git a/crates/zed/src/languages/ruby/embedding.scm b/crates/zed/src/languages/ruby/embedding.scm
new file mode 100644
index 0000000000..7a101e6b09
--- /dev/null
+++ b/crates/zed/src/languages/ruby/embedding.scm
@@ -0,0 +1,22 @@
+(
+    (comment)* @context
+    .
+    [
+        (module
+            "module" @name
+            name: (_) @name)
+        (method
+            "def" @name
+            name: (_) @name
+            body: (body_statement) @collapse)
+        (class
+            "class" @name
+            name: (_) @name)
+        (singleton_method
+            "def" @name
+            object: (_) @name
+            "." @name
+            name: (_) @name
+            body: (body_statement) @collapse)
+        ] @item
+    )

From 89edb3d1b534b962f3e4e81c6055b6fd92fef868 Mon Sep 17 00:00:00 2001
From: KCaverly <kyle@zed.dev>
Date: Mon, 31 Jul 2023 11:41:18 -0400
Subject: [PATCH 3/6] fix templating bug for parseable entire files

---
 crates/semantic_index/src/parsing.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/crates/semantic_index/src/parsing.rs b/crates/semantic_index/src/parsing.rs
index c952ef3a4e..677406931e 100644
--- a/crates/semantic_index/src/parsing.rs
+++ b/crates/semantic_index/src/parsing.rs
@@ -59,7 +59,7 @@ impl CodeContextRetriever {
         let document_span = ENTIRE_FILE_TEMPLATE
             .replace("<path>", relative_path.to_string_lossy().as_ref())
             .replace("<language>", language_name.as_ref())
-            .replace("item", &content);
+            .replace("<item>", &content);
 
         Ok(vec![Document {
             range: 0..content.len(),

From 599f6748274ca40e91e593d5d049982b2fbded45 Mon Sep 17 00:00:00 2001
From: KCaverly <kyle@zed.dev>
Date: Mon, 31 Jul 2023 16:36:09 -0400
Subject: [PATCH 4/6] add php support for semantic search

---
 Cargo.lock                                    |  65 ++----
 crates/semantic_index/Cargo.toml              |  17 +-
 crates/semantic_index/src/parsing.rs          |   7 +-
 .../src/semantic_index_tests.rs               | 205 ++++++++++++++++++
 crates/zed/src/languages/php/config.toml      |   1 +
 crates/zed/src/languages/php/embedding.scm    |  36 +++
 crates/zed/src/languages/php/outline.scm      |   7 +-
 7 files changed, 275 insertions(+), 63 deletions(-)
 create mode 100644 crates/zed/src/languages/php/embedding.scm

diff --git a/Cargo.lock b/Cargo.lock
index b4a8f13bea..f6a85aa70c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2341,7 +2341,7 @@ dependencies = [
  "tree-sitter",
  "tree-sitter-html",
  "tree-sitter-rust",
- "tree-sitter-typescript 0.20.2 (git+https://github.com/tree-sitter/tree-sitter-typescript?rev=5d20856f34315b068c41edaee2ac8a100081d259)",
+ "tree-sitter-typescript",
  "unindent",
  "util",
  "workspace",
@@ -3851,7 +3851,7 @@ dependencies = [
  "text",
  "theme",
  "tree-sitter",
- "tree-sitter-elixir 0.1.0 (git+https://github.com/elixir-lang/tree-sitter-elixir?rev=4ba9dab6e2602960d95b2b625f3386c27e08084e)",
+ "tree-sitter-elixir",
  "tree-sitter-embedded-template",
  "tree-sitter-heex",
  "tree-sitter-html",
@@ -3860,7 +3860,7 @@ dependencies = [
  "tree-sitter-python",
  "tree-sitter-ruby",
  "tree-sitter-rust",
- "tree-sitter-typescript 0.20.2 (git+https://github.com/tree-sitter/tree-sitter-typescript?rev=5d20856f34315b068c41edaee2ac8a100081d259)",
+ "tree-sitter-typescript",
  "unicase",
  "unindent",
  "util",
@@ -6685,14 +6685,15 @@ dependencies = [
  "theme",
  "tiktoken-rs 0.5.0",
  "tree-sitter",
- "tree-sitter-cpp 0.20.2",
- "tree-sitter-elixir 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "tree-sitter-json 0.19.0",
+ "tree-sitter-cpp",
+ "tree-sitter-elixir",
+ "tree-sitter-json 0.20.0",
  "tree-sitter-lua",
+ "tree-sitter-php",
  "tree-sitter-ruby",
  "tree-sitter-rust",
- "tree-sitter-toml 0.20.0",
- "tree-sitter-typescript 0.20.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tree-sitter-toml",
+ "tree-sitter-typescript",
  "unindent",
  "util",
  "workspace",
@@ -8257,16 +8258,6 @@ dependencies = [
  "tree-sitter",
 ]
 
-[[package]]
-name = "tree-sitter-cpp"
-version = "0.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c88fd925d0333e63ac64e521f5bd79c53019e569ffbbccfeef346a326f459e9"
-dependencies = [
- "cc",
- "tree-sitter",
-]
-
 [[package]]
 name = "tree-sitter-css"
 version = "0.19.0"
@@ -8276,16 +8267,6 @@ dependencies = [
  "tree-sitter",
 ]
 
-[[package]]
-name = "tree-sitter-elixir"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a9916f3e1c80b3c8aab8582604e97e8720cb9b893489b347cf999f80f9d469e"
-dependencies = [
- "cc",
- "tree-sitter",
-]
-
 [[package]]
 name = "tree-sitter-elixir"
 version = "0.1.0"
@@ -8464,26 +8445,6 @@ dependencies = [
  "tree-sitter",
 ]
 
-[[package]]
-name = "tree-sitter-toml"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca517f578a98b23d20780247cc2688407fa81effad5b627a5a364ec3339b53e8"
-dependencies = [
- "cc",
- "tree-sitter",
-]
-
-[[package]]
-name = "tree-sitter-typescript"
-version = "0.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "079c695c32d39ad089101c66393aeaca30e967fba3486a91f573d2f0e12d290a"
-dependencies = [
- "cc",
- "tree-sitter",
-]
-
 [[package]]
 name = "tree-sitter-typescript"
 version = "0.20.2"
@@ -9923,9 +9884,9 @@ dependencies = [
  "tree-sitter",
  "tree-sitter-bash",
  "tree-sitter-c",
- "tree-sitter-cpp 0.20.0",
+ "tree-sitter-cpp",
  "tree-sitter-css",
- "tree-sitter-elixir 0.1.0 (git+https://github.com/elixir-lang/tree-sitter-elixir?rev=4ba9dab6e2602960d95b2b625f3386c27e08084e)",
+ "tree-sitter-elixir",
  "tree-sitter-elm",
  "tree-sitter-embedded-template",
  "tree-sitter-glsl",
@@ -9942,8 +9903,8 @@ dependencies = [
  "tree-sitter-rust",
  "tree-sitter-scheme",
  "tree-sitter-svelte",
- "tree-sitter-toml 0.5.1",
- "tree-sitter-typescript 0.20.2 (git+https://github.com/tree-sitter/tree-sitter-typescript?rev=5d20856f34315b068c41edaee2ac8a100081d259)",
+ "tree-sitter-toml",
+ "tree-sitter-typescript",
  "tree-sitter-yaml",
  "unindent",
  "url",
diff --git a/crates/semantic_index/Cargo.toml b/crates/semantic_index/Cargo.toml
index 637f3f4487..3c7a6ff5df 100644
--- a/crates/semantic_index/Cargo.toml
+++ b/crates/semantic_index/Cargo.toml
@@ -54,11 +54,12 @@ tempdir.workspace = true
 ctor.workspace = true
 env_logger.workspace = true
 
-tree-sitter-typescript = "*"
-tree-sitter-json = "*"
-tree-sitter-rust = "*"
-tree-sitter-toml = "*"
-tree-sitter-cpp = "*"
-tree-sitter-elixir = "*"
-tree-sitter-lua = "*"
-tree-sitter-ruby = "*"
+tree-sitter-typescript.workspace = true
+tree-sitter-json.workspace = true
+tree-sitter-rust.workspace = true
+tree-sitter-toml.workspace = true
+tree-sitter-cpp.workspace = true
+tree-sitter-elixir.workspace = true
+tree-sitter-lua.workspace = true
+tree-sitter-ruby.workspace = true
+tree-sitter-php.workspace = true
diff --git a/crates/semantic_index/src/parsing.rs b/crates/semantic_index/src/parsing.rs
index 677406931e..3f7a850a57 100644
--- a/crates/semantic_index/src/parsing.rs
+++ b/crates/semantic_index/src/parsing.rs
@@ -200,7 +200,12 @@ impl CodeContextRetriever {
 
             let mut document_content = String::new();
             for context_range in &context_match.context_ranges {
-                document_content.push_str(&content[context_range.clone()]);
+                add_content_from_range(
+                    &mut document_content,
+                    content,
+                    context_range.clone(),
+                    context_match.start_col,
+                );
                 document_content.push_str("\n");
             }
 
diff --git a/crates/semantic_index/src/semantic_index_tests.rs b/crates/semantic_index/src/semantic_index_tests.rs
index 58d34649c7..0411a8e5ec 100644
--- a/crates/semantic_index/src/semantic_index_tests.rs
+++ b/crates/semantic_index/src/semantic_index_tests.rs
@@ -1017,6 +1017,156 @@ async fn test_code_context_retrieval_ruby() {
     );
 }
 
+#[gpui::test]
+async fn test_code_context_retrieval_php() {
+    let language = php_lang();
+    let mut retriever = CodeContextRetriever::new();
+
+    let text = r#"
+        <?php
+
+        namespace LevelUp\Experience\Concerns;
+
+        /*
+        This is a multiple-lines comment block
+        that spans over multiple
+        lines
+        */
+        function functionName() {
+            echo "Hello world!";
+        }
+
+        trait HasAchievements
+        {
+            /**
+            * @throws \Exception
+            */
+            public function grantAchievement(Achievement $achievement, $progress = null): void
+            {
+                if ($progress > 100) {
+                    throw new Exception(message: 'Progress cannot be greater than 100');
+                }
+
+                if ($this->achievements()->find($achievement->id)) {
+                    throw new Exception(message: 'User already has this Achievement');
+                }
+
+                $this->achievements()->attach($achievement, [
+                    'progress' => $progress ?? null,
+                ]);
+
+                $this->when(value: ($progress === null) || ($progress === 100), callback: fn (): ?array => event(new AchievementAwarded(achievement: $achievement, user: $this)));
+            }
+
+            public function achievements(): BelongsToMany
+            {
+                return $this->belongsToMany(related: Achievement::class)
+                ->withPivot(columns: 'progress')
+                ->where('is_secret', false)
+                ->using(AchievementUser::class);
+            }
+        }
+
+        interface Multiplier
+        {
+            public function qualifies(array $data): bool;
+
+            public function setMultiplier(): int;
+        }
+
+        enum AuditType: string
+        {
+            case Add = 'add';
+            case Remove = 'remove';
+            case Reset = 'reset';
+            case LevelUp = 'level_up';
+        }
+
+        ?>"#
+    .unindent();
+
+    let documents = retriever.parse_file(&text, language.clone()).unwrap();
+
+    assert_documents_eq(
+        &documents,
+        &[
+            (
+                r#"
+        /*
+        This is a multiple-lines comment block
+        that spans over multiple
+        lines
+        */
+        function functionName() {
+            echo "Hello world!";
+        }"#
+                .unindent(),
+                123,
+            ),
+            (
+                r#"
+        trait HasAchievements
+        {
+            /**
+            * @throws \Exception
+            */
+            public function grantAchievement(Achievement $achievement, $progress = null): void
+            {/* ... */}
+
+            public function achievements(): BelongsToMany
+            {/* ... */}
+        }"#
+                .unindent(),
+                177,
+            ),
+            (r#"
+            /**
+            * @throws \Exception
+            */
+            public function grantAchievement(Achievement $achievement, $progress = null): void
+            {
+                if ($progress > 100) {
+                    throw new Exception(message: 'Progress cannot be greater than 100');
+                }
+
+                if ($this->achievements()->find($achievement->id)) {
+                    throw new Exception(message: 'User already has this Achievement');
+                }
+
+                $this->achievements()->attach($achievement, [
+                    'progress' => $progress ?? null,
+                ]);
+
+                $this->when(value: ($progress === null) || ($progress === 100), callback: fn (): ?array => event(new AchievementAwarded(achievement: $achievement, user: $this)));
+            }"#.unindent(), 245),
+            (r#"
+                public function achievements(): BelongsToMany
+                {
+                    return $this->belongsToMany(related: Achievement::class)
+                    ->withPivot(columns: 'progress')
+                    ->where('is_secret', false)
+                    ->using(AchievementUser::class);
+                }"#.unindent(), 902),
+            (r#"
+                interface Multiplier
+                {
+                    public function qualifies(array $data): bool;
+
+                    public function setMultiplier(): int;
+                }"#.unindent(),
+                1146),
+            (r#"
+                enum AuditType: string
+                {
+                    case Add = 'add';
+                    case Remove = 'remove';
+                    case Reset = 'reset';
+                    case LevelUp = 'level_up';
+                }"#.unindent(), 1265)
+        ],
+    );
+}
+
 #[gpui::test]
 fn test_dot_product(mut rng: StdRng) {
     assert_eq!(dot(&[1., 0., 0., 0., 0.], &[0., 1., 0., 0., 0.]), 0.);
@@ -1376,6 +1526,61 @@ fn lua_lang() -> Arc<Language> {
     )
 }
 
+fn php_lang() -> Arc<Language> {
+    Arc::new(
+        Language::new(
+            LanguageConfig {
+                name: "PHP".into(),
+                path_suffixes: vec!["php".into()],
+                collapsed_placeholder: "/* ... */".into(),
+                ..Default::default()
+            },
+            Some(tree_sitter_php::language()),
+        )
+        .with_embedding_query(
+            r#"
+            (
+                (comment)* @context
+                .
+                [
+                    (function_definition
+                        "function" @name
+                        name: (_) @name
+                        body: (_
+                            "{" @keep
+                            "}" @keep) @collapse
+                        )
+
+                    (trait_declaration
+                        "trait" @name
+                        name: (_) @name)
+
+                    (method_declaration
+                        "function" @name
+                        name: (_) @name
+                        body: (_
+                            "{" @keep
+                            "}" @keep) @collapse
+                        )
+
+                    (interface_declaration
+                        "interface" @name
+                        name: (_) @name
+                        )
+
+                    (enum_declaration
+                        "enum" @name
+                        name: (_) @name
+                        )
+
+                ] @item
+            )
+            "#,
+        )
+        .unwrap(),
+    )
+}
+
 fn ruby_lang() -> Arc<Language> {
     Arc::new(
         Language::new(
diff --git a/crates/zed/src/languages/php/config.toml b/crates/zed/src/languages/php/config.toml
index e9de52745a..19acb949e2 100644
--- a/crates/zed/src/languages/php/config.toml
+++ b/crates/zed/src/languages/php/config.toml
@@ -9,3 +9,4 @@ brackets = [
     { start = "(", end = ")", close = true, newline = true },
     { start = "\"", end = "\"", close = true, newline = false, not_in = ["string"] },
 ]
+collapsed_placeholder = "/* ... */"
diff --git a/crates/zed/src/languages/php/embedding.scm b/crates/zed/src/languages/php/embedding.scm
new file mode 100644
index 0000000000..db277775b3
--- /dev/null
+++ b/crates/zed/src/languages/php/embedding.scm
@@ -0,0 +1,36 @@
+(
+    (comment)* @context
+    .
+    [
+        (function_definition
+            "function" @name
+            name: (_) @name
+            body: (_
+                "{" @keep
+                "}" @keep) @collapse
+            )
+
+        (trait_declaration
+            "trait" @name
+            name: (_) @name)
+
+        (method_declaration
+            "function" @name
+            name: (_) @name
+            body: (_
+                "{" @keep
+                "}" @keep) @collapse
+            )
+
+        (interface_declaration
+            "interface" @name
+            name: (_) @name
+            )
+
+        (enum_declaration
+            "enum" @name
+            name: (_) @name
+            )
+
+        ] @item
+    )
diff --git a/crates/zed/src/languages/php/outline.scm b/crates/zed/src/languages/php/outline.scm
index 4934bc494d..87986f1032 100644
--- a/crates/zed/src/languages/php/outline.scm
+++ b/crates/zed/src/languages/php/outline.scm
@@ -8,8 +8,6 @@
     name: (_) @name
     ) @item
 
-
-
 (method_declaration
     "function" @context
     name: (_) @name
@@ -24,3 +22,8 @@
     "enum" @context
     name: (_) @name
     ) @item
+
+(trait_declaration
+    "trait" @context
+    name: (_) @name
+    ) @item

From 9a50b43eaa0f52a0a223e198141b8091ec618548 Mon Sep 17 00:00:00 2001
From: KCaverly <kyle@zed.dev>
Date: Mon, 31 Jul 2023 21:03:02 -0400
Subject: [PATCH 5/6] add templating languages html, erb, heex, svelte as
 entire parseable file types

---
 crates/semantic_index/src/parsing.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/crates/semantic_index/src/parsing.rs b/crates/semantic_index/src/parsing.rs
index 3f7a850a57..643db8c798 100644
--- a/crates/semantic_index/src/parsing.rs
+++ b/crates/semantic_index/src/parsing.rs
@@ -21,7 +21,8 @@ const CODE_CONTEXT_TEMPLATE: &str =
     "The below code snippet is from file '<path>'\n\n```<language>\n<item>\n```";
 const ENTIRE_FILE_TEMPLATE: &str =
     "The below snippet is from file '<path>'\n\n```<language>\n<item>\n```";
-pub const PARSEABLE_ENTIRE_FILE_TYPES: &[&str] = &["TOML", "YAML", "CSS"];
+pub const PARSEABLE_ENTIRE_FILE_TYPES: &[&str] =
+    &["TOML", "YAML", "CSS", "HEEX", "ERB", "SVELTE", "HTML"];
 
 pub struct CodeContextRetriever {
     pub parser: Parser,

From e221f23018d9b9883327712874b1237725922a0a Mon Sep 17 00:00:00 2001
From: KCaverly <kyle@zed.dev>
Date: Tue, 1 Aug 2023 10:30:34 -0400
Subject: [PATCH 6/6] add support for markdown files to semantic search

---
 crates/semantic_index/src/parsing.rs        | 16 ++++++++++++++++
 crates/semantic_index/src/semantic_index.rs |  1 +
 2 files changed, 17 insertions(+)

diff --git a/crates/semantic_index/src/parsing.rs b/crates/semantic_index/src/parsing.rs
index 643db8c798..cef23862c5 100644
--- a/crates/semantic_index/src/parsing.rs
+++ b/crates/semantic_index/src/parsing.rs
@@ -21,6 +21,7 @@ const CODE_CONTEXT_TEMPLATE: &str =
     "The below code snippet is from file '<path>'\n\n```<language>\n<item>\n```";
 const ENTIRE_FILE_TEMPLATE: &str =
     "The below snippet is from file '<path>'\n\n```<language>\n<item>\n```";
+const MARKDOWN_CONTEXT_TEMPLATE: &str = "The below file contents is from file '<path>'\n\n<item>";
 pub const PARSEABLE_ENTIRE_FILE_TYPES: &[&str] =
     &["TOML", "YAML", "CSS", "HEEX", "ERB", "SVELTE", "HTML"];
 
@@ -70,6 +71,19 @@ impl CodeContextRetriever {
         }])
     }
 
+    fn parse_markdown_file(&self, relative_path: &Path, content: &str) -> Result<Vec<Document>> {
+        let document_span = MARKDOWN_CONTEXT_TEMPLATE
+            .replace("<path>", relative_path.to_string_lossy().as_ref())
+            .replace("<item>", &content);
+
+        Ok(vec![Document {
+            range: 0..content.len(),
+            content: document_span,
+            embedding: Vec::new(),
+            name: "Markdown".to_string(),
+        }])
+    }
+
     fn get_matches_in_file(
         &mut self,
         content: &str,
@@ -136,6 +150,8 @@ impl CodeContextRetriever {
 
         if PARSEABLE_ENTIRE_FILE_TYPES.contains(&language_name.as_ref()) {
             return self.parse_entire_file(relative_path, language_name, &content);
+        } else if &language_name.to_string() == &"Markdown".to_string() {
+            return self.parse_markdown_file(relative_path, &content);
         }
 
         let mut documents = self.parse_file(content, language)?;
diff --git a/crates/semantic_index/src/semantic_index.rs b/crates/semantic_index/src/semantic_index.rs
index bd114de216..23c75f4014 100644
--- a/crates/semantic_index/src/semantic_index.rs
+++ b/crates/semantic_index/src/semantic_index.rs
@@ -613,6 +613,7 @@ impl SemanticIndex {
                                 .await
                             {
                                 if !PARSEABLE_ENTIRE_FILE_TYPES.contains(&language.name().as_ref())
+                                    && &language.name().as_ref() != &"Markdown"
                                     && language
                                         .grammar()
                                         .and_then(|grammar| grammar.embedding_config.as_ref())