extensions: Don't recompile Tree-sitter parsers when up-to-date (#43442)
When installing local "dev" extensions that provide tree-sitter grammars, compiling the parser can be quite intensive[^anecdote]. This PR changes the logic to only compile the parser if the WASM object doesn't exist or the source files are newer than the object (just like `make(1)` would do). [^anecdote]: The tree-sitter parser for LLVM IR takes >10 minutes to compile and uses 20 GB of memory on my laptop. Release Notes: - N/A --------- Co-authored-by: Finn Evers <finn.evers@outlook.de>
This commit is contained in:
Generated
+1
@@ -5809,6 +5809,7 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"task",
|
"task",
|
||||||
|
"tempfile",
|
||||||
"toml 0.8.23",
|
"toml 0.8.23",
|
||||||
"url",
|
"url",
|
||||||
"util",
|
"util",
|
||||||
|
|||||||
@@ -38,3 +38,4 @@ wasmparser.workspace = true
|
|||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
pretty_assertions.workspace = true
|
pretty_assertions.workspace = true
|
||||||
|
tempfile.workspace = true
|
||||||
|
|||||||
@@ -247,26 +247,34 @@ impl ExtensionBuilder {
|
|||||||
let parser_path = src_path.join("parser.c");
|
let parser_path = src_path.join("parser.c");
|
||||||
let scanner_path = src_path.join("scanner.c");
|
let scanner_path = src_path.join("scanner.c");
|
||||||
|
|
||||||
log::info!("compiling {grammar_name} parser");
|
// Skip recompiling if the WASM object is already newer than the source files
|
||||||
let clang_output = util::command::new_smol_command(&clang_path)
|
if file_newer_than_deps(&grammar_wasm_path, &[&parser_path, &scanner_path]).unwrap_or(false)
|
||||||
.args(["-fPIC", "-shared", "-Os"])
|
{
|
||||||
.arg(format!("-Wl,--export=tree_sitter_{grammar_name}"))
|
log::info!(
|
||||||
.arg("-o")
|
"skipping compilation of {grammar_name} parser because the existing compiled grammar is up to date"
|
||||||
.arg(&grammar_wasm_path)
|
|
||||||
.arg("-I")
|
|
||||||
.arg(&src_path)
|
|
||||||
.arg(&parser_path)
|
|
||||||
.args(scanner_path.exists().then_some(scanner_path))
|
|
||||||
.output()
|
|
||||||
.await
|
|
||||||
.context("failed to run clang")?;
|
|
||||||
|
|
||||||
if !clang_output.status.success() {
|
|
||||||
bail!(
|
|
||||||
"failed to compile {} parser with clang: {}",
|
|
||||||
grammar_name,
|
|
||||||
String::from_utf8_lossy(&clang_output.stderr),
|
|
||||||
);
|
);
|
||||||
|
} else {
|
||||||
|
log::info!("compiling {grammar_name} parser");
|
||||||
|
let clang_output = util::command::new_smol_command(&clang_path)
|
||||||
|
.args(["-fPIC", "-shared", "-Os"])
|
||||||
|
.arg(format!("-Wl,--export=tree_sitter_{grammar_name}"))
|
||||||
|
.arg("-o")
|
||||||
|
.arg(&grammar_wasm_path)
|
||||||
|
.arg("-I")
|
||||||
|
.arg(&src_path)
|
||||||
|
.arg(&parser_path)
|
||||||
|
.args(scanner_path.exists().then_some(scanner_path))
|
||||||
|
.output()
|
||||||
|
.await
|
||||||
|
.context("failed to run clang")?;
|
||||||
|
|
||||||
|
if !clang_output.status.success() {
|
||||||
|
bail!(
|
||||||
|
"failed to compile {} parser with clang: {}",
|
||||||
|
grammar_name,
|
||||||
|
String::from_utf8_lossy(&clang_output.stderr),
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -643,3 +651,71 @@ fn populate_defaults(manifest: &mut ExtensionManifest, extension_path: &Path) ->
|
|||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns `true` if the target exists and its last modified time is greater than that
|
||||||
|
/// of each dependency which exists (i.e., dependency paths which do not exist are ignored).
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// Returns `Err` if any of the underlying file I/O operations fail.
|
||||||
|
fn file_newer_than_deps(target: &Path, dependencies: &[&Path]) -> Result<bool, std::io::Error> {
|
||||||
|
if !target.try_exists()? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
let target_modified = target.metadata()?.modified()?;
|
||||||
|
for dependency in dependencies {
|
||||||
|
if !dependency.try_exists()? {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let dep_modified = dependency.metadata()?.modified()?;
|
||||||
|
if target_modified < dep_modified {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
use std::{fs, thread::sleep, time::Duration};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_file_newer_than_deps() {
|
||||||
|
// Don't use TempTree because we need to guarantee the order
|
||||||
|
let tmpdir = tempfile::tempdir().unwrap();
|
||||||
|
let target = tmpdir.path().join("target.wasm");
|
||||||
|
let dep1 = tmpdir.path().join("parser.c");
|
||||||
|
let dep2 = tmpdir.path().join("scanner.c");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
!file_newer_than_deps(&target, &[&dep1, &dep2]).unwrap(),
|
||||||
|
"target doesn't exist"
|
||||||
|
);
|
||||||
|
fs::write(&target, "foo").unwrap(); // Create target
|
||||||
|
assert!(
|
||||||
|
file_newer_than_deps(&target, &[&dep1, &dep2]).unwrap(),
|
||||||
|
"dependencies don't exist; target is newer"
|
||||||
|
);
|
||||||
|
sleep(Duration::from_secs(1));
|
||||||
|
fs::write(&dep1, "foo").unwrap(); // Create dep1 (newer than target)
|
||||||
|
// Dependency is newer
|
||||||
|
assert!(
|
||||||
|
!file_newer_than_deps(&target, &[&dep1, &dep2]).unwrap(),
|
||||||
|
"a dependency is newer (target {:?}, dep1 {:?})",
|
||||||
|
target.metadata().unwrap().modified().unwrap(),
|
||||||
|
dep1.metadata().unwrap().modified().unwrap(),
|
||||||
|
);
|
||||||
|
sleep(Duration::from_secs(1));
|
||||||
|
fs::write(&dep2, "foo").unwrap(); // Create dep2
|
||||||
|
sleep(Duration::from_secs(1));
|
||||||
|
fs::write(&target, "foobar").unwrap(); // Update target
|
||||||
|
assert!(
|
||||||
|
file_newer_than_deps(&target, &[&dep1, &dep2]).unwrap(),
|
||||||
|
"target is newer than dependencies (target {:?}, dep2 {:?})",
|
||||||
|
target.metadata().unwrap().modified().unwrap(),
|
||||||
|
dep2.metadata().unwrap().modified().unwrap(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user