From 473b3c820869ad1eecb9ce7d634dd37eb210111e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 31 Oct 2023 16:56:11 -0700 Subject: [PATCH] Add a CLI feature flag for wasm runtime support --- cli/Cargo.toml | 5 +- cli/loader/src/lib.rs | 41 +++++++------ cli/src/main.rs | 30 +++++----- cli/src/wasm.rs | 133 +++++------------------------------------- highlight/src/lib.rs | 2 +- tags/src/lib.rs | 2 +- 6 files changed, 57 insertions(+), 156 deletions(-) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index b69bd7c0..d5066668 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -20,6 +20,9 @@ doc = false name = "benchmark" harness = false +[features] +wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"] + [dependencies] ansi_term = "0.12.1" anyhow = "1.0.72" @@ -49,7 +52,6 @@ which = "4.4.0" [dependencies.tree-sitter] version = "0.20.10" path = "../lib" -features = ["wasm"] [dependencies.tree-sitter-config] version = "0.19.0" @@ -62,7 +64,6 @@ path = "../highlight" [dependencies.tree-sitter-loader] version = "0.20" path = "loader" -features = ["wasm"] [dependencies.tree-sitter-tags] version = "0.20" diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index 619aa05e..79ea706a 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -354,29 +354,23 @@ impl Loader { lib_name.push_str(".debug._"); } - let mut library_path = self.parser_lib_path.join(lib_name); fs::create_dir_all(&self.parser_lib_path)?; - let parser_path = src_path.join("parser.c"); - let mut scanner_path = None; - let mut try_scanner_path = src_path.join("scanner.c"); - for extension in ["c", "cc", "cpp"] { - try_scanner_path.set_extension(extension); - if try_scanner_path.exists() { - scanner_path = Some(try_scanner_path); - break; - } - } + let mut library_path = self.parser_lib_path.join(lib_name); + library_path.set_extension(DYLIB_EXTENSION); + let parser_path = src_path.join("parser.c"); + let scanner_path = self.get_scanner_path(&src_path); + + #[cfg(feature = "wasm")] if self.wasm_store.lock().unwrap().is_some() { library_path.set_extension("wasm"); - } else { - library_path.set_extension(DYLIB_EXTENSION); } let recompile = needs_recompile(&library_path, &parser_path, scanner_path.as_deref()) .with_context(|| "Failed to compare source and binary timestamps")?; + #[cfg(feature = "wasm")] if let Some(wasm_store) = self.wasm_store.lock().unwrap().as_mut() { if recompile { self.compile_parser_to_wasm( @@ -391,8 +385,10 @@ impl Loader { } let wasm_bytes = fs::read(&library_path)?; - Ok(wasm_store.load_language(name, &wasm_bytes)) - } else { + return Ok(wasm_store.load_language(name, &wasm_bytes)); + } + + { if recompile { self.compile_parser_to_dylib( header_path, @@ -411,7 +407,7 @@ impl Loader { language_fn() }; mem::forget(library); - Ok(language) + return Ok(language); } } @@ -528,7 +524,7 @@ impl Loader { Ok(()) } - fn compile_parser_to_wasm( + pub fn compile_parser_to_wasm( &self, language_name: &str, src_path: &Path, @@ -874,6 +870,17 @@ impl Loader { pub fn use_wasm(&mut self, engine: tree_sitter::wasmtime::Engine) { *self.wasm_store.lock().unwrap() = Some(tree_sitter::WasmStore::new(engine)) } + + pub fn get_scanner_path(&self, src_path: &Path) -> Option { + let mut path = src_path.join("scanner.c"); + for extension in ["c", "cc", "cpp"] { + path.set_extension(extension); + if path.exists() { + return Some(path); + } + } + None + } } impl<'a> LanguageConfiguration<'a> { diff --git a/cli/src/main.rs b/cli/src/main.rs index 0e59a150..b6c4932e 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -4,7 +4,7 @@ use glob::glob; use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::{env, fs, u64}; -use tree_sitter::{ffi, Parser, Point, WasmStore}; +use tree_sitter::{ffi, Parser, Point}; use tree_sitter_cli::{ generate, highlight, logger, parse::{self, ParseFileOptions, ParseOutput}, @@ -390,8 +390,6 @@ fn run() -> Result<()> { let debug_build = matches.is_present("debug-build"); let update = matches.is_present("update"); let filter = matches.value_of("filter"); - let wasm = matches.is_present("wasm"); - let mut parser = Parser::new(); let apply_all_captures = matches.is_present("apply-all-captures"); if debug { @@ -401,10 +399,13 @@ fn run() -> Result<()> { loader.use_debug_build(debug_build); - if wasm { + let mut parser = Parser::new(); + + #[cfg(feature = "wasm")] + if matches.is_present("wasm") { let engine = tree_sitter::wasmtime::Engine::default(); parser - .set_wasm_store(WasmStore::new(engine.clone())) + .set_wasm_store(tree_sitter::WasmStore::new(engine.clone())) .unwrap(); loader.use_wasm(engine); } @@ -433,8 +434,6 @@ fn run() -> Result<()> { )?; } - let mut store = parser.take_wasm_store(); - // Check that all of the queries are valid. test::check_queries_at_path(*language, ¤t_dir.join("queries"))?; @@ -442,24 +441,20 @@ fn run() -> Result<()> { let test_highlight_dir = test_dir.join("highlight"); if test_highlight_dir.is_dir() { let mut highlighter = Highlighter::new(); - if let Some(store) = store.take() { - highlighter.parser().set_wasm_store(store).unwrap(); - } + highlighter.parser = parser; test_highlight::test_highlights( &loader, &mut highlighter, &test_highlight_dir, apply_all_captures, )?; - store = highlighter.parser().take_wasm_store(); + parser = highlighter.parser; } let test_tag_dir = test_dir.join("tags"); if test_tag_dir.is_dir() { let mut tags_context = TagsContext::new(); - if let Some(store) = store.take() { - tags_context.parser().set_wasm_store(store).unwrap(); - } + tags_context.parser = parser; test_tags::test_tags(&loader, &mut tags_context, &test_tag_dir)?; } } @@ -490,7 +485,6 @@ fn run() -> Result<()> { })?; let time = matches.is_present("time"); - let wasm = matches.is_present("wasm"); let edits = matches .values_of("edits") .map_or(Vec::new(), |e| e.collect()); @@ -504,10 +498,11 @@ fn run() -> Result<()> { loader.use_debug_build(debug_build); - if wasm { + #[cfg(feature = "wasm")] + if matches.is_present("wasm") { let engine = tree_sitter::wasmtime::Engine::default(); parser - .set_wasm_store(WasmStore::new(engine.clone())) + .set_wasm_store(tree_sitter::WasmStore::new(engine.clone())) .unwrap(); loader.use_wasm(engine); } @@ -744,6 +739,7 @@ fn run() -> Result<()> { ("build-wasm", Some(matches)) => { let grammar_path = current_dir.join(matches.value_of("path").unwrap_or("")); wasm::compile_language_to_wasm( + &loader, &grammar_path, ¤t_dir, matches.is_present("docker"), diff --git a/cli/src/wasm.rs b/cli/src/wasm.rs index 794ba353..5b3b093f 100644 --- a/cli/src/wasm.rs +++ b/cli/src/wasm.rs @@ -1,14 +1,7 @@ use super::generate::parse_grammar::GrammarJSON; -use anyhow::{anyhow, Context, Result}; -use path_slash::PathExt as _; -use std::{ - ffi::{OsStr, OsString}, - fs, - path::Path, - process::Command, -}; -use tree_sitter_loader::EMSCRIPTEN_TAG; -use which::which; +use anyhow::{Context, Result}; +use std::{fs, path::Path}; +use tree_sitter_loader::Loader; pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec)> { let grammar_name = get_grammar_name(&language_dir) @@ -35,119 +28,23 @@ pub fn get_grammar_name(language_dir: &Path) -> Result { } pub fn compile_language_to_wasm( + loader: &Loader, language_dir: &Path, output_dir: &Path, force_docker: bool, ) -> Result<()> { let grammar_name = get_grammar_name(&language_dir)?; let output_filename = output_dir.join(&format!("tree-sitter-{}.wasm", grammar_name)); - - let emcc_bin = if cfg!(windows) { "emcc.bat" } else { "emcc" }; - let emcc_path = which(emcc_bin) - .ok() - .and_then(|p| Command::new(&p).output().and(Ok(p)).ok()); - - let mut command; - if !force_docker && emcc_path.is_some() { - command = Command::new(emcc_path.unwrap()); - command.current_dir(&language_dir); - } else if Command::new("docker").output().is_ok() { - command = Command::new("docker"); - command.args(&["run", "--rm"]); - - // Mount the parser directory as a volume - let mut volume_string; - if let (Some(parent), Some(filename)) = (language_dir.parent(), language_dir.file_name()) { - volume_string = OsString::from(parent); - volume_string.push(":/src:Z"); - command.arg("--workdir"); - command.arg(Path::new("/src").join(filename).to_slash_lossy().as_ref()); - } else { - volume_string = OsString::from(language_dir); - volume_string.push(":/src:Z"); - command.args(&["--workdir", "/src"]); - } - - command.args(&[OsStr::new("--volume"), &volume_string]); - - // Get the current user id so that files created in the docker container will have - // the same owner. - if cfg!(unix) { - let user_id_output = Command::new("id") - .arg("-u") - .output() - .with_context(|| "Failed to get get current user id")?; - let user_id = String::from_utf8_lossy(&user_id_output.stdout); - let user_id = user_id.trim(); - command.args(&["--user", user_id]); - } - - // Run `emcc` in a container using the `emscripten-slim` image - command.args(&[EMSCRIPTEN_TAG, "emcc"]); - } else { - if force_docker { - return Err(anyhow!( - "You must have docker on your PATH to run this command with --docker" - )); - } - return Err(anyhow!( - "You must have either emcc or docker on your PATH to run this command" - )); - } - - command.arg("-o").arg(&output_filename); - command.args(&[ - "-Os", - "-s", - "WASM=1", - "-s", - "SIDE_MODULE=1", - "-s", - "TOTAL_MEMORY=33554432", - "-s", - "NODEJS_CATCH_EXIT=0", - "-s", - "NODEJS_CATCH_REJECTION=0", - "-s", - &format!("EXPORTED_FUNCTIONS=[\"_tree_sitter_{}\"]", grammar_name), - "-fno-exceptions", - "-I", - "src", - ]); - - let src = Path::new("src"); - let parser_c_path = src.join("parser.c"); - let scanner_c_path = src.join("scanner.c"); - let scanner_cc_path = src.join("scanner.cc"); - let scanner_cpp_path = src.join("scanner.cpp"); - - if language_dir.join(&scanner_cc_path).exists() { - command - .arg("-xc++") - .arg(scanner_cc_path.to_slash_lossy().as_ref()); - } else if language_dir.join(&scanner_cpp_path).exists() { - command - .arg("-xc++") - .arg(scanner_cpp_path.to_slash_lossy().as_ref()); - } else if language_dir.join(&scanner_c_path).exists() { - command.arg(scanner_c_path.to_slash_lossy().as_ref()); - } - - command.arg(parser_c_path.to_slash_lossy().as_ref()); - - let output = command - .output() - .with_context(|| "Failed to run emcc command")?; - if !output.status.success() { - return Err(anyhow!( - "emcc command failed - {}", - String::from_utf8_lossy(&output.stderr) - )); - } - - // Move the created `.wasm` file into the current working directory. - fs::rename(&language_dir.join(&output_filename), &output_filename) - .with_context(|| format!("Couldn't find output file {:?}", output_filename))?; - + let src_path = language_dir.join("src"); + let scanner_path = loader.get_scanner_path(&src_path); + loader.compile_parser_to_wasm( + &grammar_name, + &src_path, + scanner_path + .as_ref() + .and_then(|p| Some(Path::new(p.file_name()?))), + &output_filename, + force_docker, + )?; Ok(()) } diff --git a/highlight/src/lib.rs b/highlight/src/lib.rs index 2170b07f..304d5e95 100644 --- a/highlight/src/lib.rs +++ b/highlight/src/lib.rs @@ -127,7 +127,7 @@ pub struct HighlightConfiguration { /// syntax highlighting calls. A separate highlighter is needed for each thread that /// is performing highlighting. pub struct Highlighter { - parser: Parser, + pub parser: Parser, cursors: Vec, } diff --git a/tags/src/lib.rs b/tags/src/lib.rs index e151e3ee..495a5221 100644 --- a/tags/src/lib.rs +++ b/tags/src/lib.rs @@ -43,7 +43,7 @@ pub struct NamedCapture { } pub struct TagsContext { - parser: Parser, + pub parser: Parser, cursor: QueryCursor, }