traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add a CLI feature flag for wasm runtime support

Browse source
This commit is contained in:
Max Brunsfeld 2023-10-31 16:56:11 -07:00
parent 3380e0eed7
commit 473b3c8208
6 changed files with 57 additions and 156 deletions
Download patch file Download diff file Expand all files Collapse all files

5
cli/Cargo.toml
View file

@ -20,6 +20,9 @@ doc = false
name = "benchmark"
harness = false
[features]
wasm = ["tree-sitter/wasm", "tree-sitter-loader/wasm"]
[dependencies]
ansi_term = "0.12.1"
anyhow = "1.0.72"
@ -49,7 +52,6 @@ which = "4.4.0"
[dependencies.tree-sitter]
version = "0.20.10"
path = "../lib"
features = ["wasm"]
[dependencies.tree-sitter-config]
version = "0.19.0"
@ -62,7 +64,6 @@ path = "../highlight"
[dependencies.tree-sitter-loader]
version = "0.20"
path = "loader"
features = ["wasm"]
[dependencies.tree-sitter-tags]
version = "0.20"

41
cli/loader/src/lib.rs
View file

@ -354,29 +354,23 @@ impl Loader {
lib_name.push_str(".debug._");
}
let mut library_path = self.parser_lib_path.join(lib_name);
fs::create_dir_all(&self.parser_lib_path)?;
let parser_path = src_path.join("parser.c");
let mut scanner_path = None;
let mut try_scanner_path = src_path.join("scanner.c");
for extension in ["c", "cc", "cpp"] {
try_scanner_path.set_extension(extension);
if try_scanner_path.exists() {
scanner_path = Some(try_scanner_path);
break;
}
}
let mut library_path = self.parser_lib_path.join(lib_name);
library_path.set_extension(DYLIB_EXTENSION);
let parser_path = src_path.join("parser.c");
let scanner_path = self.get_scanner_path(&src_path);
#[cfg(feature = "wasm")]
if self.wasm_store.lock().unwrap().is_some() {
library_path.set_extension("wasm");
} else {
library_path.set_extension(DYLIB_EXTENSION);
}
let recompile = needs_recompile(&library_path, &parser_path, scanner_path.as_deref())
.with_context(|| "Failed to compare source and binary timestamps")?;
#[cfg(feature = "wasm")]
if let Some(wasm_store) = self.wasm_store.lock().unwrap().as_mut() {
if recompile {
self.compile_parser_to_wasm(
@ -391,8 +385,10 @@ impl Loader {
}
let wasm_bytes = fs::read(&library_path)?;
Ok(wasm_store.load_language(name, &wasm_bytes))
} else {
return Ok(wasm_store.load_language(name, &wasm_bytes));
}
{
if recompile {
self.compile_parser_to_dylib(
header_path,
@ -411,7 +407,7 @@ impl Loader {
language_fn()
};
mem::forget(library);
Ok(language)
return Ok(language);
}
}
@ -528,7 +524,7 @@ impl Loader {
Ok(())
}
fn compile_parser_to_wasm(
pub fn compile_parser_to_wasm(
&self,
language_name: &str,
src_path: &Path,
@ -874,6 +870,17 @@ impl Loader {
pub fn use_wasm(&mut self, engine: tree_sitter::wasmtime::Engine) {
*self.wasm_store.lock().unwrap() = Some(tree_sitter::WasmStore::new(engine))
}
pub fn get_scanner_path(&self, src_path: &Path) -> Option<PathBuf> {
let mut path = src_path.join("scanner.c");
for extension in ["c", "cc", "cpp"] {
path.set_extension(extension);
if path.exists() {
return Some(path);
}
}
None
}
}
impl<'a> LanguageConfiguration<'a> {

30
cli/src/main.rs
View file

@ -4,7 +4,7 @@ use glob::glob;
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::{env, fs, u64};
use tree_sitter::{ffi, Parser, Point, WasmStore};
use tree_sitter::{ffi, Parser, Point};
use tree_sitter_cli::{
generate, highlight, logger,
parse::{self, ParseFileOptions, ParseOutput},
@ -390,8 +390,6 @@ fn run() -> Result<()> {
let debug_build = matches.is_present("debug-build");
let update = matches.is_present("update");
let filter = matches.value_of("filter");
let wasm = matches.is_present("wasm");
let mut parser = Parser::new();
let apply_all_captures = matches.is_present("apply-all-captures");
if debug {
@ -401,10 +399,13 @@ fn run() -> Result<()> {
loader.use_debug_build(debug_build);
if wasm {
let mut parser = Parser::new();
#[cfg(feature = "wasm")]
if matches.is_present("wasm") {
let engine = tree_sitter::wasmtime::Engine::default();
parser
.set_wasm_store(WasmStore::new(engine.clone()))
.set_wasm_store(tree_sitter::WasmStore::new(engine.clone()))
.unwrap();
loader.use_wasm(engine);
}
@ -433,8 +434,6 @@ fn run() -> Result<()> {
)?;
}
let mut store = parser.take_wasm_store();
// Check that all of the queries are valid.
test::check_queries_at_path(*language, &current_dir.join("queries"))?;
@ -442,24 +441,20 @@ fn run() -> Result<()> {
let test_highlight_dir = test_dir.join("highlight");
if test_highlight_dir.is_dir() {
let mut highlighter = Highlighter::new();
if let Some(store) = store.take() {
highlighter.parser().set_wasm_store(store).unwrap();
}
highlighter.parser = parser;
test_highlight::test_highlights(
&loader,
&mut highlighter,
&test_highlight_dir,
apply_all_captures,
)?;
store = highlighter.parser().take_wasm_store();
parser = highlighter.parser;
}
let test_tag_dir = test_dir.join("tags");
if test_tag_dir.is_dir() {
let mut tags_context = TagsContext::new();
if let Some(store) = store.take() {
tags_context.parser().set_wasm_store(store).unwrap();
}
tags_context.parser = parser;
test_tags::test_tags(&loader, &mut tags_context, &test_tag_dir)?;
}
}
@ -490,7 +485,6 @@ fn run() -> Result<()> {
})?;
let time = matches.is_present("time");
let wasm = matches.is_present("wasm");
let edits = matches
.values_of("edits")
.map_or(Vec::new(), |e| e.collect());
@ -504,10 +498,11 @@ fn run() -> Result<()> {
loader.use_debug_build(debug_build);
if wasm {
#[cfg(feature = "wasm")]
if matches.is_present("wasm") {
let engine = tree_sitter::wasmtime::Engine::default();
parser
.set_wasm_store(WasmStore::new(engine.clone()))
.set_wasm_store(tree_sitter::WasmStore::new(engine.clone()))
.unwrap();
loader.use_wasm(engine);
}
@ -744,6 +739,7 @@ fn run() -> Result<()> {
("build-wasm", Some(matches)) => {
let grammar_path = current_dir.join(matches.value_of("path").unwrap_or(""));
wasm::compile_language_to_wasm(
&loader,
&grammar_path,
&current_dir,
matches.is_present("docker"),

133
cli/src/wasm.rs
View file

@ -1,14 +1,7 @@
use super::generate::parse_grammar::GrammarJSON;
use anyhow::{anyhow, Context, Result};
use path_slash::PathExt as _;
use std::{
ffi::{OsStr, OsString},
fs,
path::Path,
process::Command,
};
use tree_sitter_loader::EMSCRIPTEN_TAG;
use which::which;
use anyhow::{Context, Result};
use std::{fs, path::Path};
use tree_sitter_loader::Loader;
pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec<u8>)> {
let grammar_name = get_grammar_name(&language_dir)
@ -35,119 +28,23 @@ pub fn get_grammar_name(language_dir: &Path) -> Result<String> {
}
pub fn compile_language_to_wasm(
loader: &Loader,
language_dir: &Path,
output_dir: &Path,
force_docker: bool,
) -> Result<()> {
let grammar_name = get_grammar_name(&language_dir)?;
let output_filename = output_dir.join(&format!("tree-sitter-{}.wasm", grammar_name));
let emcc_bin = if cfg!(windows) { "emcc.bat" } else { "emcc" };
let emcc_path = which(emcc_bin)
.ok()
.and_then(|p| Command::new(&p).output().and(Ok(p)).ok());
let mut command;
if !force_docker && emcc_path.is_some() {
command = Command::new(emcc_path.unwrap());
command.current_dir(&language_dir);
} else if Command::new("docker").output().is_ok() {
command = Command::new("docker");
command.args(&["run", "--rm"]);
// Mount the parser directory as a volume
let mut volume_string;
if let (Some(parent), Some(filename)) = (language_dir.parent(), language_dir.file_name()) {
volume_string = OsString::from(parent);
volume_string.push(":/src:Z");
command.arg("--workdir");
command.arg(Path::new("/src").join(filename).to_slash_lossy().as_ref());
} else {
volume_string = OsString::from(language_dir);
volume_string.push(":/src:Z");
command.args(&["--workdir", "/src"]);
}
command.args(&[OsStr::new("--volume"), &volume_string]);
// Get the current user id so that files created in the docker container will have
// the same owner.
if cfg!(unix) {
let user_id_output = Command::new("id")
.arg("-u")
.output()
.with_context(|| "Failed to get get current user id")?;
let user_id = String::from_utf8_lossy(&user_id_output.stdout);
let user_id = user_id.trim();
command.args(&["--user", user_id]);
}
// Run `emcc` in a container using the `emscripten-slim` image
command.args(&[EMSCRIPTEN_TAG, "emcc"]);
} else {
if force_docker {
return Err(anyhow!(
"You must have docker on your PATH to run this command with --docker"
));
}
return Err(anyhow!(
"You must have either emcc or docker on your PATH to run this command"
));
}
command.arg("-o").arg(&output_filename);
command.args(&[
"-Os",
"-s",
"WASM=1",
"-s",
"SIDE_MODULE=1",
"-s",
"TOTAL_MEMORY=33554432",
"-s",
"NODEJS_CATCH_EXIT=0",
"-s",
"NODEJS_CATCH_REJECTION=0",
"-s",
&format!("EXPORTED_FUNCTIONS=[\"_tree_sitter_{}\"]", grammar_name),
"-fno-exceptions",
"-I",
"src",
]);
let src = Path::new("src");
let parser_c_path = src.join("parser.c");
let scanner_c_path = src.join("scanner.c");
let scanner_cc_path = src.join("scanner.cc");
let scanner_cpp_path = src.join("scanner.cpp");
if language_dir.join(&scanner_cc_path).exists() {
command
.arg("-xc++")
.arg(scanner_cc_path.to_slash_lossy().as_ref());
} else if language_dir.join(&scanner_cpp_path).exists() {
command
.arg("-xc++")
.arg(scanner_cpp_path.to_slash_lossy().as_ref());
} else if language_dir.join(&scanner_c_path).exists() {
command.arg(scanner_c_path.to_slash_lossy().as_ref());
}
command.arg(parser_c_path.to_slash_lossy().as_ref());
let output = command
.output()
.with_context(|| "Failed to run emcc command")?;
if !output.status.success() {
return Err(anyhow!(
"emcc command failed - {}",
String::from_utf8_lossy(&output.stderr)
));
}
// Move the created `.wasm` file into the current working directory.
fs::rename(&language_dir.join(&output_filename), &output_filename)
.with_context(|| format!("Couldn't find output file {:?}", output_filename))?;
let src_path = language_dir.join("src");
let scanner_path = loader.get_scanner_path(&src_path);
loader.compile_parser_to_wasm(
&grammar_name,
&src_path,
scanner_path
.as_ref()
.and_then(|p| Some(Path::new(p.file_name()?))),
&output_filename,
force_docker,
)?;
Ok(())
}

2
highlight/src/lib.rs
View file

@ -127,7 +127,7 @@ pub struct HighlightConfiguration {
/// syntax highlighting calls. A separate highlighter is needed for each thread that
/// is performing highlighting.
pub struct Highlighter {
parser: Parser,
pub parser: Parser,
cursors: Vec<QueryCursor>,
}

2
tags/src/lib.rs
View file

@ -43,7 +43,7 @@ pub struct NamedCapture {
}
pub struct TagsContext {
parser: Parser,
pub parser: Parser,
cursor: QueryCursor,
}