traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Integrate WASM compilation into the CLI's Loader

Browse source
This commit is contained in:
Max Brunsfeld 2022-09-06 22:41:52 -07:00
parent 042e6f9d57
commit d47713ee4a
15 changed files with 310 additions and 151 deletions
Download patch file Download diff file Expand all files Collapse all files

30
cli/src/main.rs
View file

@ -3,6 +3,7 @@ use clap::{App, AppSettings, Arg, SubCommand};
use glob::glob;
use std::path::Path;
use std::{env, fs, u64};
use tree_sitter::{Parser, WasmStore};
use tree_sitter_cli::{
generate, highlight, logger, parse, playground, query, tags, test, test_highlight, test_tags,
util, wasm,
@ -359,6 +360,7 @@ fn run() -> Result<()> {
.values_of("edits")
.map_or(Vec::new(), |e| e.collect());
let cancellation_flag = util::cancel_on_stdin();
let mut parser = Parser::new();
if debug {
// For augmenting debug logging in external scanners
@ -367,6 +369,14 @@ fn run() -> Result<()> {
loader.use_debug_build(debug_build);
if wasm {
let engine = tree_sitter::wasmtime::Engine::default();
parser
.set_wasm_store(WasmStore::new(engine.clone()))
.unwrap();
loader.use_wasm(engine);
}
let timeout = matches
.value_of("timeout")
.map_or(0, |t| u64::from_str_radix(t, 10).unwrap());
@ -381,23 +391,17 @@ fn run() -> Result<()> {
let should_track_stats = matches.is_present("stat");
let mut stats = parse::Stats::default();
let mut wasm_language = None;
if wasm {
let (language_name, wasm_file) = wasm::load_language_wasm_file(&current_dir)?;
let engine = tree_sitter::wasmtime::Engine::default();
let mut context = tree_sitter::WasmStore::new(engine);
wasm_language = Some(context.load_language(&language_name, &wasm_file));
std::mem::forget(context);
}
for path in paths {
let path = Path::new(&path);
let language =
loader.select_language(path, &current_dir, matches.value_of("scope"))?;
parser
.set_language(language)
.context("incompatible language")?;
let this_file_errored = parse::parse_file_at_path(
language,
&mut parser,
path,
&edits,
max_path_length,
@ -539,7 +543,11 @@ fn run() -> Result<()> {
("build-wasm", Some(matches)) => {
let grammar_path = current_dir.join(matches.value_of("path").unwrap_or(""));
wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?;
wasm::compile_language_to_wasm(
&grammar_path,
&current_dir,
matches.is_present("docker"),
)?;
}
("playground", Some(matches)) => {

8
cli/src/parse.rs
View file

@ -5,7 +5,7 @@ use std::path::Path;
use std::sync::atomic::AtomicUsize;
use std::time::Instant;
use std::{fmt, fs, usize};
use tree_sitter::{InputEdit, Language, LogType, Parser, Point, Tree};
use tree_sitter::{InputEdit, LogType, Parser, Point, Tree};
#[derive(Debug)]
pub struct Edit {
@ -31,7 +31,7 @@ impl fmt::Display for Stats {
}
pub fn parse_file_at_path(
language: Language,
parser: &mut Parser,
path: &Path,
edits: &Vec<&str>,
max_path_length: usize,
@ -44,8 +44,6 @@ pub fn parse_file_at_path(
cancellation_flag: Option<&AtomicUsize>,
) -> Result<bool> {
let mut _log_session = None;
let mut parser = Parser::new();
parser.set_language(language)?;
let mut source_code =
fs::read(path).with_context(|| format!("Error reading source file {:?}", path))?;
@ -58,7 +56,7 @@ pub fn parse_file_at_path(
// Render an HTML graph if `--debug-graph` was passed
if debug_graph {
_log_session = Some(util::log_graphs(&mut parser, "log.html")?);
_log_session = Some(util::log_graphs(parser, "log.html")?);
}
// Log to stderr if `--debug` was passed
else if debug {

43
cli/src/tests/helpers/fixtures.rs
View file

@ -9,7 +9,7 @@ use tree_sitter_tags::TagsConfiguration;
include!("./dirs.rs");
lazy_static! {
static ref TEST_LOADER: Loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone());
static ref TEST_LOADER: Loader = Loader::with_parser_lib_path(SCRATCH_DIR.join("lib"));
}
pub fn test_loader<'a>() -> &'a Loader {
@ -63,29 +63,28 @@ pub fn get_tags_config(language_name: &str) -> TagsConfiguration {
TagsConfiguration::new(language, &tags_query, &locals_query).unwrap()
}
pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> Language {
let parser_c_path = SCRATCH_DIR.join(&format!("{}-parser.c", name));
if !fs::read_to_string(&parser_c_path)
.map(|content| content == parser_code)
.unwrap_or(false)
{
fs::write(&parser_c_path, parser_code).unwrap();
pub fn get_test_language(
name: &str,
parser_code: &str,
scanner_src_path: Option<&Path>,
) -> Language {
let src_dir = SCRATCH_DIR.join("src").join(name);
fs::create_dir_all(&src_dir).unwrap();
let parser_path = src_dir.join("parser.c");
if !fs::read_to_string(&parser_path).map_or(false, |content| content == parser_code) {
fs::write(&parser_path, parser_code).unwrap();
}
let scanner_path = path.and_then(|p| {
let result = p.join("scanner.c");
if result.exists() {
Some(result)
} else {
None
if let Some(scanner_src_path) = scanner_src_path {
let scanner_code = fs::read_to_string(&scanner_src_path).unwrap();
let scanner_path = src_dir.join("scanner.c");
if !fs::read_to_string(&scanner_path).map_or(false, |content| content == scanner_code) {
fs::write(&scanner_path, scanner_code).unwrap();
}
});
}
TEST_LOADER
.load_language_from_sources(name, &HEADER_DIR, &parser_c_path, &scanner_path)
.load_language_at_path_with_name(&src_dir, &HEADER_DIR, name)
.unwrap()
}
pub fn get_test_grammar(name: &str) -> (String, Option<PathBuf>) {
let dir = fixtures_dir().join("test_grammars").join(name);
let grammar = fs::read_to_string(&dir.join("grammar.json")).unwrap();
(grammar, Some(dir))
}

12
cli/src/tests/parser_test.rs
View file

@ -2,13 +2,15 @@ use super::helpers::{
allocations,
edits::invert_edit,
edits::ReadRecorder,
fixtures::{get_language, get_test_grammar, get_test_language},
fixtures::{get_language, get_test_language},
};
use crate::{
generate::generate_parser_for_grammar,
parse::{perform_edit, Edit},
tests::helpers::fixtures::fixtures_dir,
};
use std::{
fs,
sync::atomic::{AtomicUsize, Ordering},
thread, time,
};
@ -421,7 +423,11 @@ fn test_parsing_empty_file_with_reused_tree() {
#[test]
fn test_parsing_after_editing_tree_that_depends_on_column_values() {
let (grammar, path) = get_test_grammar("uses_current_column");
let dir = fixtures_dir()
.join("test_grammars")
.join("uses_current_column");
let grammar = fs::read_to_string(&dir.join("grammar.json")).unwrap();
let scanner_path = dir.join("scanner.c");
let (grammar_name, parser_code) = generate_parser_for_grammar(&grammar).unwrap();
let mut parser = Parser::new();
@ -429,7 +435,7 @@ fn test_parsing_after_editing_tree_that_depends_on_column_values() {
.set_language(get_test_language(
&grammar_name,
&parser_code,
path.as_ref().map(AsRef::as_ref),
Some(&scanner_path),
))
.unwrap();

14
cli/src/wasm.rs
View file

@ -6,10 +6,9 @@ use std::{
path::Path,
process::Command,
};
use tree_sitter_loader::EMSCRIPTEN_TAG;
use which::which;
const EMSCRIPTEN_TAG: &'static str = concat!("emscripten/emsdk:", env!("EMSCRIPTEN_VERSION"));
pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec<u8>)> {
let grammar_name = get_grammar_name(&language_dir)
.with_context(|| "Failed to get wasm filename")
@ -34,9 +33,13 @@ pub fn get_grammar_name(language_dir: &Path) -> Result<String> {
Ok(grammar.name)
}
pub fn compile_language_to_wasm(language_dir: &Path, force_docker: bool) -> Result<()> {
pub fn compile_language_to_wasm(
language_dir: &Path,
output_dir: &Path,
force_docker: bool,
) -> Result<()> {
let grammar_name = get_grammar_name(&language_dir)?;
let output_filename = format!("tree-sitter-{}.wasm", grammar_name);
let output_filename = output_dir.join(&format!("tree-sitter-{}.wasm", grammar_name));
let emcc_bin = if cfg!(windows) { "emcc.bat" } else { "emcc" };
let emcc_path = which(emcc_bin)
@ -86,9 +89,8 @@ pub fn compile_language_to_wasm(language_dir: &Path, force_docker: bool) -> Resu
));
}
command.arg("-o").arg(&output_filename);
command.args(&[
"-o",
&output_filename,
"-Os",
"-s",
"WASM=1",