traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(cli)!: add a separate build command to compile parsers

Browse source 
This allows users to build parsers without having to run `test` or
`parse` to invoke the compilation process, and allows them to output the
object file to wherever they like. The `build-wasm` command was merged
into this by just specifying the `--wasm` flag.
This commit is contained in:
Amaan Qureshi 2024-02-23 17:40:20 -05:00
parent a1c39d4760
commit 99b93d83a1
12 changed files with 249 additions and 139 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cli/src/generate/templates/package.json
View file

@ -40,7 +40,7 @@
"install": "node-gyp-build",
"prebuildify": "prebuildify --napi --strip",
"build": "tree-sitter generate --no-bindings",
"build-wasm": "tree-sitter build-wasm",
"build-wasm": "tree-sitter build --wasm",
"test": "tree-sitter test",
"parse": "tree-sitter parse"
},

106
cli/src/main.rs
View file

@ -27,12 +27,12 @@ const DEFAULT_GENERATE_ABI_VERSION: usize = 14;
enum Commands {
InitConfig(InitConfig),
Generate(Generate),
Build(Build),
Parse(Parse),
Test(Test),
Query(Query),
Highlight(Highlight),
Tags(Tags),
BuildWasm(BuildWasm),
Playground(Playground),
DumpLanguages(DumpLanguages),
}
@ -91,6 +91,30 @@ struct Generate {
pub js_runtime: Option<String>,
}
#[derive(Args)]
#[command(about = "Compile a parser", alias = "b")]
struct Build {
#[arg(short, long, help = "Build a WASM module instead of a dynamic library")]
pub wasm: bool,
#[arg(
short,
long,
help = "Run emscripten via docker even if it is installed locally (only if building a WASM module with --wasm)"
)]
pub docker: bool,
#[arg(short, long, help = "The path to output the compiled file")]
pub output: Option<String>,
#[arg(index = 1, num_args = 1, help = "The path to the grammar directory")]
pub path: Option<String>,
#[arg(long, help = "Make the parser reuse the same allocator as the library")]
pub reuse_allocator: bool,
#[arg(
long,
help = "Build the parser with `TREE_SITTER_INTERNAL_BUILD` defined"
)]
pub internal_build: bool,
}
#[derive(Args)]
#[command(about = "Parse files", alias = "p")]
struct Parse {
@ -298,18 +322,6 @@ struct Tags {
pub config_path: Option<PathBuf>,
}
#[derive(Args)]
#[command(about = "Compile a parser to WASM", alias = "bw")]
struct BuildWasm {
#[arg(
long,
help = "Run emscripten via docker even if it is installed locally"
)]
pub docker: bool,
#[arg(index = 1, num_args = 1.., help = "The path to output the wasm file")]
pub path: Option<String>,
}
#[derive(Args)]
#[command(
about = "Start local playground for a parser in the browser",
@ -428,6 +440,64 @@ fn run() -> Result<()> {
}
}
Commands::Build(build_options) => {
if build_options.wasm {
let grammar_path =
current_dir.join(build_options.path.as_deref().unwrap_or_default());
let (output_dir, output_path) = if let Some(ref path) = build_options.output {
(current_dir.clone(), Some(current_dir.join(path)))
} else {
(loader.parser_lib_path.clone(), None)
};
wasm::compile_language_to_wasm(
&loader,
&grammar_path,
&output_dir,
output_path,
build_options.docker,
)?;
} else {
let grammar_path =
current_dir.join(build_options.path.as_deref().unwrap_or_default());
let output_path = if let Some(ref path) = build_options.output {
let path = Path::new(path);
if path.is_absolute() {
path.to_path_buf()
} else {
current_dir.join(path)
}
} else {
let file_name = grammar_path
.file_stem()
.unwrap()
.to_str()
.unwrap()
.strip_prefix("tree-sitter-")
.unwrap_or("parser");
current_dir
.join(file_name)
.with_extension(env::consts::DLL_EXTENSION)
};
let flags: &[&str] =
match (build_options.reuse_allocator, build_options.internal_build) {
(true, true) => {
&["TREE_SITTER_REUSE_ALLOCATOR", "TREE_SITTER_INTERNAL_BUILD"]
}
(true, false) => &["TREE_SITTER_REUSE_ALLOCATOR"],
(false, true) => &["TREE_SITTER_INTERNAL_BUILD"],
(false, false) => &[""],
};
let config = Config::load(None)?;
let loader_config = config.get()?;
loader.find_all_languages(&loader_config).unwrap();
loader
.compile_parser_at_path(&grammar_path, output_path, flags)
.unwrap();
}
}
Commands::Parse(parse_options) => {
let config = Config::load(parse_options.config_path)?;
let output = if parse_options.output_dot {
@ -770,16 +840,6 @@ fn run() -> Result<()> {
)?;
}
Commands::BuildWasm(wasm_options) => {
let grammar_path = current_dir.join(wasm_options.path.unwrap_or_default());
wasm::compile_language_to_wasm(
&loader,
&grammar_path,
&current_dir,
wasm_options.docker,
)?;
}
Commands::Playground(playground_options) => {
let open_in_browser = !playground_options.quiet;
let grammar_path = playground_options

21
cli/src/tests/helpers/fixtures.rs
View file

@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
use std::{env, fs};
use tree_sitter::Language;
use tree_sitter_highlight::HighlightConfiguration;
use tree_sitter_loader::Loader;
use tree_sitter_loader::{CompileConfig, Loader};
use tree_sitter_tags::TagsConfiguration;
include!("./dirs.rs");
@ -32,13 +32,10 @@ pub fn scratch_dir() -> &'static Path {
}
pub fn get_language(name: &str) -> Language {
TEST_LOADER
.load_language_at_path(
&GRAMMARS_DIR.join(name).join("src"),
&[&HEADER_DIR, &GRAMMARS_DIR.join(name).join("src")],
None,
)
.unwrap()
let src_dir = GRAMMARS_DIR.join(name).join("src");
let mut config = CompileConfig::new(&src_dir, None, None);
config.header_paths.push(&HEADER_DIR);
TEST_LOADER.load_language_at_path(config).unwrap()
}
pub fn get_language_queries_path(language_name: &str) -> PathBuf {
@ -141,7 +138,9 @@ pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) ->
vec![parser_path]
};
TEST_LOADER
.load_language_at_path_with_name(&src_dir, &[&HEADER_DIR], name, Some(&paths_to_check))
.unwrap()
let mut config = CompileConfig::new(&src_dir, Some(&paths_to_check), None);
config.header_paths = vec![&HEADER_DIR];
config.name = name.to_string();
TEST_LOADER.load_language_at_path_with_name(config).unwrap()
}

11
cli/src/wasm.rs
View file

@ -1,6 +1,9 @@
use super::generate::parse_grammar::GrammarJSON;
use anyhow::{anyhow, Context, Result};
use std::{fs, path::Path};
use std::{
fs,
path::{Path, PathBuf},
};
use tree_sitter::wasm_stdlib_symbols;
use tree_sitter_loader::Loader;
use wasmparser::Parser;
@ -11,7 +14,7 @@ pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec<u8>)>
.unwrap();
let wasm_filename = format!("tree-sitter-{grammar_name}.wasm");
let contents = fs::read(language_dir.join(&wasm_filename)).with_context(|| {
format!("Failed to read {wasm_filename}. Run `tree-sitter build-wasm` first.",)
format!("Failed to read {wasm_filename}. Run `tree-sitter build --wasm` first.",)
})?;
Ok((grammar_name, contents))
}
@ -30,10 +33,12 @@ pub fn compile_language_to_wasm(
loader: &Loader,
language_dir: &Path,
output_dir: &Path,
output_file: Option<PathBuf>,
force_docker: bool,
) -> Result<()> {
let grammar_name = get_grammar_name(language_dir)?;
let output_filename = output_dir.join(format!("tree-sitter-{grammar_name}.wasm"));
let output_filename =
output_file.unwrap_or_else(|| output_dir.join(format!("tree-sitter-{grammar_name}.wasm")));
let src_path = language_dir.join("src");
let scanner_path = loader.get_scanner_path(&src_path);
loader.compile_parser_to_wasm(