diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4b3d145..922ba512 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,6 @@ on: env: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: 0 - EMSCRIPTEN_VERSION: 2.0.11 jobs: tests: @@ -35,6 +34,10 @@ jobs: run: | echo PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" >> $GITHUB_ENV + - name: Read Emscripten version + run: | + printf 'EMSCRIPTEN_VERSION=%s\n' "$(cat emscripten-version)" >> $GITHUB_ENV + - name: Cache artifacts id: cache uses: actions/cache@v2 diff --git a/cli/build.rs b/cli/build.rs index 37ea3b32..83be39a5 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -13,7 +13,13 @@ fn main() { let rust_binding_version = read_rust_binding_version(); println!( "cargo:rustc-env={}={}", - "RUST_BINDING_VERSION", rust_binding_version + "RUST_BINDING_VERSION", rust_binding_version, + ); + + let emscripten_version = fs::read_to_string("../emscripten-version").unwrap(); + println!( + "cargo:rustc-env={}={}", + "EMSCRIPTEN_VERSION", emscripten_version, ); } diff --git a/cli/config/src/lib.rs b/cli/config/src/lib.rs index cc52ea6e..94e857c1 100644 --- a/cli/config/src/lib.rs +++ b/cli/config/src/lib.rs @@ -20,7 +20,7 @@ pub struct Config { } impl Config { - fn find_config_file() -> Result> { + pub fn find_config_file() -> Result> { if let Ok(path) = env::var("TREE_SITTER_DIR") { let mut path = PathBuf::from(path); path.push("config.json"); diff --git a/cli/src/main.rs b/cli/src/main.rs index 353cc8a8..6d99ad85 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -36,13 +36,18 @@ fn run() -> Result<()> { }; let matches = App::new("tree-sitter") - .version(version.as_str()) - .setting(AppSettings::SubcommandRequiredElseHelp) .author("Max Brunsfeld ") .about("Generates and tests parsers") + .version(version.as_str()) + .setting(AppSettings::SubcommandRequiredElseHelp) + .global_setting(AppSettings::ColoredHelp) + .global_setting(AppSettings::DeriveDisplayOrder) + .global_setting(AppSettings::DisableHelpSubcommand) .subcommand(SubCommand::with_name("init-config").about("Generate a default config file")) .subcommand( SubCommand::with_name("generate") + .alias("gen") + .alias("g") .about("Generate a parser") .arg(Arg::with_name("grammar-path").index(1)) .arg(Arg::with_name("log").long("log")) @@ -58,6 +63,7 @@ fn run() -> Result<()> { ) .subcommand( SubCommand::with_name("parse") + .alias("p") .about("Parse files") .arg(Arg::with_name("paths-file").long("paths").takes_value(true)) .arg( @@ -85,6 +91,7 @@ fn run() -> Result<()> { ) .subcommand( SubCommand::with_name("query") + .alias("q") .about("Search files using a syntax tree query") .arg(Arg::with_name("query-path").index(1).required(true)) .arg(Arg::with_name("paths-file").long("paths").takes_value(true)) @@ -119,6 +126,7 @@ fn run() -> Result<()> { ) .subcommand( SubCommand::with_name("test") + .alias("t") .about("Run a parser's tests") .arg( Arg::with_name("filter") @@ -147,12 +155,13 @@ fn run() -> Result<()> { .required(false), ) .arg(Arg::with_name("scope").long("scope").takes_value(true)) - .arg(Arg::with_name("html").long("html").short("h")) + .arg(Arg::with_name("html").long("html").short("H")) .arg(Arg::with_name("time").long("time").short("t")) .arg(Arg::with_name("quiet").long("quiet").short("q")), ) .subcommand( SubCommand::with_name("build-wasm") + .alias("bw") .about("Compile a parser to WASM") .arg( Arg::with_name("docker") @@ -162,8 +171,11 @@ fn run() -> Result<()> { .arg(Arg::with_name("path").index(1).multiple(true)), ) .subcommand( - SubCommand::with_name("web-ui") - .about("Test a parser interactively in the browser") + SubCommand::with_name("playground") + .alias("play") + .alias("pg") + .alias("web-ui") + .about("Start local playground for a parser in the browser") .arg( Arg::with_name("quiet") .long("quiet") @@ -181,261 +193,290 @@ fn run() -> Result<()> { let config = Config::load()?; let mut loader = loader::Loader::new()?; - if matches.subcommand_matches("init-config").is_some() { - let mut config = Config::initial()?; - config.add(tree_sitter_loader::Config::initial())?; - config.add(tree_sitter_cli::highlight::ThemeConfig::default())?; - config.save()?; - println!( - "Saved initial configuration to {}", - config.location.display() - ); - } else if let Some(matches) = matches.subcommand_matches("generate") { - let grammar_path = matches.value_of("grammar-path"); - let report_symbol_name = matches.value_of("report-states-for-rule").or_else(|| { - if matches.is_present("report-states") { - Some("") - } else { - None + match matches.subcommand() { + ("init-config", Some(_)) => { + if let Ok(Some(config_path)) = Config::find_config_file() { + return Err(anyhow!( + "Remove your existing config file first: {}", + config_path.to_string_lossy() + )); } - }); - if matches.is_present("log") { - logger::init(); - } - let new_abi = !matches.is_present("prev-abi"); - let generate_bindings = !matches.is_present("no-bindings"); - generate::generate_parser_in_directory( - ¤t_dir, - grammar_path, - new_abi, - generate_bindings, - report_symbol_name, - )?; - } else if let Some(matches) = matches.subcommand_matches("test") { - let debug = matches.is_present("debug"); - let debug_graph = matches.is_present("debug-graph"); - let update = matches.is_present("update"); - let filter = matches.value_of("filter"); - let languages = loader.languages_at_path(¤t_dir)?; - let language = languages - .first() - .ok_or_else(|| anyhow!("No language found"))?; - let test_dir = current_dir.join("test"); - - // Run the corpus tests. Look for them at two paths: `test/corpus` and `corpus`. - let mut test_corpus_dir = test_dir.join("corpus"); - if !test_corpus_dir.is_dir() { - test_corpus_dir = current_dir.join("corpus"); - } - if test_corpus_dir.is_dir() { - test::run_tests_at_path( - *language, - &test_corpus_dir, - debug, - debug_graph, - filter, - update, - )?; - } - - // Check that all of the queries are valid. - test::check_queries_at_path(*language, ¤t_dir.join("queries"))?; - - // Run the syntax highlighting tests. - let test_highlight_dir = test_dir.join("highlight"); - if test_highlight_dir.is_dir() { - test_highlight::test_highlights(&loader, &test_highlight_dir)?; - } - } else if let Some(matches) = matches.subcommand_matches("parse") { - let debug = matches.is_present("debug"); - let debug_graph = matches.is_present("debug-graph"); - let debug_xml = matches.is_present("debug-xml"); - let quiet = matches.is_present("quiet"); - let time = matches.is_present("time"); - let edits = matches - .values_of("edits") - .map_or(Vec::new(), |e| e.collect()); - let cancellation_flag = util::cancel_on_stdin(); - - let timeout = matches - .value_of("timeout") - .map_or(0, |t| u64::from_str_radix(t, 10).unwrap()); - - let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; - - let max_path_length = paths.iter().map(|p| p.chars().count()).max().unwrap(); - let mut has_error = false; - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - - let should_track_stats = matches.is_present("stat"); - let mut stats = parse::Stats::default(); - - for path in paths { - let path = Path::new(&path); - let language = loader.select_language(path, ¤t_dir, matches.value_of("scope"))?; - - let this_file_errored = parse::parse_file_at_path( - language, - path, - &edits, - max_path_length, - quiet, - time, - timeout, - debug, - debug_graph, - debug_xml, - Some(&cancellation_flag), - )?; - - if should_track_stats { - stats.total_parses += 1; - if !this_file_errored { - stats.successful_parses += 1; - } - } - - has_error |= this_file_errored; - } - - if should_track_stats { - println!("{}", stats) - } - - if has_error { - return Err(anyhow!("")); - } - } else if let Some(matches) = matches.subcommand_matches("query") { - let ordered_captures = matches.values_of("captures").is_some(); - let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - let language = loader.select_language( - Path::new(&paths[0]), - ¤t_dir, - matches.value_of("scope"), - )?; - let query_path = Path::new(matches.value_of("query-path").unwrap()); - let range = matches.value_of("byte-range").map(|br| { - let r: Vec<&str> = br.split(":").collect(); - r[0].parse().unwrap()..r[1].parse().unwrap() - }); - let should_test = matches.is_present("test"); - query::query_files_at_paths( - language, - paths, - query_path, - ordered_captures, - range, - should_test, - )?; - } else if let Some(matches) = matches.subcommand_matches("tags") { - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; - tags::generate_tags( - &loader, - matches.value_of("scope"), - &paths, - matches.is_present("quiet"), - matches.is_present("time"), - )?; - } else if let Some(matches) = matches.subcommand_matches("highlight") { - let theme_config: tree_sitter_cli::highlight::ThemeConfig = config.get()?; - loader.configure_highlights(&theme_config.theme.highlight_names); - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - - let time = matches.is_present("time"); - let quiet = matches.is_present("quiet"); - let html_mode = quiet || matches.is_present("html"); - let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; - - if html_mode && !quiet { - println!("{}", highlight::HTML_HEADER); - } - - let cancellation_flag = util::cancel_on_stdin(); - - let mut lang = None; - if let Some(scope) = matches.value_of("scope") { - lang = loader.language_configuration_for_scope(scope)?; - if lang.is_none() { - return Err(anyhow!("Unknown scope '{}'", scope)); - } - } - - for path in paths { - let path = Path::new(&path); - let (language, language_config) = match lang { - Some(v) => v, - None => match loader.language_configuration_for_file_name(path)? { - Some(v) => v, - None => { - eprintln!("No language found for path {:?}", path); - continue; - } - }, - }; - - if let Some(highlight_config) = language_config.highlight_config(language)? { - let source = fs::read(path)?; - let theme_config = config.get()?; - if html_mode { - highlight::html( - &loader, - &theme_config, - &source, - highlight_config, - quiet, - time, - )?; - } else { - highlight::ansi( - &loader, - &theme_config, - &source, - highlight_config, - time, - Some(&cancellation_flag), - )?; - } - } else { - eprintln!("No syntax highlighting config found for path {:?}", path); - } - } - - if html_mode && !quiet { - println!("{}", highlight::HTML_FOOTER); - } - } else if let Some(matches) = matches.subcommand_matches("build-wasm") { - let grammar_path = current_dir.join(matches.value_of("path").unwrap_or("")); - wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?; - } else if let Some(matches) = matches.subcommand_matches("web-ui") { - let open_in_browser = !matches.is_present("quiet"); - web_ui::serve(¤t_dir, open_in_browser); - } else if matches.subcommand_matches("dump-languages").is_some() { - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - for (configuration, language_path) in loader.get_all_language_configurations() { + let mut config = Config::initial()?; + config.add(tree_sitter_loader::Config::initial())?; + config.add(tree_sitter_cli::highlight::ThemeConfig::default())?; + config.save()?; println!( - concat!( - "scope: {}\n", - "parser: {:?}\n", - "highlights: {:?}\n", - "file_types: {:?}\n", - "content_regex: {:?}\n", - "injection_regex: {:?}\n", - ), - configuration.scope.as_ref().unwrap_or(&String::new()), - language_path, - configuration.highlights_filenames, - configuration.file_types, - configuration.content_regex, - configuration.injection_regex, + "Saved initial configuration to {}", + config.location.display() ); } + + ("generate", Some(matches)) => { + let grammar_path = matches.value_of("grammar-path"); + let report_symbol_name = matches.value_of("report-states-for-rule").or_else(|| { + if matches.is_present("report-states") { + Some("") + } else { + None + } + }); + if matches.is_present("log") { + logger::init(); + } + let new_abi = !matches.is_present("prev-abi"); + let generate_bindings = !matches.is_present("no-bindings"); + generate::generate_parser_in_directory( + ¤t_dir, + grammar_path, + new_abi, + generate_bindings, + report_symbol_name, + )?; + } + + ("test", Some(matches)) => { + let debug = matches.is_present("debug"); + let debug_graph = matches.is_present("debug-graph"); + let update = matches.is_present("update"); + let filter = matches.value_of("filter"); + let languages = loader.languages_at_path(¤t_dir)?; + let language = languages + .first() + .ok_or_else(|| anyhow!("No language found"))?; + let test_dir = current_dir.join("test"); + + // Run the corpus tests. Look for them at two paths: `test/corpus` and `corpus`. + let mut test_corpus_dir = test_dir.join("corpus"); + if !test_corpus_dir.is_dir() { + test_corpus_dir = current_dir.join("corpus"); + } + if test_corpus_dir.is_dir() { + test::run_tests_at_path( + *language, + &test_corpus_dir, + debug, + debug_graph, + filter, + update, + )?; + } + + // Check that all of the queries are valid. + test::check_queries_at_path(*language, ¤t_dir.join("queries"))?; + + // Run the syntax highlighting tests. + let test_highlight_dir = test_dir.join("highlight"); + if test_highlight_dir.is_dir() { + test_highlight::test_highlights(&loader, &test_highlight_dir)?; + } + } + + ("parse", Some(matches)) => { + let debug = matches.is_present("debug"); + let debug_graph = matches.is_present("debug-graph"); + let debug_xml = matches.is_present("debug-xml"); + let quiet = matches.is_present("quiet"); + let time = matches.is_present("time"); + let edits = matches + .values_of("edits") + .map_or(Vec::new(), |e| e.collect()); + let cancellation_flag = util::cancel_on_stdin(); + + let timeout = matches + .value_of("timeout") + .map_or(0, |t| u64::from_str_radix(t, 10).unwrap()); + + let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; + + let max_path_length = paths.iter().map(|p| p.chars().count()).max().unwrap_or(0); + let mut has_error = false; + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + + let should_track_stats = matches.is_present("stat"); + let mut stats = parse::Stats::default(); + + for path in paths { + let path = Path::new(&path); + let language = + loader.select_language(path, ¤t_dir, matches.value_of("scope"))?; + + let this_file_errored = parse::parse_file_at_path( + language, + path, + &edits, + max_path_length, + quiet, + time, + timeout, + debug, + debug_graph, + debug_xml, + Some(&cancellation_flag), + )?; + + if should_track_stats { + stats.total_parses += 1; + if !this_file_errored { + stats.successful_parses += 1; + } + } + + has_error |= this_file_errored; + } + + if should_track_stats { + println!("{}", stats) + } + + if has_error { + return Err(anyhow!("")); + } + } + + ("query", Some(matches)) => { + let ordered_captures = matches.values_of("captures").is_some(); + let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + let language = loader.select_language( + Path::new(&paths[0]), + ¤t_dir, + matches.value_of("scope"), + )?; + let query_path = Path::new(matches.value_of("query-path").unwrap()); + let range = matches.value_of("byte-range").map(|br| { + let r: Vec<&str> = br.split(":").collect(); + r[0].parse().unwrap()..r[1].parse().unwrap() + }); + let should_test = matches.is_present("test"); + query::query_files_at_paths( + language, + paths, + query_path, + ordered_captures, + range, + should_test, + )?; + } + + ("tags", Some(matches)) => { + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; + tags::generate_tags( + &loader, + matches.value_of("scope"), + &paths, + matches.is_present("quiet"), + matches.is_present("time"), + )?; + } + + ("highlight", Some(matches)) => { + let theme_config: tree_sitter_cli::highlight::ThemeConfig = config.get()?; + loader.configure_highlights(&theme_config.theme.highlight_names); + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + + let time = matches.is_present("time"); + let quiet = matches.is_present("quiet"); + let html_mode = quiet || matches.is_present("html"); + let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; + + if html_mode && !quiet { + println!("{}", highlight::HTML_HEADER); + } + + let cancellation_flag = util::cancel_on_stdin(); + + let mut lang = None; + if let Some(scope) = matches.value_of("scope") { + lang = loader.language_configuration_for_scope(scope)?; + if lang.is_none() { + return Err(anyhow!("Unknown scope '{}'", scope)); + } + } + + for path in paths { + let path = Path::new(&path); + let (language, language_config) = match lang { + Some(v) => v, + None => match loader.language_configuration_for_file_name(path)? { + Some(v) => v, + None => { + eprintln!("No language found for path {:?}", path); + continue; + } + }, + }; + + if let Some(highlight_config) = language_config.highlight_config(language)? { + let source = fs::read(path)?; + let theme_config = config.get()?; + if html_mode { + highlight::html( + &loader, + &theme_config, + &source, + highlight_config, + quiet, + time, + )?; + } else { + highlight::ansi( + &loader, + &theme_config, + &source, + highlight_config, + time, + Some(&cancellation_flag), + )?; + } + } else { + eprintln!("No syntax highlighting config found for path {:?}", path); + } + } + + if html_mode && !quiet { + println!("{}", highlight::HTML_FOOTER); + } + } + + ("build-wasm", Some(matches)) => { + let grammar_path = current_dir.join(matches.value_of("path").unwrap_or("")); + wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?; + } + + ("playground", Some(matches)) => { + let open_in_browser = !matches.is_present("quiet"); + web_ui::serve(¤t_dir, open_in_browser); + } + + ("dump-languages", Some(_)) => { + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + for (configuration, language_path) in loader.get_all_language_configurations() { + println!( + concat!( + "scope: {}\n", + "parser: {:?}\n", + "highlights: {:?}\n", + "file_types: {:?}\n", + "content_regex: {:?}\n", + "injection_regex: {:?}\n", + ), + configuration.scope.as_ref().unwrap_or(&String::new()), + language_path, + configuration.highlights_filenames, + configuration.file_types, + configuration.content_regex, + configuration.injection_regex, + ); + } + } + + _ => unreachable!(), } Ok(()) diff --git a/cli/src/parse.rs b/cli/src/parse.rs index 904ee396..15a9d4c9 100644 --- a/cli/src/parse.rs +++ b/cli/src/parse.rs @@ -311,7 +311,9 @@ fn parse_edit_flag(source_code: &Vec, flag: &str) -> Result { let inserted_text = parts.collect::>().join(" ").into_bytes(); // Position can either be a byte_offset or row,column pair, separated by a comma - let position = if position.contains(",") { + let position = if position == "$" { + source_code.len() + } else if position.contains(",") { let mut parts = position.split(","); let row = parts.next().ok_or_else(error)?; let row = usize::from_str_radix(row, 10).map_err(|_| error())?; diff --git a/cli/src/wasm.rs b/cli/src/wasm.rs index 1abbb042..9689245d 100644 --- a/cli/src/wasm.rs +++ b/cli/src/wasm.rs @@ -6,6 +6,8 @@ use std::path::Path; use std::process::Command; use which::which; +const EMSCRIPTEN_TAG: &'static str = concat!("emscripten/emsdk:", env!("EMSCRIPTEN_VERSION")); + pub fn get_grammar_name(src_dir: &Path) -> Result { let grammar_json_path = src_dir.join("grammar.json"); let grammar_json = fs::read_to_string(&grammar_json_path) @@ -61,7 +63,7 @@ pub fn compile_language_to_wasm(language_dir: &Path, force_docker: bool) -> Resu } // Run `emcc` in a container using the `emscripten-slim` image - command.args(&["emscripten/emsdk", "emcc"]); + command.args(&[EMSCRIPTEN_TAG, "emcc"]); } else { return Err(anyhow!( "You must have either emcc or docker on your PATH to run this command" diff --git a/emscripten-version b/emscripten-version new file mode 100644 index 00000000..da37822f --- /dev/null +++ b/emscripten-version @@ -0,0 +1 @@ +2.0.11 \ No newline at end of file diff --git a/script/build-wasm b/script/build-wasm index 1b5e48ec..201d0b50 100755 --- a/script/build-wasm +++ b/script/build-wasm @@ -33,6 +33,7 @@ web_dir=lib/binding_web emscripten_flags="-O3" minify_js=1 force_docker=0 +emscripen_version=$(cat "$(dirname "$0")"/../emscripten-version) while [[ $# > 0 ]]; do case "$1" in @@ -63,11 +64,11 @@ emcc= if which emcc > /dev/null && [[ "$force_docker" == "0" ]]; then emcc=emcc elif which docker > /dev/null; then - emcc="docker run \ - --rm \ - -v $(pwd):/src:Z \ - -u $(id -u) \ - emscripten/emsdk \ + emcc="docker run \ + --rm \ + -v $(pwd):/src:Z \ + -u $(id -u) \ + emscripten/emsdk:$emscripen_version \ emcc" else echo 'You must have either `docker` or `emcc` on your PATH to run this script' diff --git a/script/fetch-emscripten b/script/fetch-emscripten index c1b072ad..157d0cae 100755 --- a/script/fetch-emscripten +++ b/script/fetch-emscripten @@ -2,7 +2,7 @@ set -e -EMSCRIPTEN_VERSION=2.0.9 +EMSCRIPTEN_VERSION=$(cat "$(dirname "$0")/../emscripten-version") mkdir -p target EMSDK_DIR="./target/emsdk"