From 421815a68d69b8a3e9215a95b62a038272c2e76a Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Wed, 23 Jun 2021 10:51:55 +0300 Subject: [PATCH 01/12] feat(cli): Make more clearer sub command selection --- cli/src/main.rs | 527 +++++++++++++++++++++++++----------------------- 1 file changed, 275 insertions(+), 252 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 353cc8a8..53598bc6 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -181,261 +181,284 @@ fn run() -> Result<()> { let config = Config::load()?; let mut loader = loader::Loader::new()?; - if matches.subcommand_matches("init-config").is_some() { - let mut config = Config::initial()?; - config.add(tree_sitter_loader::Config::initial())?; - config.add(tree_sitter_cli::highlight::ThemeConfig::default())?; - config.save()?; - println!( - "Saved initial configuration to {}", - config.location.display() - ); - } else if let Some(matches) = matches.subcommand_matches("generate") { - let grammar_path = matches.value_of("grammar-path"); - let report_symbol_name = matches.value_of("report-states-for-rule").or_else(|| { - if matches.is_present("report-states") { - Some("") - } else { - None - } - }); - if matches.is_present("log") { - logger::init(); - } - let new_abi = !matches.is_present("prev-abi"); - let generate_bindings = !matches.is_present("no-bindings"); - generate::generate_parser_in_directory( - ¤t_dir, - grammar_path, - new_abi, - generate_bindings, - report_symbol_name, - )?; - } else if let Some(matches) = matches.subcommand_matches("test") { - let debug = matches.is_present("debug"); - let debug_graph = matches.is_present("debug-graph"); - let update = matches.is_present("update"); - let filter = matches.value_of("filter"); - let languages = loader.languages_at_path(¤t_dir)?; - let language = languages - .first() - .ok_or_else(|| anyhow!("No language found"))?; - let test_dir = current_dir.join("test"); - - // Run the corpus tests. Look for them at two paths: `test/corpus` and `corpus`. - let mut test_corpus_dir = test_dir.join("corpus"); - if !test_corpus_dir.is_dir() { - test_corpus_dir = current_dir.join("corpus"); - } - if test_corpus_dir.is_dir() { - test::run_tests_at_path( - *language, - &test_corpus_dir, - debug, - debug_graph, - filter, - update, - )?; - } - - // Check that all of the queries are valid. - test::check_queries_at_path(*language, ¤t_dir.join("queries"))?; - - // Run the syntax highlighting tests. - let test_highlight_dir = test_dir.join("highlight"); - if test_highlight_dir.is_dir() { - test_highlight::test_highlights(&loader, &test_highlight_dir)?; - } - } else if let Some(matches) = matches.subcommand_matches("parse") { - let debug = matches.is_present("debug"); - let debug_graph = matches.is_present("debug-graph"); - let debug_xml = matches.is_present("debug-xml"); - let quiet = matches.is_present("quiet"); - let time = matches.is_present("time"); - let edits = matches - .values_of("edits") - .map_or(Vec::new(), |e| e.collect()); - let cancellation_flag = util::cancel_on_stdin(); - - let timeout = matches - .value_of("timeout") - .map_or(0, |t| u64::from_str_radix(t, 10).unwrap()); - - let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; - - let max_path_length = paths.iter().map(|p| p.chars().count()).max().unwrap(); - let mut has_error = false; - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - - let should_track_stats = matches.is_present("stat"); - let mut stats = parse::Stats::default(); - - for path in paths { - let path = Path::new(&path); - let language = loader.select_language(path, ¤t_dir, matches.value_of("scope"))?; - - let this_file_errored = parse::parse_file_at_path( - language, - path, - &edits, - max_path_length, - quiet, - time, - timeout, - debug, - debug_graph, - debug_xml, - Some(&cancellation_flag), - )?; - - if should_track_stats { - stats.total_parses += 1; - if !this_file_errored { - stats.successful_parses += 1; - } - } - - has_error |= this_file_errored; - } - - if should_track_stats { - println!("{}", stats) - } - - if has_error { - return Err(anyhow!("")); - } - } else if let Some(matches) = matches.subcommand_matches("query") { - let ordered_captures = matches.values_of("captures").is_some(); - let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - let language = loader.select_language( - Path::new(&paths[0]), - ¤t_dir, - matches.value_of("scope"), - )?; - let query_path = Path::new(matches.value_of("query-path").unwrap()); - let range = matches.value_of("byte-range").map(|br| { - let r: Vec<&str> = br.split(":").collect(); - r[0].parse().unwrap()..r[1].parse().unwrap() - }); - let should_test = matches.is_present("test"); - query::query_files_at_paths( - language, - paths, - query_path, - ordered_captures, - range, - should_test, - )?; - } else if let Some(matches) = matches.subcommand_matches("tags") { - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; - tags::generate_tags( - &loader, - matches.value_of("scope"), - &paths, - matches.is_present("quiet"), - matches.is_present("time"), - )?; - } else if let Some(matches) = matches.subcommand_matches("highlight") { - let theme_config: tree_sitter_cli::highlight::ThemeConfig = config.get()?; - loader.configure_highlights(&theme_config.theme.highlight_names); - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - - let time = matches.is_present("time"); - let quiet = matches.is_present("quiet"); - let html_mode = quiet || matches.is_present("html"); - let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; - - if html_mode && !quiet { - println!("{}", highlight::HTML_HEADER); - } - - let cancellation_flag = util::cancel_on_stdin(); - - let mut lang = None; - if let Some(scope) = matches.value_of("scope") { - lang = loader.language_configuration_for_scope(scope)?; - if lang.is_none() { - return Err(anyhow!("Unknown scope '{}'", scope)); - } - } - - for path in paths { - let path = Path::new(&path); - let (language, language_config) = match lang { - Some(v) => v, - None => match loader.language_configuration_for_file_name(path)? { - Some(v) => v, - None => { - eprintln!("No language found for path {:?}", path); - continue; - } - }, - }; - - if let Some(highlight_config) = language_config.highlight_config(language)? { - let source = fs::read(path)?; - let theme_config = config.get()?; - if html_mode { - highlight::html( - &loader, - &theme_config, - &source, - highlight_config, - quiet, - time, - )?; - } else { - highlight::ansi( - &loader, - &theme_config, - &source, - highlight_config, - time, - Some(&cancellation_flag), - )?; - } - } else { - eprintln!("No syntax highlighting config found for path {:?}", path); - } - } - - if html_mode && !quiet { - println!("{}", highlight::HTML_FOOTER); - } - } else if let Some(matches) = matches.subcommand_matches("build-wasm") { - let grammar_path = current_dir.join(matches.value_of("path").unwrap_or("")); - wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?; - } else if let Some(matches) = matches.subcommand_matches("web-ui") { - let open_in_browser = !matches.is_present("quiet"); - web_ui::serve(¤t_dir, open_in_browser); - } else if matches.subcommand_matches("dump-languages").is_some() { - let loader_config = config.get()?; - loader.find_all_languages(&loader_config)?; - for (configuration, language_path) in loader.get_all_language_configurations() { + match matches.subcommand() { + ("init-config", Some(_)) => { + let mut config = Config::initial()?; + config.add(tree_sitter_loader::Config::initial())?; + config.add(tree_sitter_cli::highlight::ThemeConfig::default())?; + config.save()?; println!( - concat!( - "scope: {}\n", - "parser: {:?}\n", - "highlights: {:?}\n", - "file_types: {:?}\n", - "content_regex: {:?}\n", - "injection_regex: {:?}\n", - ), - configuration.scope.as_ref().unwrap_or(&String::new()), - language_path, - configuration.highlights_filenames, - configuration.file_types, - configuration.content_regex, - configuration.injection_regex, + "Saved initial configuration to {}", + config.location.display() ); } + + ("generate", Some(matches)) => { + let grammar_path = matches.value_of("grammar-path"); + let report_symbol_name = matches.value_of("report-states-for-rule").or_else(|| { + if matches.is_present("report-states") { + Some("") + } else { + None + } + }); + if matches.is_present("log") { + logger::init(); + } + let new_abi = !matches.is_present("prev-abi"); + let generate_bindings = !matches.is_present("no-bindings"); + generate::generate_parser_in_directory( + ¤t_dir, + grammar_path, + new_abi, + generate_bindings, + report_symbol_name, + )?; + } + + ("test", Some(matches)) => { + let debug = matches.is_present("debug"); + let debug_graph = matches.is_present("debug-graph"); + let update = matches.is_present("update"); + let filter = matches.value_of("filter"); + let languages = loader.languages_at_path(¤t_dir)?; + let language = languages + .first() + .ok_or_else(|| anyhow!("No language found"))?; + let test_dir = current_dir.join("test"); + + // Run the corpus tests. Look for them at two paths: `test/corpus` and `corpus`. + let mut test_corpus_dir = test_dir.join("corpus"); + if !test_corpus_dir.is_dir() { + test_corpus_dir = current_dir.join("corpus"); + } + if test_corpus_dir.is_dir() { + test::run_tests_at_path( + *language, + &test_corpus_dir, + debug, + debug_graph, + filter, + update, + )?; + } + + // Check that all of the queries are valid. + test::check_queries_at_path(*language, ¤t_dir.join("queries"))?; + + // Run the syntax highlighting tests. + let test_highlight_dir = test_dir.join("highlight"); + if test_highlight_dir.is_dir() { + test_highlight::test_highlights(&loader, &test_highlight_dir)?; + } + } + + ("parse", Some(matches)) => { + let debug = matches.is_present("debug"); + let debug_graph = matches.is_present("debug-graph"); + let debug_xml = matches.is_present("debug-xml"); + let quiet = matches.is_present("quiet"); + let time = matches.is_present("time"); + let edits = matches + .values_of("edits") + .map_or(Vec::new(), |e| e.collect()); + let cancellation_flag = util::cancel_on_stdin(); + + let timeout = matches + .value_of("timeout") + .map_or(0, |t| u64::from_str_radix(t, 10).unwrap()); + + let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; + + let max_path_length = paths.iter().map(|p| p.chars().count()).max().unwrap(); + let mut has_error = false; + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + + let should_track_stats = matches.is_present("stat"); + let mut stats = parse::Stats::default(); + + for path in paths { + let path = Path::new(&path); + let language = + loader.select_language(path, ¤t_dir, matches.value_of("scope"))?; + + let this_file_errored = parse::parse_file_at_path( + language, + path, + &edits, + max_path_length, + quiet, + time, + timeout, + debug, + debug_graph, + debug_xml, + Some(&cancellation_flag), + )?; + + if should_track_stats { + stats.total_parses += 1; + if !this_file_errored { + stats.successful_parses += 1; + } + } + + has_error |= this_file_errored; + } + + if should_track_stats { + println!("{}", stats) + } + + if has_error { + return Err(anyhow!("")); + } + } + + ("query", Some(matches)) => { + let ordered_captures = matches.values_of("captures").is_some(); + let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + let language = loader.select_language( + Path::new(&paths[0]), + ¤t_dir, + matches.value_of("scope"), + )?; + let query_path = Path::new(matches.value_of("query-path").unwrap()); + let range = matches.value_of("byte-range").map(|br| { + let r: Vec<&str> = br.split(":").collect(); + r[0].parse().unwrap()..r[1].parse().unwrap() + }); + let should_test = matches.is_present("test"); + query::query_files_at_paths( + language, + paths, + query_path, + ordered_captures, + range, + should_test, + )?; + } + + ("tags", Some(matches)) => { + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; + tags::generate_tags( + &loader, + matches.value_of("scope"), + &paths, + matches.is_present("quiet"), + matches.is_present("time"), + )?; + } + + ("highlight", Some(matches)) => { + let theme_config: tree_sitter_cli::highlight::ThemeConfig = config.get()?; + loader.configure_highlights(&theme_config.theme.highlight_names); + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + + let time = matches.is_present("time"); + let quiet = matches.is_present("quiet"); + let html_mode = quiet || matches.is_present("html"); + let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; + + if html_mode && !quiet { + println!("{}", highlight::HTML_HEADER); + } + + let cancellation_flag = util::cancel_on_stdin(); + + let mut lang = None; + if let Some(scope) = matches.value_of("scope") { + lang = loader.language_configuration_for_scope(scope)?; + if lang.is_none() { + return Err(anyhow!("Unknown scope '{}'", scope)); + } + } + + for path in paths { + let path = Path::new(&path); + let (language, language_config) = match lang { + Some(v) => v, + None => match loader.language_configuration_for_file_name(path)? { + Some(v) => v, + None => { + eprintln!("No language found for path {:?}", path); + continue; + } + }, + }; + + if let Some(highlight_config) = language_config.highlight_config(language)? { + let source = fs::read(path)?; + let theme_config = config.get()?; + if html_mode { + highlight::html( + &loader, + &theme_config, + &source, + highlight_config, + quiet, + time, + )?; + } else { + highlight::ansi( + &loader, + &theme_config, + &source, + highlight_config, + time, + Some(&cancellation_flag), + )?; + } + } else { + eprintln!("No syntax highlighting config found for path {:?}", path); + } + } + + if html_mode && !quiet { + println!("{}", highlight::HTML_FOOTER); + } + } + + ("build-wasm", Some(matches)) => { + let grammar_path = current_dir.join(matches.value_of("path").unwrap_or("")); + wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?; + } + + ("web-ui", Some(matches)) => { + let open_in_browser = !matches.is_present("quiet"); + web_ui::serve(¤t_dir, open_in_browser); + } + + ("dump-languages", Some(_)) => { + let loader_config = config.get()?; + loader.find_all_languages(&loader_config)?; + for (configuration, language_path) in loader.get_all_language_configurations() { + println!( + concat!( + "scope: {}\n", + "parser: {:?}\n", + "highlights: {:?}\n", + "file_types: {:?}\n", + "content_regex: {:?}\n", + "injection_regex: {:?}\n", + ), + configuration.scope.as_ref().unwrap_or(&String::new()), + language_path, + configuration.highlights_filenames, + configuration.file_types, + configuration.content_regex, + configuration.injection_regex, + ); + } + } + + _ => unreachable!(), } Ok(()) From 71c43a869b367ac28972df6895d9fffb9c9cd552 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Wed, 23 Jun 2021 10:04:29 -0400 Subject: [PATCH 02/12] Allow `$` in `tree-sitter parse --edit` You can now specify `$` as the position to apply an edit, signifying the end of the file. (That prevents you from having to calculate the size of the file yourself.) --- cli/src/parse.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/src/parse.rs b/cli/src/parse.rs index 904ee396..15a9d4c9 100644 --- a/cli/src/parse.rs +++ b/cli/src/parse.rs @@ -311,7 +311,9 @@ fn parse_edit_flag(source_code: &Vec, flag: &str) -> Result { let inserted_text = parts.collect::>().join(" ").into_bytes(); // Position can either be a byte_offset or row,column pair, separated by a comma - let position = if position.contains(",") { + let position = if position == "$" { + source_code.len() + } else if position.contains(",") { let mut parts = position.split(","); let row = parts.next().ok_or_else(error)?; let row = usize::from_str_radix(row, 10).map_err(|_| error())?; From 04badf8a8ed0f20239a9bc9f7c6b932167b005e9 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Wed, 23 Jun 2021 10:56:23 +0300 Subject: [PATCH 03/12] feat(cli): Enable clap colored help --- cli/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/src/main.rs b/cli/src/main.rs index 53598bc6..718dcd02 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -40,6 +40,7 @@ fn run() -> Result<()> { .setting(AppSettings::SubcommandRequiredElseHelp) .author("Max Brunsfeld ") .about("Generates and tests parsers") + .global_setting(AppSettings::ColoredHelp) .subcommand(SubCommand::with_name("init-config").about("Generate a default config file")) .subcommand( SubCommand::with_name("generate") From 80ee33b073f79d1c3cbf2afe9cef097469b79233 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Wed, 23 Jun 2021 10:57:26 +0300 Subject: [PATCH 04/12] feat(cli): Add short aliases for frequent subcommands --- cli/src/main.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cli/src/main.rs b/cli/src/main.rs index 718dcd02..c0615d9a 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -44,6 +44,8 @@ fn run() -> Result<()> { .subcommand(SubCommand::with_name("init-config").about("Generate a default config file")) .subcommand( SubCommand::with_name("generate") + .alias("gen") + .alias("g") .about("Generate a parser") .arg(Arg::with_name("grammar-path").index(1)) .arg(Arg::with_name("log").long("log")) @@ -59,6 +61,7 @@ fn run() -> Result<()> { ) .subcommand( SubCommand::with_name("parse") + .alias("p") .about("Parse files") .arg(Arg::with_name("paths-file").long("paths").takes_value(true)) .arg( @@ -86,6 +89,7 @@ fn run() -> Result<()> { ) .subcommand( SubCommand::with_name("query") + .alias("q") .about("Search files using a syntax tree query") .arg(Arg::with_name("query-path").index(1).required(true)) .arg(Arg::with_name("paths-file").long("paths").takes_value(true)) @@ -120,6 +124,7 @@ fn run() -> Result<()> { ) .subcommand( SubCommand::with_name("test") + .alias("t") .about("Run a parser's tests") .arg( Arg::with_name("filter") @@ -154,6 +159,7 @@ fn run() -> Result<()> { ) .subcommand( SubCommand::with_name("build-wasm") + .alias("bw") .about("Compile a parser to WASM") .arg( Arg::with_name("docker") From 1220ec3852d17b203bb91b14481f6a66304d5316 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Wed, 23 Jun 2021 10:58:01 +0300 Subject: [PATCH 05/12] feat(cli): Rename web-ui -> playground with fallback alias --- cli/src/main.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index c0615d9a..6334d916 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -169,8 +169,11 @@ fn run() -> Result<()> { .arg(Arg::with_name("path").index(1).multiple(true)), ) .subcommand( - SubCommand::with_name("web-ui") - .about("Test a parser interactively in the browser") + SubCommand::with_name("playground") + .alias("play") + .alias("pg") + .alias("web-ui") + .about("Start local playground for a parser in the browser") .arg( Arg::with_name("quiet") .long("quiet") @@ -437,7 +440,7 @@ fn run() -> Result<()> { wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?; } - ("web-ui", Some(matches)) => { + ("playground", Some(matches)) => { let open_in_browser = !matches.is_present("quiet"); web_ui::serve(¤t_dir, open_in_browser); } From 650ee948004c2a448dfcf19d4241be6f643498ba Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Wed, 23 Jun 2021 11:16:41 +0300 Subject: [PATCH 06/12] fix(cli): disable help subcommand, -h, --help are enough --- cli/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/src/main.rs b/cli/src/main.rs index 6334d916..af25c2fd 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -41,6 +41,7 @@ fn run() -> Result<()> { .author("Max Brunsfeld ") .about("Generates and tests parsers") .global_setting(AppSettings::ColoredHelp) + .global_setting(AppSettings::DisableHelpSubcommand) .subcommand(SubCommand::with_name("init-config").about("Generate a default config file")) .subcommand( SubCommand::with_name("generate") From bd0e60240eccf91b4573e1a268d8d12a98741e67 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Thu, 24 Jun 2021 12:03:59 +0300 Subject: [PATCH 07/12] fix(cli): Change -h -> -H for the highlight subcommand to don't clash with the help flag --- cli/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index af25c2fd..983fd1e4 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -154,7 +154,7 @@ fn run() -> Result<()> { .required(false), ) .arg(Arg::with_name("scope").long("scope").takes_value(true)) - .arg(Arg::with_name("html").long("html").short("h")) + .arg(Arg::with_name("html").long("html").short("H")) .arg(Arg::with_name("time").long("time").short("t")) .arg(Arg::with_name("quiet").long("quiet").short("q")), ) From 725f3f7f2b7da6f71fb4254445bc300ba7681025 Mon Sep 17 00:00:00 2001 From: Vladimir Panteleev Date: Fri, 18 Jun 2021 11:04:18 +0000 Subject: [PATCH 08/12] Pin emscripten/emsdk Docker version Fixes issues caused by incompatible changes in Emscripten since the time that tree-sitter was built. --- cli/src/wasm.rs | 2 +- script/build-wasm | 10 +++++----- script/fetch-emscripten | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/src/wasm.rs b/cli/src/wasm.rs index 1abbb042..c93eb246 100644 --- a/cli/src/wasm.rs +++ b/cli/src/wasm.rs @@ -61,7 +61,7 @@ pub fn compile_language_to_wasm(language_dir: &Path, force_docker: bool) -> Resu } // Run `emcc` in a container using the `emscripten-slim` image - command.args(&["emscripten/emsdk", "emcc"]); + command.args(&["emscripten/emsdk:2.0.11", "emcc"]); } else { return Err(anyhow!( "You must have either emcc or docker on your PATH to run this command" diff --git a/script/build-wasm b/script/build-wasm index 1b5e48ec..d33dd4b5 100755 --- a/script/build-wasm +++ b/script/build-wasm @@ -63,11 +63,11 @@ emcc= if which emcc > /dev/null && [[ "$force_docker" == "0" ]]; then emcc=emcc elif which docker > /dev/null; then - emcc="docker run \ - --rm \ - -v $(pwd):/src:Z \ - -u $(id -u) \ - emscripten/emsdk \ + emcc="docker run \ + --rm \ + -v $(pwd):/src:Z \ + -u $(id -u) \ + emscripten/emsdk:2.0.11 \ emcc" else echo 'You must have either `docker` or `emcc` on your PATH to run this script' diff --git a/script/fetch-emscripten b/script/fetch-emscripten index c1b072ad..e2817ad8 100755 --- a/script/fetch-emscripten +++ b/script/fetch-emscripten @@ -2,7 +2,7 @@ set -e -EMSCRIPTEN_VERSION=2.0.9 +EMSCRIPTEN_VERSION=2.0.11 mkdir -p target EMSDK_DIR="./target/emsdk" From 5adf7bdcdeca291b03260536f3f5a505611c9218 Mon Sep 17 00:00:00 2001 From: Vladimir Panteleev Date: Sat, 26 Jun 2021 18:10:09 +0000 Subject: [PATCH 09/12] cli/src/main.rs: Fix panic with empty file list to parse E.g.: tree-sitter parse --paths /dev/null --- cli/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 983fd1e4..97247fd8 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -281,7 +281,7 @@ fn run() -> Result<()> { let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; - let max_path_length = paths.iter().map(|p| p.chars().count()).max().unwrap(); + let max_path_length = paths.iter().map(|p| p.chars().count()).max().unwrap_or(0); let mut has_error = false; let loader_config = config.get()?; loader.find_all_languages(&loader_config)?; From 2e2b239c42dc8b73e62b037d53f76de68221cfe8 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Sun, 27 Jun 2021 03:03:47 +0300 Subject: [PATCH 10/12] fix(cli): Don't silently overwrite existing config file --- cli/config/src/lib.rs | 2 +- cli/src/main.rs | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/config/src/lib.rs b/cli/config/src/lib.rs index cc52ea6e..94e857c1 100644 --- a/cli/config/src/lib.rs +++ b/cli/config/src/lib.rs @@ -20,7 +20,7 @@ pub struct Config { } impl Config { - fn find_config_file() -> Result> { + pub fn find_config_file() -> Result> { if let Ok(path) = env::var("TREE_SITTER_DIR") { let mut path = PathBuf::from(path); path.push("config.json"); diff --git a/cli/src/main.rs b/cli/src/main.rs index 97247fd8..8e2b2eb8 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -194,6 +194,12 @@ fn run() -> Result<()> { match matches.subcommand() { ("init-config", Some(_)) => { + if let Ok(Some(config_path)) = Config::find_config_file() { + return Err(anyhow!( + "Remove your existing config file first: {}", + config_path.to_string_lossy() + )); + } let mut config = Config::initial()?; config.add(tree_sitter_loader::Config::initial())?; config.add(tree_sitter_cli::highlight::ThemeConfig::default())?; From 202a37f1a141d4a2328a37609da7dae88ff2db7d Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Sun, 27 Jun 2021 03:06:15 +0300 Subject: [PATCH 11/12] chore(cli): Use DeriveDisplayOrder Clap's setting To show sub commands in the defined order that looks more accurate and have more meaning for sub commands use order. --- cli/src/main.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 8e2b2eb8..6d99ad85 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -36,11 +36,12 @@ fn run() -> Result<()> { }; let matches = App::new("tree-sitter") - .version(version.as_str()) - .setting(AppSettings::SubcommandRequiredElseHelp) .author("Max Brunsfeld ") .about("Generates and tests parsers") + .version(version.as_str()) + .setting(AppSettings::SubcommandRequiredElseHelp) .global_setting(AppSettings::ColoredHelp) + .global_setting(AppSettings::DeriveDisplayOrder) .global_setting(AppSettings::DisableHelpSubcommand) .subcommand(SubCommand::with_name("init-config").about("Generate a default config file")) .subcommand( From b14ea51e3df4f5614d8913513a4d1eed8be07d71 Mon Sep 17 00:00:00 2001 From: Vladimir Panteleev Date: Tue, 29 Jun 2021 21:11:21 +0000 Subject: [PATCH 12/12] Refactor emscripten/emsdk version to a single file --- .github/workflows/ci.yml | 5 ++++- cli/build.rs | 6 ++++++ cli/src/wasm.rs | 4 +++- emscripten-version | 1 + script/build-wasm | 11 ++++++----- script/fetch-emscripten | 2 +- 6 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 emscripten-version diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4b3d145..922ba512 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,6 @@ on: env: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: 0 - EMSCRIPTEN_VERSION: 2.0.11 jobs: tests: @@ -35,6 +34,10 @@ jobs: run: | echo PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" >> $GITHUB_ENV + - name: Read Emscripten version + run: | + printf 'EMSCRIPTEN_VERSION=%s\n' "$(cat emscripten-version)" >> $GITHUB_ENV + - name: Cache artifacts id: cache uses: actions/cache@v2 diff --git a/cli/build.rs b/cli/build.rs index 322d239b..29017e11 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -9,6 +9,12 @@ fn main() { if wasm_files_present() { println!("cargo:rustc-cfg={}", "TREE_SITTER_EMBED_WASM_BINDING"); } + + let emscripten_version = fs::read_to_string("../emscripten-version").unwrap(); + println!( + "cargo:rustc-env={}={}", + "EMSCRIPTEN_VERSION", emscripten_version + ); } fn wasm_files_present() -> bool { diff --git a/cli/src/wasm.rs b/cli/src/wasm.rs index c93eb246..9689245d 100644 --- a/cli/src/wasm.rs +++ b/cli/src/wasm.rs @@ -6,6 +6,8 @@ use std::path::Path; use std::process::Command; use which::which; +const EMSCRIPTEN_TAG: &'static str = concat!("emscripten/emsdk:", env!("EMSCRIPTEN_VERSION")); + pub fn get_grammar_name(src_dir: &Path) -> Result { let grammar_json_path = src_dir.join("grammar.json"); let grammar_json = fs::read_to_string(&grammar_json_path) @@ -61,7 +63,7 @@ pub fn compile_language_to_wasm(language_dir: &Path, force_docker: bool) -> Resu } // Run `emcc` in a container using the `emscripten-slim` image - command.args(&["emscripten/emsdk:2.0.11", "emcc"]); + command.args(&[EMSCRIPTEN_TAG, "emcc"]); } else { return Err(anyhow!( "You must have either emcc or docker on your PATH to run this command" diff --git a/emscripten-version b/emscripten-version new file mode 100644 index 00000000..da37822f --- /dev/null +++ b/emscripten-version @@ -0,0 +1 @@ +2.0.11 \ No newline at end of file diff --git a/script/build-wasm b/script/build-wasm index d33dd4b5..201d0b50 100755 --- a/script/build-wasm +++ b/script/build-wasm @@ -33,6 +33,7 @@ web_dir=lib/binding_web emscripten_flags="-O3" minify_js=1 force_docker=0 +emscripen_version=$(cat "$(dirname "$0")"/../emscripten-version) while [[ $# > 0 ]]; do case "$1" in @@ -63,11 +64,11 @@ emcc= if which emcc > /dev/null && [[ "$force_docker" == "0" ]]; then emcc=emcc elif which docker > /dev/null; then - emcc="docker run \ - --rm \ - -v $(pwd):/src:Z \ - -u $(id -u) \ - emscripten/emsdk:2.0.11 \ + emcc="docker run \ + --rm \ + -v $(pwd):/src:Z \ + -u $(id -u) \ + emscripten/emsdk:$emscripen_version \ emcc" else echo 'You must have either `docker` or `emcc` on your PATH to run this script' diff --git a/script/fetch-emscripten b/script/fetch-emscripten index e2817ad8..157d0cae 100755 --- a/script/fetch-emscripten +++ b/script/fetch-emscripten @@ -2,7 +2,7 @@ set -e -EMSCRIPTEN_VERSION=2.0.11 +EMSCRIPTEN_VERSION=$(cat "$(dirname "$0")/../emscripten-version") mkdir -p target EMSDK_DIR="./target/emsdk"