use anyhow::{anyhow, Context, Result}; use clap::{App, AppSettings, Arg, SubCommand}; use glob::glob; use std::path::Path; use std::{env, fs, u64}; use tree_sitter_cli::{ generate, highlight, logger, parse, playground, query, tags, test, test_highlight, util, wasm, }; use tree_sitter_config::Config; use tree_sitter_loader as loader; const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION"); const BUILD_SHA: Option<&'static str> = option_env!("BUILD_SHA"); fn main() { let result = run(); if let Err(err) = &result { // Ignore BrokenPipe errors if let Some(error) = err.downcast_ref::() { if error.kind() == std::io::ErrorKind::BrokenPipe { return; } } if !err.to_string().is_empty() { eprintln!("{:?}", err); } std::process::exit(1); } } fn run() -> Result<()> { let version = if let Some(build_sha) = BUILD_SHA { format!("{} ({})", BUILD_VERSION, build_sha) } else { BUILD_VERSION.to_string() }; let debug_arg = Arg::with_name("debug") .help("Show parsing debug log") .long("debug") .short("d"); let debug_graph_arg = Arg::with_name("debug-graph") .help("Produce the log.html file with debug graphs") .long("debug-graph") .short("D"); let debug_build_arg = Arg::with_name("debug-build") .help("Compile a parser in debug mode") .long("debug-build") .short("0"); let paths_file_arg = Arg::with_name("paths-file") .help("The path to a file with paths to source file(s)") .long("paths") .takes_value(true); let paths_arg = Arg::with_name("paths") .help("The source file(s) to use") .multiple(true); let scope_arg = Arg::with_name("scope") .help("Select a language by the scope instead of a file extension") .long("scope") .takes_value(true); let time_arg = Arg::with_name("time") .help("Measure execution time") .long("time") .short("t"); let quiet_arg = Arg::with_name("quiet") .help("Suppress main output") .long("quiet") .short("q"); let matches = App::new("tree-sitter") .author("Max Brunsfeld ") .about("Generates and tests parsers") .version(version.as_str()) .setting(AppSettings::SubcommandRequiredElseHelp) .global_setting(AppSettings::ColoredHelp) .global_setting(AppSettings::DeriveDisplayOrder) .global_setting(AppSettings::DisableHelpSubcommand) .subcommand(SubCommand::with_name("init-config").about("Generate a default config file")) .subcommand( SubCommand::with_name("generate") .alias("gen") .alias("g") .about("Generate a parser") .arg(Arg::with_name("grammar-path").index(1)) .arg(Arg::with_name("log").long("log")) .arg(Arg::with_name("prev-abi").long("prev-abi")) .arg(Arg::with_name("no-bindings").long("no-bindings")) .arg( Arg::with_name("report-states-for-rule") .long("report-states-for-rule") .value_name("rule-name") .takes_value(true), ) .arg(Arg::with_name("no-minimize").long("no-minimize")), ) .subcommand( SubCommand::with_name("parse") .alias("p") .about("Parse files") .arg(&paths_file_arg) .arg(&paths_arg) .arg(&scope_arg) .arg(&debug_arg) .arg(&debug_build_arg) .arg(&debug_graph_arg) .arg(Arg::with_name("debug-xml").long("xml").short("x")) .arg( Arg::with_name("stat") .help("Show parsing statistic") .long("stat") .short("s"), ) .arg( Arg::with_name("timeout") .help("Interrupt the parsing process by timeout (µs)") .long("timeout") .takes_value(true), ) .arg(&time_arg) .arg(&quiet_arg) .arg( Arg::with_name("edits") .help("Apply edits in the format: \"row,col del_count insert_text\"") .long("edit") .short("edit") .takes_value(true) .multiple(true) .number_of_values(1), ), ) .subcommand( SubCommand::with_name("query") .alias("q") .about("Search files using a syntax tree query") .arg( Arg::with_name("query-path") .help("Path to a file with queries") .index(1) .required(true), ) .arg(&paths_file_arg) .arg(&paths_arg.clone().index(2)) .arg( Arg::with_name("byte-range") .help("The range of byte offsets in which the query will be executed") .long("byte-range") .takes_value(true), ) .arg(&scope_arg) .arg(Arg::with_name("captures").long("captures").short("c")) .arg(Arg::with_name("test").long("test")), ) .subcommand( SubCommand::with_name("tags") .about("Generate a list of tags") .arg(&scope_arg) .arg(&time_arg) .arg(&quiet_arg) .arg(&paths_file_arg) .arg(&paths_arg), ) .subcommand( SubCommand::with_name("test") .alias("t") .about("Run a parser's tests") .arg( Arg::with_name("filter") .long("filter") .short("f") .takes_value(true) .help("Only run corpus test cases whose name includes the given string"), ) .arg( Arg::with_name("update") .long("update") .short("u") .help("Update all syntax trees in corpus files with current parser output"), ) .arg(&debug_arg) .arg(&debug_build_arg) .arg(&debug_graph_arg), ) .subcommand( SubCommand::with_name("highlight") .about("Highlight a file") .arg( Arg::with_name("html") .help("Generate highlighting as an HTML document") .long("html") .short("H"), ) .arg(&scope_arg) .arg(&time_arg) .arg(&quiet_arg) .arg(&paths_file_arg) .arg(&paths_arg), ) .subcommand( SubCommand::with_name("build-wasm") .alias("bw") .about("Compile a parser to WASM") .arg( Arg::with_name("docker") .long("docker") .help("Run emscripten via docker even if it is installed locally"), ) .arg(Arg::with_name("path").index(1).multiple(true)), ) .subcommand( SubCommand::with_name("playground") .alias("play") .alias("pg") .alias("web-ui") .about("Start local playground for a parser in the browser") .arg( Arg::with_name("quiet") .long("quiet") .short("q") .help("Don't open in default browser"), ), ) .subcommand( SubCommand::with_name("dump-languages") .about("Print info about all known language parsers"), ) .get_matches(); let current_dir = env::current_dir().unwrap(); let config = Config::load()?; let mut loader = loader::Loader::new()?; match matches.subcommand() { ("init-config", Some(_)) => { if let Ok(Some(config_path)) = Config::find_config_file() { return Err(anyhow!( "Remove your existing config file first: {}", config_path.to_string_lossy() )); } let mut config = Config::initial()?; config.add(tree_sitter_loader::Config::initial())?; config.add(tree_sitter_cli::highlight::ThemeConfig::default())?; config.save()?; println!( "Saved initial configuration to {}", config.location.display() ); } ("generate", Some(matches)) => { let grammar_path = matches.value_of("grammar-path"); let report_symbol_name = matches.value_of("report-states-for-rule").or_else(|| { if matches.is_present("report-states") { Some("") } else { None } }); if matches.is_present("log") { logger::init(); } let new_abi = !matches.is_present("prev-abi"); let generate_bindings = !matches.is_present("no-bindings"); generate::generate_parser_in_directory( ¤t_dir, grammar_path, new_abi, generate_bindings, report_symbol_name, )?; } ("test", Some(matches)) => { let debug = matches.is_present("debug"); let debug_graph = matches.is_present("debug-graph"); let debug_build = matches.is_present("debug-build"); let update = matches.is_present("update"); let filter = matches.value_of("filter"); loader.use_debug_build(debug_build); let languages = loader.languages_at_path(¤t_dir)?; let language = languages .first() .ok_or_else(|| anyhow!("No language found"))?; let test_dir = current_dir.join("test"); // Run the corpus tests. Look for them at two paths: `test/corpus` and `corpus`. let mut test_corpus_dir = test_dir.join("corpus"); if !test_corpus_dir.is_dir() { test_corpus_dir = current_dir.join("corpus"); } if test_corpus_dir.is_dir() { test::run_tests_at_path( *language, &test_corpus_dir, debug, debug_graph, filter, update, )?; } // Check that all of the queries are valid. test::check_queries_at_path(*language, ¤t_dir.join("queries"))?; // Run the syntax highlighting tests. let test_highlight_dir = test_dir.join("highlight"); if test_highlight_dir.is_dir() { test_highlight::test_highlights(&loader, &test_highlight_dir)?; } } ("parse", Some(matches)) => { let debug = matches.is_present("debug"); let debug_graph = matches.is_present("debug-graph"); let debug_build = matches.is_present("debug-build"); let debug_xml = matches.is_present("debug-xml"); let quiet = matches.is_present("quiet"); let time = matches.is_present("time"); let edits = matches .values_of("edits") .map_or(Vec::new(), |e| e.collect()); let cancellation_flag = util::cancel_on_stdin(); if debug { // For augmenting debug logging in external scanners env::set_var("TREE_SITTER_DEBUG", "1"); } loader.use_debug_build(debug_build); let timeout = matches .value_of("timeout") .map_or(0, |t| u64::from_str_radix(t, 10).unwrap()); let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; let max_path_length = paths.iter().map(|p| p.chars().count()).max().unwrap_or(0); let mut has_error = false; let loader_config = config.get()?; loader.find_all_languages(&loader_config)?; let should_track_stats = matches.is_present("stat"); let mut stats = parse::Stats::default(); for path in paths { let path = Path::new(&path); let language = loader.select_language(path, ¤t_dir, matches.value_of("scope"))?; let this_file_errored = parse::parse_file_at_path( language, path, &edits, max_path_length, quiet, time, timeout, debug, debug_graph, debug_xml, Some(&cancellation_flag), )?; if should_track_stats { stats.total_parses += 1; if !this_file_errored { stats.successful_parses += 1; } } has_error |= this_file_errored; } if should_track_stats { println!("{}", stats) } if has_error { return Err(anyhow!("")); } } ("query", Some(matches)) => { let ordered_captures = matches.values_of("captures").is_some(); let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; let loader_config = config.get()?; loader.find_all_languages(&loader_config)?; let language = loader.select_language( Path::new(&paths[0]), ¤t_dir, matches.value_of("scope"), )?; let query_path = Path::new(matches.value_of("query-path").unwrap()); let range = matches.value_of("byte-range").map(|br| { let r: Vec<&str> = br.split(":").collect(); r[0].parse().unwrap()..r[1].parse().unwrap() }); let should_test = matches.is_present("test"); query::query_files_at_paths( language, paths, query_path, ordered_captures, range, should_test, )?; } ("tags", Some(matches)) => { let loader_config = config.get()?; loader.find_all_languages(&loader_config)?; let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; tags::generate_tags( &loader, matches.value_of("scope"), &paths, matches.is_present("quiet"), matches.is_present("time"), )?; } ("highlight", Some(matches)) => { let theme_config: tree_sitter_cli::highlight::ThemeConfig = config.get()?; loader.configure_highlights(&theme_config.theme.highlight_names); let loader_config = config.get()?; loader.find_all_languages(&loader_config)?; let time = matches.is_present("time"); let quiet = matches.is_present("quiet"); let html_mode = quiet || matches.is_present("html"); let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; if html_mode && !quiet { println!("{}", highlight::HTML_HEADER); } let cancellation_flag = util::cancel_on_stdin(); let mut lang = None; if let Some(scope) = matches.value_of("scope") { lang = loader.language_configuration_for_scope(scope)?; if lang.is_none() { return Err(anyhow!("Unknown scope '{}'", scope)); } } for path in paths { let path = Path::new(&path); let (language, language_config) = match lang { Some(v) => v, None => match loader.language_configuration_for_file_name(path)? { Some(v) => v, None => { eprintln!("No language found for path {:?}", path); continue; } }, }; if let Some(highlight_config) = language_config.highlight_config(language)? { let source = fs::read(path)?; if html_mode { highlight::html( &loader, &theme_config.theme, &source, highlight_config, quiet, time, )?; } else { highlight::ansi( &loader, &theme_config.theme, &source, highlight_config, time, Some(&cancellation_flag), )?; } } else { eprintln!("No syntax highlighting config found for path {:?}", path); } } if html_mode && !quiet { println!("{}", highlight::HTML_FOOTER); } } ("build-wasm", Some(matches)) => { let grammar_path = current_dir.join(matches.value_of("path").unwrap_or("")); wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?; } ("playground", Some(matches)) => { let open_in_browser = !matches.is_present("quiet"); playground::serve(¤t_dir, open_in_browser); } ("dump-languages", Some(_)) => { let loader_config = config.get()?; loader.find_all_languages(&loader_config)?; for (configuration, language_path) in loader.get_all_language_configurations() { println!( concat!( "scope: {}\n", "parser: {:?}\n", "highlights: {:?}\n", "file_types: {:?}\n", "content_regex: {:?}\n", "injection_regex: {:?}\n", ), configuration.scope.as_ref().unwrap_or(&String::new()), language_path, configuration.highlights_filenames, configuration.file_types, configuration.content_regex, configuration.injection_regex, ); } } _ => unreachable!(), } Ok(()) } fn collect_paths<'a>( paths_file: Option<&str>, paths: Option>, ) -> Result> { if let Some(paths_file) = paths_file { return Ok(fs::read_to_string(paths_file) .with_context(|| format!("Failed to read paths file {}", paths_file))? .trim() .lines() .map(String::from) .collect::>()); } if let Some(paths) = paths { let mut result = Vec::new(); let mut incorporate_path = |path: &str, positive| { if positive { result.push(path.to_string()); } else { if let Some(index) = result.iter().position(|p| p == path) { result.remove(index); } } }; for mut path in paths { let mut positive = true; if path.starts_with("!") { positive = false; path = path.trim_start_matches("!"); } if Path::new(path).exists() { incorporate_path(path, positive); } else { let paths = glob(path).with_context(|| format!("Invalid glob pattern {:?}", path))?; for path in paths { if let Some(path) = path?.to_str() { incorporate_path(path, positive); } } } } if result.is_empty() { return Err(anyhow!( "No files were found at or matched by the provided pathname/glob" )); } return Ok(result); } Err(anyhow!("Must provide one or more paths")) }