From 66c30648c2c6f1bfe76c0763dc712f29d4b2a1a0 Mon Sep 17 00:00:00 2001 From: Douglas Creager Date: Wed, 9 Jun 2021 12:51:28 -0400 Subject: [PATCH] rust: Extract runtime language detection into separate crate This patch adds a new `tree-sitter-loader` crate, which holds the CLI's logic for finding and building local grammar definitions at runtime. This allows other command-line tools to use this logic too! --- Cargo.lock | 21 ++++++++-- cli/Cargo.toml | 7 ++-- cli/benches/benchmark.rs | 2 +- cli/build.rs | 5 --- cli/loader/Cargo.toml | 36 +++++++++++++++++ cli/loader/README.md | 6 +++ cli/loader/build.rs | 6 +++ cli/{src/loader.rs => loader/src/lib.rs} | 37 ++++++++++++++++++ cli/src/highlight.rs | 2 +- cli/src/lib.rs | 1 - cli/src/main.rs | 50 +++--------------------- cli/src/tags.rs | 2 +- cli/src/test_highlight.rs | 2 +- cli/src/tests/helpers/fixtures.rs | 2 +- 14 files changed, 117 insertions(+), 62 deletions(-) create mode 100644 cli/loader/Cargo.toml create mode 100644 cli/loader/README.md create mode 100644 cli/loader/build.rs rename cli/{src/loader.rs => loader/src/lib.rs} (95%) diff --git a/Cargo.lock b/Cargo.lock index 97e7fea6..06be258b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -675,16 +675,13 @@ dependencies = [ "ansi_term 0.12.1", "anyhow", "atty", - "cc", "clap", "difference", "dirs", "glob", "html-escape", "lazy_static", - "libloading", "log", - "once_cell", "rand", "regex", "regex-syntax", @@ -696,6 +693,7 @@ dependencies = [ "tiny_http", "tree-sitter", "tree-sitter-highlight", + "tree-sitter-loader", "tree-sitter-tags", "walkdir", "webbrowser", @@ -711,6 +709,23 @@ dependencies = [ "tree-sitter", ] +[[package]] +name = "tree-sitter-loader" +version = "0.19.0" +dependencies = [ + "anyhow", + "cc", + "libloading", + "once_cell", + "regex", + "serde", + "serde_derive", + "serde_json", + "tree-sitter", + "tree-sitter-highlight", + "tree-sitter-tags", +] + [[package]] name = "tree-sitter-tags" version = "0.19.2" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 99dbb8c7..9e3e2fb7 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -22,15 +22,12 @@ harness = false ansi_term = "0.12" anyhow = "1.0" atty = "0.2" -cc = "^1.0.58" clap = "2.32" difference = "2.0" dirs = "3.0" glob = "0.3.0" html-escape = "0.2.6" lazy_static = "1.2.0" -libloading = "0.7" -once_cell = "1.7" regex = "1" regex-syntax = "0.6.4" serde = "1.0" @@ -54,6 +51,10 @@ features = ["allocation-tracking"] version = ">= 0.3.0" path = "../highlight" +[dependencies.tree-sitter-loader] +version = ">= 0.19.0" +path = "loader" + [dependencies.tree-sitter-tags] version = ">= 0.1.0" path = "../tags" diff --git a/cli/benches/benchmark.rs b/cli/benches/benchmark.rs index 9e3ed833..9df1b099 100644 --- a/cli/benches/benchmark.rs +++ b/cli/benches/benchmark.rs @@ -5,7 +5,7 @@ use std::path::{Path, PathBuf}; use std::time::Instant; use std::{env, fs, str, usize}; use tree_sitter::{Language, Parser, Query}; -use tree_sitter_cli::loader::Loader; +use tree_sitter_loader::Loader; include!("../src/tests/helpers/dirs.rs"); diff --git a/cli/build.rs b/cli/build.rs index 47506018..322d239b 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -9,11 +9,6 @@ fn main() { if wasm_files_present() { println!("cargo:rustc-cfg={}", "TREE_SITTER_EMBED_WASM_BINDING"); } - - println!( - "cargo:rustc-env=BUILD_TARGET={}", - std::env::var("TARGET").unwrap() - ); } fn wasm_files_present() -> bool { diff --git a/cli/loader/Cargo.toml b/cli/loader/Cargo.toml new file mode 100644 index 00000000..2b78f01f --- /dev/null +++ b/cli/loader/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "tree-sitter-loader" +description = "Locates, builds, and loads tree-sitter grammars at runtime" +version = "0.19.0" +authors = ["Max Brunsfeld "] +edition = "2018" +license = "MIT" +readme = "README.md" +keywords = ["incremental", "parsing"] +categories = ["command-line-utilities", "parsing"] +repository = "https://github.com/tree-sitter/tree-sitter" + +[dependencies] +anyhow = "1.0" +cc = "^1.0.58" +libloading = "0.7" +once_cell = "1.7" +regex = "1" +serde = "1.0" +serde_derive = "1.0" + +[dependencies.serde_json] +version = "1.0" +features = ["preserve_order"] + +[dependencies.tree-sitter] +version = ">= 0.19" +path = "../../lib" + +[dependencies.tree-sitter-highlight] +version = ">= 0.19" +path = "../../highlight" + +[dependencies.tree-sitter-tags] +version = ">= 0.19" +path = "../../tags" diff --git a/cli/loader/README.md b/cli/loader/README.md new file mode 100644 index 00000000..9889ec71 --- /dev/null +++ b/cli/loader/README.md @@ -0,0 +1,6 @@ +# `tree-sitter-loader` + +The `tree-sitter` command-line program will dynamically find and build grammars +at runtime, if you have cloned the grammars' repositories to your local +filesystem. This helper crate implements that logic, so that you can use it in +your own program analysis tools, as well. diff --git a/cli/loader/build.rs b/cli/loader/build.rs new file mode 100644 index 00000000..e0ebd1c4 --- /dev/null +++ b/cli/loader/build.rs @@ -0,0 +1,6 @@ +fn main() { + println!( + "cargo:rustc-env=BUILD_TARGET={}", + std::env::var("TARGET").unwrap() + ); +} diff --git a/cli/src/loader.rs b/cli/loader/src/lib.rs similarity index 95% rename from cli/src/loader.rs rename to cli/loader/src/lib.rs index 04af4716..b8402b89 100644 --- a/cli/src/loader.rs +++ b/cli/loader/src/lib.rs @@ -534,6 +534,43 @@ impl Loader { fn regex(pattern: Option) -> Option { pattern.and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok()) } + + pub fn select_language( + &mut self, + path: &Path, + current_dir: &Path, + scope: Option<&str>, + ) -> Result { + if let Some(scope) = scope { + if let Some(config) = self + .language_configuration_for_scope(scope) + .with_context(|| format!("Failed to load language for scope '{}'", scope))? + { + Ok(config.0) + } else { + return Err(anyhow!("Unknown scope '{}'", scope)); + } + } else if let Some((lang, _)) = self + .language_configuration_for_file_name(path) + .with_context(|| { + format!( + "Failed to load language for file name {}", + &path.file_name().unwrap().to_string_lossy() + ) + })? + { + Ok(lang) + } else if let Some(lang) = self + .languages_at_path(¤t_dir) + .with_context(|| "Failed to load language in current directory")? + .first() + .cloned() + { + Ok(lang) + } else { + Err(anyhow!("No language found")) + } + } } impl<'a> LanguageConfiguration<'a> { diff --git a/cli/src/highlight.rs b/cli/src/highlight.rs index d5b1da97..38cb3d3b 100644 --- a/cli/src/highlight.rs +++ b/cli/src/highlight.rs @@ -1,5 +1,4 @@ use super::util; -use crate::loader::Loader; use ansi_term::Color; use anyhow::Result; use lazy_static::lazy_static; @@ -12,6 +11,7 @@ use std::sync::atomic::AtomicUsize; use std::time::Instant; use std::{fs, io, path, str, usize}; use tree_sitter_highlight::{HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer}; +use tree_sitter_loader::Loader; pub const HTML_HEADER: &'static str = " diff --git a/cli/src/lib.rs b/cli/src/lib.rs index b6f01dfe..9d5abe0d 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1,7 +1,6 @@ pub mod config; pub mod generate; pub mod highlight; -pub mod loader; pub mod logger; pub mod parse; pub mod query; diff --git a/cli/src/main.rs b/cli/src/main.rs index 11ed7c73..5762cf65 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -3,11 +3,11 @@ use clap::{App, AppSettings, Arg, SubCommand}; use glob::glob; use std::path::Path; use std::{env, fs, u64}; -use tree_sitter::Language; use tree_sitter_cli::{ - config, generate, highlight, loader, logger, parse, query, tags, test, test_highlight, util, - wasm, web_ui, + config, generate, highlight, logger, parse, query, tags, test, test_highlight, util, wasm, + web_ui, }; +use tree_sitter_loader as loader; const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION"); const BUILD_SHA: Option<&'static str> = option_env!("BUILD_SHA"); @@ -264,8 +264,7 @@ fn run() -> Result<()> { for path in paths { let path = Path::new(&path); - let language = - select_language(&mut loader, path, ¤t_dir, matches.value_of("scope"))?; + let language = loader.select_language(path, ¤t_dir, matches.value_of("scope"))?; let this_file_errored = parse::parse_file_at_path( language, @@ -302,8 +301,7 @@ fn run() -> Result<()> { let ordered_captures = matches.values_of("captures").is_some(); let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; loader.find_all_languages(&config.parser_directories)?; - let language = select_language( - &mut loader, + let language = loader.select_language( Path::new(&paths[0]), ¤t_dir, matches.value_of("scope"), @@ -485,41 +483,3 @@ fn collect_paths<'a>( Err(anyhow!("Must provide one or more paths")) } - -fn select_language( - loader: &mut loader::Loader, - path: &Path, - current_dir: &Path, - scope: Option<&str>, -) -> Result { - if let Some(scope) = scope { - if let Some(config) = loader - .language_configuration_for_scope(scope) - .with_context(|| format!("Failed to load language for scope '{}'", scope))? - { - Ok(config.0) - } else { - return Err(anyhow!("Unknown scope '{}'", scope)); - } - } else if let Some((lang, _)) = loader - .language_configuration_for_file_name(path) - .with_context(|| { - format!( - "Failed to load language for file name {}", - &path.file_name().unwrap().to_string_lossy() - ) - })? - { - Ok(lang) - } else if let Some(lang) = loader - .languages_at_path(¤t_dir) - .with_context(|| "Failed to load language in current directory")? - .first() - .cloned() - { - Ok(lang) - } else { - eprintln!("No language found"); - Err(anyhow!("No language found")) - } -} diff --git a/cli/src/tags.rs b/cli/src/tags.rs index aeb1c3cc..457955dd 100644 --- a/cli/src/tags.rs +++ b/cli/src/tags.rs @@ -1,10 +1,10 @@ -use super::loader::Loader; use super::util; use anyhow::{anyhow, Result}; use std::io::{self, Write}; use std::path::Path; use std::time::Instant; use std::{fs, str}; +use tree_sitter_loader::Loader; use tree_sitter_tags::TagsContext; pub fn generate_tags( diff --git a/cli/src/test_highlight.rs b/cli/src/test_highlight.rs index dcc42081..2d9d536a 100644 --- a/cli/src/test_highlight.rs +++ b/cli/src/test_highlight.rs @@ -1,4 +1,3 @@ -use crate::loader::Loader; use crate::query_testing::{parse_position_comments, Assertion}; use ansi_term::Colour; use anyhow::{anyhow, Result}; @@ -6,6 +5,7 @@ use std::fs; use std::path::Path; use tree_sitter::Point; use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, Highlighter}; +use tree_sitter_loader::Loader; #[derive(Debug)] pub struct Failure { diff --git a/cli/src/tests/helpers/fixtures.rs b/cli/src/tests/helpers/fixtures.rs index d098bd28..7b223c25 100644 --- a/cli/src/tests/helpers/fixtures.rs +++ b/cli/src/tests/helpers/fixtures.rs @@ -1,9 +1,9 @@ -use crate::loader::Loader; use lazy_static::lazy_static; use std::fs; use std::path::{Path, PathBuf}; use tree_sitter::Language; use tree_sitter_highlight::HighlightConfiguration; +use tree_sitter_loader::Loader; include!("./dirs.rs");