Merge pull request #1153 from dcreager/extract-loader

rust: Extract runtime language detection into separate crate

Cargo.lock

@@ -675,16 +675,13 @@ dependencies = [
  "ansi_term 0.12.1",
  "anyhow",
  "atty",
- "cc",
  "clap",
  "difference",
  "dirs",
  "glob",
  "html-escape",
  "lazy_static",
- "libloading",
  "log",
- "once_cell",
  "rand",
  "regex",
  "regex-syntax",
@@ -696,6 +693,7 @@ dependencies = [
  "tiny_http",
  "tree-sitter",
  "tree-sitter-highlight",
+ "tree-sitter-loader",
  "tree-sitter-tags",
  "walkdir",
  "webbrowser",
@@ -711,6 +709,23 @@ dependencies = [
  "tree-sitter",
 ]
 
+[[package]]
+name = "tree-sitter-loader"
+version = "0.19.0"
+dependencies = [
+ "anyhow",
+ "cc",
+ "libloading",
+ "once_cell",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tree-sitter",
+ "tree-sitter-highlight",
+ "tree-sitter-tags",
+]
+
 [[package]]
 name = "tree-sitter-tags"
 version = "0.19.2"

cli/Cargo.toml

@@ -22,15 +22,12 @@ harness = false
 ansi_term = "0.12"
 anyhow = "1.0"
 atty = "0.2"
-cc = "^1.0.58"
 clap = "2.32"
 difference = "2.0"
 dirs = "3.0"
 glob = "0.3.0"
 html-escape = "0.2.6"
 lazy_static = "1.2.0"
-libloading = "0.7"
-once_cell = "1.7"
 regex = "1"
 regex-syntax = "0.6.4"
 serde = "1.0"
@@ -54,6 +51,10 @@ features = ["allocation-tracking"]
 version = ">= 0.3.0"
 path = "../highlight"
 
+[dependencies.tree-sitter-loader]
+version = ">= 0.19.0"
+path = "loader"
+
 [dependencies.tree-sitter-tags]
 version = ">= 0.1.0"
 path = "../tags"

cli/benches/benchmark.rs

@@ -5,7 +5,7 @@ use std::path::{Path, PathBuf};
 use std::time::Instant;
 use std::{env, fs, str, usize};
 use tree_sitter::{Language, Parser, Query};
-use tree_sitter_cli::loader::Loader;
+use tree_sitter_loader::Loader;
 
 include!("../src/tests/helpers/dirs.rs");
 

cli/build.rs

@@ -9,11 +9,6 @@ fn main() {
     if wasm_files_present() {
         println!("cargo:rustc-cfg={}", "TREE_SITTER_EMBED_WASM_BINDING");
     }
-
-    println!(
-        "cargo:rustc-env=BUILD_TARGET={}",
-        std::env::var("TARGET").unwrap()
-    );
 }
 
 fn wasm_files_present() -> bool {

cli/loader/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "tree-sitter-loader"
+description = "Locates, builds, and loads tree-sitter grammars at runtime"
+version = "0.19.0"
+authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
+edition = "2018"
+license = "MIT"
+readme = "README.md"
+keywords = ["incremental", "parsing"]
+categories = ["command-line-utilities", "parsing"]
+repository = "https://github.com/tree-sitter/tree-sitter"
+
+[dependencies]
+anyhow = "1.0"
+cc = "^1.0.58"
+libloading = "0.7"
+once_cell = "1.7"
+regex = "1"
+serde = "1.0"
+serde_derive = "1.0"
+
+[dependencies.serde_json]
+version = "1.0"
+features = ["preserve_order"]
+
+[dependencies.tree-sitter]
+version = ">= 0.19"
+path = "../../lib"
+
+[dependencies.tree-sitter-highlight]
+version = ">= 0.19"
+path = "../../highlight"
+
+[dependencies.tree-sitter-tags]
+version = ">= 0.19"
+path = "../../tags"

cli/loader/README.md

@@ -0,0 +1,6 @@
+# `tree-sitter-loader`
+
+The `tree-sitter` command-line program will dynamically find and build grammars
+at runtime, if you have cloned the grammars' repositories to your local
+filesystem.  This helper crate implements that logic, so that you can use it in
+your own program analysis tools, as well.

cli/loader/build.rs

@@ -0,0 +1,6 @@
+fn main() {
+    println!(
+        "cargo:rustc-env=BUILD_TARGET={}",
+        std::env::var("TARGET").unwrap()
+    );
+}

cli/loader/src/lib.rs

@@ -534,6 +534,43 @@ impl Loader {
     fn regex(pattern: Option<String>) -> Option<Regex> {
         pattern.and_then(|r| RegexBuilder::new(&r).multi_line(true).build().ok())
     }
+
+    pub fn select_language(
+        &mut self,
+        path: &Path,
+        current_dir: &Path,
+        scope: Option<&str>,
+    ) -> Result<Language> {
+        if let Some(scope) = scope {
+            if let Some(config) = self
+                .language_configuration_for_scope(scope)
+                .with_context(|| format!("Failed to load language for scope '{}'", scope))?
+            {
+                Ok(config.0)
+            } else {
+                return Err(anyhow!("Unknown scope '{}'", scope));
+            }
+        } else if let Some((lang, _)) = self
+            .language_configuration_for_file_name(path)
+            .with_context(|| {
+                format!(
+                    "Failed to load language for file name {}",
+                    &path.file_name().unwrap().to_string_lossy()
+                )
+            })?
+        {
+            Ok(lang)
+        } else if let Some(lang) = self
+            .languages_at_path(&current_dir)
+            .with_context(|| "Failed to load language in current directory")?
+            .first()
+            .cloned()
+        {
+            Ok(lang)
+        } else {
+            Err(anyhow!("No language found"))
+        }
+    }
 }
 
 impl<'a> LanguageConfiguration<'a> {

cli/src/highlight.rs

@@ -1,5 +1,4 @@
 use super::util;
-use crate::loader::Loader;
 use ansi_term::Color;
 use anyhow::Result;
 use lazy_static::lazy_static;
@@ -12,6 +11,7 @@ use std::sync::atomic::AtomicUsize;
 use std::time::Instant;
 use std::{fs, io, path, str, usize};
 use tree_sitter_highlight::{HighlightConfiguration, HighlightEvent, Highlighter, HtmlRenderer};
+use tree_sitter_loader::Loader;
 
 pub const HTML_HEADER: &'static str = "
 <!doctype HTML>

cli/src/lib.rs

@@ -1,7 +1,6 @@
 pub mod config;
 pub mod generate;
 pub mod highlight;
-pub mod loader;
 pub mod logger;
 pub mod parse;
 pub mod query;

cli/src/main.rs

@@ -3,11 +3,11 @@ use clap::{App, AppSettings, Arg, SubCommand};
 use glob::glob;
 use std::path::Path;
 use std::{env, fs, u64};
-use tree_sitter::Language;
 use tree_sitter_cli::{
-    config, generate, highlight, loader, logger, parse, query, tags, test, test_highlight, util,
-    wasm, web_ui,
+    config, generate, highlight, logger, parse, query, tags, test, test_highlight, util, wasm,
+    web_ui,
 };
+use tree_sitter_loader as loader;
 
 const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION");
 const BUILD_SHA: Option<&'static str> = option_env!("BUILD_SHA");
@@ -264,8 +264,7 @@ fn run() -> Result<()> {
 
         for path in paths {
             let path = Path::new(&path);
-            let language =
-                select_language(&mut loader, path, &current_dir, matches.value_of("scope"))?;
+            let language = loader.select_language(path, &current_dir, matches.value_of("scope"))?;
 
             let this_file_errored = parse::parse_file_at_path(
                 language,
@@ -302,8 +301,7 @@ fn run() -> Result<()> {
         let ordered_captures = matches.values_of("captures").is_some();
         let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?;
         loader.find_all_languages(&config.parser_directories)?;
-        let language = select_language(
-            &mut loader,
+        let language = loader.select_language(
             Path::new(&paths[0]),
             &current_dir,
             matches.value_of("scope"),
@@ -485,41 +483,3 @@ fn collect_paths<'a>(
 
     Err(anyhow!("Must provide one or more paths"))
 }
-
-fn select_language(
-    loader: &mut loader::Loader,
-    path: &Path,
-    current_dir: &Path,
-    scope: Option<&str>,
-) -> Result<Language> {
-    if let Some(scope) = scope {
-        if let Some(config) = loader
-            .language_configuration_for_scope(scope)
-            .with_context(|| format!("Failed to load language for scope '{}'", scope))?
-        {
-            Ok(config.0)
-        } else {
-            return Err(anyhow!("Unknown scope '{}'", scope));
-        }
-    } else if let Some((lang, _)) = loader
-        .language_configuration_for_file_name(path)
-        .with_context(|| {
-            format!(
-                "Failed to load language for file name {}",
-                &path.file_name().unwrap().to_string_lossy()
-            )
-        })?
-    {
-        Ok(lang)
-    } else if let Some(lang) = loader
-        .languages_at_path(&current_dir)
-        .with_context(|| "Failed to load language in current directory")?
-        .first()
-        .cloned()
-    {
-        Ok(lang)
-    } else {
-        eprintln!("No language found");
-        Err(anyhow!("No language found"))
-    }
-}

cli/src/tags.rs

@@ -1,10 +1,10 @@
-use super::loader::Loader;
 use super::util;
 use anyhow::{anyhow, Result};
 use std::io::{self, Write};
 use std::path::Path;
 use std::time::Instant;
 use std::{fs, str};
+use tree_sitter_loader::Loader;
 use tree_sitter_tags::TagsContext;
 
 pub fn generate_tags(

cli/src/test_highlight.rs

@@ -1,4 +1,3 @@
-use crate::loader::Loader;
 use crate::query_testing::{parse_position_comments, Assertion};
 use ansi_term::Colour;
 use anyhow::{anyhow, Result};
@@ -6,6 +5,7 @@ use std::fs;
 use std::path::Path;
 use tree_sitter::Point;
 use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, Highlighter};
+use tree_sitter_loader::Loader;
 
 #[derive(Debug)]
 pub struct Failure {

cli/src/tests/helpers/fixtures.rs

@@ -1,9 +1,9 @@
-use crate::loader::Loader;
 use lazy_static::lazy_static;
 use std::fs;
 use std::path::{Path, PathBuf};
 use tree_sitter::Language;
 use tree_sitter_highlight::HighlightConfiguration;
+use tree_sitter_loader::Loader;
 
 include!("./dirs.rs");