traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cli: Extract CLI configuration into separate crate

Browse source 
This patch adds the `tree-sitter-config` crate, which manages
tree-sitter's configuration file.  This new setup allows different
components to define their own serializable configuration types, instead
of having to create a single monolithic configuration type.  But the
configuration itself is still stored in a single JSON file.

Before, the default location for the configuration file was
`~/.tree-sitter/config.json`.  This patch updates the default location
to follow the XDG Base Directory spec (or other relevant platform-
specific spec).  So on Linux, for instance, the new default location is
`~/.config/tree-sitter/config.json`.  We will look in the new location
_first_, and fall back on reading from the legacy location if we can't
find anything.
This commit is contained in:
Douglas Creager 2021-06-09 15:03:27 -04:00
parent ebae034b0c
commit e841fcfa1b
13 changed files with 220 additions and 91 deletions
Download patch file Download diff file Expand all files Collapse all files

12
Cargo.lock generated
View file

@ -692,6 +692,7 @@ dependencies = [
"tempfile",
"tiny_http",
"tree-sitter",
"tree-sitter-config",
"tree-sitter-highlight",
"tree-sitter-loader",
"tree-sitter-tags",
@ -700,6 +701,16 @@ dependencies = [
"which",
]
[[package]]
name = "tree-sitter-config"
version = "0.19.0"
dependencies = [
"anyhow",
"dirs",
"serde",
"serde_json",
]
[[package]]
name = "tree-sitter-highlight"
version = "0.19.2"
@ -715,6 +726,7 @@ version = "0.19.0"
dependencies = [
"anyhow",
"cc",
"dirs",
"libloading",
"once_cell",
"regex",

4
cli/Cargo.toml
View file

@ -47,6 +47,10 @@ version = ">= 0.17.0"
path = "../lib"
features = ["allocation-tracking"]
[dependencies.tree-sitter-config]
version = ">= 0.19.0"
path = "config"
[dependencies.tree-sitter-highlight]
version = ">= 0.3.0"
path = "../highlight"

2
cli/benches/benchmark.rs
View file

@ -17,7 +17,7 @@ lazy_static! {
static ref REPETITION_COUNT: usize = env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT")
.map(|s| usize::from_str_radix(&s, 10).unwrap())
.unwrap_or(5);
static ref TEST_LOADER: Loader = Loader::new(SCRATCH_DIR.clone());
static ref TEST_LOADER: Loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone());
static ref EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)> = {
fn process_dir(result: &mut BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)>, dir: &Path) {
if dir.join("grammar.js").exists() {

20
cli/config/Cargo.toml Normal file
View file

@ -0,0 +1,20 @@
[package]
name = "tree-sitter-config"
description = "User configuration of tree-sitter's command line programs"
version = "0.19.0"
authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
edition = "2018"
license = "MIT"
readme = "README.md"
keywords = ["incremental", "parsing"]
categories = ["command-line-utilities", "parsing"]
repository = "https://github.com/tree-sitter/tree-sitter"
[dependencies]
anyhow = "1.0"
dirs = "3.0"
serde = "1.0"
[dependencies.serde_json]
version = "1.0"
features = ["preserve_order"]

5
cli/config/README.md Normal file
View file

@ -0,0 +1,5 @@
# `tree-sitter-config`
You can use a configuration file to control the behavior of the `tree-sitter`
command-line program. This crate implements the logic for finding and the
parsing the contents of the configuration file.

114
cli/config/src/lib.rs Normal file
View file

@ -0,0 +1,114 @@
//! Manages tree-sitter's configuration file.
use anyhow::{anyhow, Result};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::path::PathBuf;
use std::{env, fs};
/// Holds the contents of tree-sitter's configuration file.
///
/// The file typically lives at `~/.config/tree-sitter/config.json`, but see the [`load`][] method
/// for the full details on where it might be located.
///
/// This type holds the generic JSON content of the configuration file. Individual tree-sitter
/// components will use the [`get`][] method to parse that JSON to extract configuration fields
/// that are specific to that component.
pub struct Config {
location: PathBuf,
config: Value,
}
impl Config {
fn find_config_file() -> Result<PathBuf> {
if let Ok(path) = env::var("TREE_SITTER_DIR") {
let mut path = PathBuf::from(path);
path.push("config.json");
return Ok(path);
}
let xdg_path = Self::xdg_config_file()?;
if xdg_path.is_file() {
return Ok(xdg_path);
}
let legacy_path = dirs::home_dir()
.ok_or(anyhow!("Cannot determine home directory"))?
.join(".tree-sitter/config.json");
if legacy_path.is_file() {
return Ok(legacy_path);
}
Err(anyhow!(concat!(
"Cannot find a tree-sitter configuration file.\n",
"Please run `tree-sitter init-config` to create one!"
)))
}
fn xdg_config_file() -> Result<PathBuf> {
let xdg_path = dirs::config_dir()
.ok_or(anyhow!("Cannot determine config directory"))?
.join("tree-sitter/config.json");
Ok(xdg_path)
}
/// Locates and loads in the user's configuration file. We search for the configuration file
/// in the following locations, in order:
///
/// - `$TREE_SITTER_DIR/config.json`, if the `TREE_SITTER_DIR` environment variable is set
/// - `tree-sitter/config.json` in your default user configuration directory, as determined
/// by [`dirs::config_dir`](https://docs.rs/dirs/*/dirs/fn.config_dir.html)
/// - `$HOME/.tree-sitter/config.json` as a fallback from where tree-sitter _used_ to store
/// its configuration
pub fn load() -> Result<Config> {
let location = Self::find_config_file()?;
let content = fs::read_to_string(&location)?;
let config = serde_json::from_str(&content)?;
Ok(Config { location, config })
}
/// Creates an empty initial configuration file. You can then use the [`add`][] method to add
/// the component-specific configuration types for any components that want to add content to
/// the default file, and then use [`save`][] to write the configuration to disk.
///
/// (Note that this is typically only done by the `tree-sitter init-config` command.)
pub fn initial() -> Result<Config> {
let location = Self::xdg_config_file()?;
let config = serde_json::json!({});
Ok(Config { location, config })
}
/// Saves this configuration to the file that it was originally loaded from.
pub fn save(&self) -> Result<()> {
let json = serde_json::to_string_pretty(&self.config)?;
fs::create_dir_all(self.location.parent().unwrap())?;
fs::write(&self.location, json)?;
Ok(())
}
/// Parses a component-specific configuration from the configuration file. The type `C` must
/// be [deserializable](https://docs.rs/serde/*/serde/trait.Deserialize.html) from a JSON
/// object, and must only include the fields relevant to that component.
pub fn get<C>(&self) -> Result<C>
where
C: for<'de> Deserialize<'de>,
{
let config = serde_json::from_value(self.config.clone())?;
Ok(config)
}
/// Adds a component-specific configuration to the configuration file. The type `C` must be
/// [serializable](https://docs.rs/serde/*/serde/trait.Serialize.html) into a JSON object, and
/// must only include the fields relevant to that component.
pub fn add<C>(&mut self, config: C) -> Result<()>
where
C: Serialize,
{
let mut config = serde_json::to_value(&config)?;
self.config
.as_object_mut()
.unwrap()
.append(config.as_object_mut().unwrap());
Ok(())
}
}

1
cli/loader/Cargo.toml
View file

@ -13,6 +13,7 @@ repository = "https://github.com/tree-sitter/tree-sitter"
[dependencies]
anyhow = "1.0"
cc = "^1.0.58"
dirs = "3.0"
libloading = "0.7"
once_cell = "1.7"
regex = "1"

36
cli/loader/src/lib.rs
View file

@ -2,7 +2,7 @@ use anyhow::{anyhow, Context, Error, Result};
use libloading::{Library, Symbol};
use once_cell::unsync::OnceCell;
use regex::{Regex, RegexBuilder};
use serde_derive::Deserialize;
use serde_derive::{Deserialize, Serialize};
use std::collections::HashMap;
use std::io::BufReader;
use std::ops::Range;
@ -15,6 +15,26 @@ use tree_sitter::{Language, QueryError};
use tree_sitter_highlight::HighlightConfiguration;
use tree_sitter_tags::{Error as TagsError, TagsConfiguration};
#[derive(Default, Deserialize, Serialize)]
pub struct Config {
#[serde(default)]
#[serde(rename = "parser-directories")]
pub parser_directories: Vec<PathBuf>,
}
impl Config {
pub fn initial() -> Config {
let home_dir = dirs::home_dir().expect("Cannot determine home directory");
Config {
parser_directories: vec![
home_dir.join("github"),
home_dir.join("src"),
home_dir.join("source"),
],
}
}
}
#[cfg(unix)]
const DYLIB_EXTENSION: &'static str = "so";
@ -54,7 +74,14 @@ unsafe impl Send for Loader {}
unsafe impl Sync for Loader {}
impl Loader {
pub fn new(parser_lib_path: PathBuf) -> Self {
pub fn new() -> Result<Self> {
let parser_lib_path = dirs::cache_dir()
.ok_or(anyhow!("Cannot determine cache directory"))?
.join("tree-sitter/lib");
Ok(Self::with_parser_lib_path(parser_lib_path))
}
pub fn with_parser_lib_path(parser_lib_path: PathBuf) -> Self {
Loader {
parser_lib_path,
languages_by_id: Vec::new(),
@ -76,8 +103,8 @@ impl Loader {
self.highlight_names.lock().unwrap().clone()
}
pub fn find_all_languages(&mut self, parser_src_paths: &Vec<PathBuf>) -> Result<()> {
for parser_container_dir in parser_src_paths.iter() {
pub fn find_all_languages(&mut self, config: &Config) -> Result<()> {
for parser_container_dir in &config.parser_directories {
if let Ok(entries) = fs::read_dir(parser_container_dir) {
for entry in entries {
let entry = entry?;
@ -287,6 +314,7 @@ impl Loader {
.with_context(|| "Failed to compare source and binary timestamps")?;
if recompile {
fs::create_dir_all(&self.parser_lib_path)?;
let mut config = cc::Build::new();
config
.cpp(true)

69
cli/src/config.rs
View file

@ -1,69 +0,0 @@
use super::highlight::Theme;
use serde_derive::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
use std::{env, fs, io};
#[derive(Default, Deserialize, Serialize)]
pub struct Config {
#[serde(skip)]
pub binary_directory: PathBuf,
#[serde(default)]
#[serde(rename = "parser-directories")]
pub parser_directories: Vec<PathBuf>,
#[serde(default)]
pub theme: Theme,
}
impl Config {
pub fn get_path(home_dir: &Path) -> PathBuf {
env::var("TREE_SITTER_DIR")
.map(|p| p.into())
.unwrap_or_else(|_| home_dir.join(".tree-sitter"))
}
pub fn load(home_dir: &Path) -> Self {
let tree_sitter_dir = Self::get_path(home_dir);
let config_path = tree_sitter_dir.join("config.json");
let mut result = fs::read_to_string(&config_path)
.map_err(drop)
.and_then(|json| serde_json::from_str(&json).map_err(drop))
.unwrap_or_else(|_| Self::default());
result.init(home_dir, &tree_sitter_dir);
result
}
pub fn save(&self, home_dir: &Path) -> io::Result<()> {
let tree_sitter_dir = Self::get_path(home_dir);
let config_path = tree_sitter_dir.join("config.json");
let json = serde_json::to_string_pretty(self).expect("Failed to serialize config");
fs::write(config_path, json)
}
pub fn new(home_dir: &Path) -> Self {
let tree_sitter_dir = Self::get_path(home_dir);
let mut result = Self::default();
result.init(home_dir, &tree_sitter_dir);
result
}
fn init(&mut self, home_dir: &Path, tree_sitter_dir: &Path) {
if self.parser_directories.is_empty() {
self.parser_directories = vec![
home_dir.join("github"),
home_dir.join("src"),
home_dir.join("source"),
]
}
let binary_path = tree_sitter_dir.join("bin");
self.binary_directory = binary_path;
fs::create_dir_all(&self.binary_directory).unwrap_or_else(|error| {
panic!(
"Could not find or create parser binary directory {:?}. Error: {}",
self.binary_directory, error
)
});
}
}

7
cli/src/highlight.rs
View file

@ -4,6 +4,7 @@ use anyhow::Result;
use lazy_static::lazy_static;
use serde::ser::SerializeMap;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_derive::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::collections::HashMap;
use std::fmt::Write;
@ -56,6 +57,12 @@ pub struct Theme {
pub highlight_names: Vec<String>,
}
#[derive(Default, Deserialize, Serialize)]
pub struct ThemeConfig {
#[serde(default)]
pub theme: Theme,
}
impl Theme {
pub fn load(path: &path::Path) -> io::Result<Self> {
let json = fs::read_to_string(path)?;

1
cli/src/lib.rs
View file

@ -1,4 +1,3 @@
pub mod config;
pub mod generate;
pub mod highlight;
pub mod logger;

38
cli/src/main.rs
View file

@ -4,9 +4,9 @@ use glob::glob;
use std::path::Path;
use std::{env, fs, u64};
use tree_sitter_cli::{
config, generate, highlight, logger, parse, query, tags, test, test_highlight, util, wasm,
web_ui,
generate, highlight, logger, parse, query, tags, test, test_highlight, util, wasm, web_ui,
};
use tree_sitter_config::Config;
use tree_sitter_loader as loader;
const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION");
@ -174,14 +174,15 @@ fn run() -> Result<()> {
)
.get_matches();
let home_dir = dirs::home_dir().expect("Failed to read home directory");
let current_dir = env::current_dir().unwrap();
let config = config::Config::load(&home_dir);
let mut loader = loader::Loader::new(config.binary_directory.clone());
let config = Config::load()?;
let mut loader = loader::Loader::new()?;
if matches.subcommand_matches("init-config").is_some() {
let config = config::Config::new(&home_dir);
config.save(&home_dir)?;
let mut config = Config::initial()?;
config.add(tree_sitter_loader::Config::initial())?;
config.add(tree_sitter_cli::highlight::ThemeConfig::default())?;
config.save()?;
} else if let Some(matches) = matches.subcommand_matches("generate") {
let grammar_path = matches.value_of("grammar-path");
let report_symbol_name = matches.value_of("report-states-for-rule").or_else(|| {
@ -257,7 +258,8 @@ fn run() -> Result<()> {
let max_path_length = paths.iter().map(|p| p.chars().count()).max().unwrap();
let mut has_error = false;
loader.find_all_languages(&config.parser_directories)?;
let loader_config = config.get()?;
loader.find_all_languages(&loader_config)?;
let should_track_stats = matches.is_present("stat");
let mut stats = parse::Stats::default();
@ -300,7 +302,8 @@ fn run() -> Result<()> {
} else if let Some(matches) = matches.subcommand_matches("query") {
let ordered_captures = matches.values_of("captures").is_some();
let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?;
loader.find_all_languages(&config.parser_directories)?;
let loader_config = config.get()?;
loader.find_all_languages(&loader_config)?;
let language = loader.select_language(
Path::new(&paths[0]),
&current_dir,
@ -321,7 +324,8 @@ fn run() -> Result<()> {
should_test,
)?;
} else if let Some(matches) = matches.subcommand_matches("tags") {
loader.find_all_languages(&config.parser_directories)?;
let loader_config = config.get()?;
loader.find_all_languages(&loader_config)?;
let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?;
tags::generate_tags(
&loader,
@ -331,8 +335,10 @@ fn run() -> Result<()> {
matches.is_present("time"),
)?;
} else if let Some(matches) = matches.subcommand_matches("highlight") {
loader.configure_highlights(&config.theme.highlight_names);
loader.find_all_languages(&config.parser_directories)?;
let theme_config: tree_sitter_cli::highlight::ThemeConfig = config.get()?;
loader.configure_highlights(&theme_config.theme.highlight_names);
let loader_config = config.get()?;
loader.find_all_languages(&loader_config)?;
let time = matches.is_present("time");
let quiet = matches.is_present("quiet");
@ -368,10 +374,11 @@ fn run() -> Result<()> {
if let Some(highlight_config) = language_config.highlight_config(language)? {
let source = fs::read(path)?;
let theme_config = config.get()?;
if html_mode {
highlight::html(
&loader,
&config.theme,
&theme_config,
&source,
highlight_config,
quiet,
@ -380,7 +387,7 @@ fn run() -> Result<()> {
} else {
highlight::ansi(
&loader,
&config.theme,
&theme_config,
&source,
highlight_config,
time,
@ -402,7 +409,8 @@ fn run() -> Result<()> {
let open_in_browser = !matches.is_present("quiet");
web_ui::serve(&current_dir, open_in_browser);
} else if matches.subcommand_matches("dump-languages").is_some() {
loader.find_all_languages(&config.parser_directories)?;
let loader_config = config.get()?;
loader.find_all_languages(&loader_config)?;
for (configuration, language_path) in loader.get_all_language_configurations() {
println!(
concat!(

2
cli/src/tests/helpers/fixtures.rs
View file

@ -8,7 +8,7 @@ use tree_sitter_loader::Loader;
include!("./dirs.rs");
lazy_static! {
static ref TEST_LOADER: Loader = Loader::new(SCRATCH_DIR.clone());
static ref TEST_LOADER: Loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone());
}
pub fn test_loader<'a>() -> &'a Loader {