traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #1157 from dcreager/xdg-config

Browse source 
cli: Extract CLI configuration into separate crate
This commit is contained in:
Douglas Creager 2021-06-10 15:39:45 -04:00 • committed by GitHub
commit 6ed42747a4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 247 additions and 97 deletions
Download patch file Download diff file Expand all files Collapse all files

12
Cargo.lock generated
View file

@ -692,6 +692,7 @@ dependencies = [
"tempfile",
"tiny_http",
"tree-sitter",
"tree-sitter-config",
"tree-sitter-highlight",
"tree-sitter-loader",
"tree-sitter-tags",
@ -700,6 +701,16 @@ dependencies = [
"which",
]
[[package]]
name = "tree-sitter-config"
version = "0.19.0"
dependencies = [
"anyhow",
"dirs",
"serde",
"serde_json",
]
[[package]]
name = "tree-sitter-highlight"
version = "0.19.2"
@ -715,6 +726,7 @@ version = "0.19.0"
dependencies = [
"anyhow",
"cc",
"dirs",
"libloading",
"once_cell",
"regex",

4
cli/Cargo.toml
View file

@ -47,6 +47,10 @@ version = ">= 0.17.0"
path = "../lib"
features = ["allocation-tracking"]
[dependencies.tree-sitter-config]
version = ">= 0.19.0"
path = "config"
[dependencies.tree-sitter-highlight]
version = ">= 0.3.0"
path = "../highlight"

2
cli/benches/benchmark.rs
View file

@ -17,7 +17,7 @@ lazy_static! {
static ref REPETITION_COUNT: usize = env::var("TREE_SITTER_BENCHMARK_REPETITION_COUNT")
.map(|s| usize::from_str_radix(&s, 10).unwrap())
.unwrap_or(5);
static ref TEST_LOADER: Loader = Loader::new(SCRATCH_DIR.clone());
static ref TEST_LOADER: Loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone());
static ref EXAMPLE_AND_QUERY_PATHS_BY_LANGUAGE_DIR: BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)> = {
fn process_dir(result: &mut BTreeMap<PathBuf, (Vec<PathBuf>, Vec<PathBuf>)>, dir: &Path) {
if dir.join("grammar.js").exists() {

20
cli/config/Cargo.toml Normal file
View file

@ -0,0 +1,20 @@
[package]
name = "tree-sitter-config"
description = "User configuration of tree-sitter's command line programs"
version = "0.19.0"
authors = ["Max Brunsfeld <maxbrunsfeld@gmail.com>"]
edition = "2018"
license = "MIT"
readme = "README.md"
keywords = ["incremental", "parsing"]
categories = ["command-line-utilities", "parsing"]
repository = "https://github.com/tree-sitter/tree-sitter"
[dependencies]
anyhow = "1.0"
dirs = "3.0"
serde = "1.0"
[dependencies.serde_json]
version = "1.0"
features = ["preserve_order"]

5
cli/config/README.md Normal file
View file

@ -0,0 +1,5 @@
# `tree-sitter-config`
You can use a configuration file to control the behavior of the `tree-sitter`
command-line program. This crate implements the logic for finding and the
parsing the contents of the configuration file.

114
cli/config/src/lib.rs Normal file
View file

@ -0,0 +1,114 @@
//! Manages tree-sitter's configuration file.
use anyhow::{anyhow, Result};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::path::PathBuf;
use std::{env, fs};
/// Holds the contents of tree-sitter's configuration file.
///
/// The file typically lives at `~/.config/tree-sitter/config.json`, but see the [`load`][] method
/// for the full details on where it might be located.
///
/// This type holds the generic JSON content of the configuration file. Individual tree-sitter
/// components will use the [`get`][] method to parse that JSON to extract configuration fields
/// that are specific to that component.
pub struct Config {
pub location: PathBuf,
pub config: Value,
}
impl Config {
fn find_config_file() -> Result<Option<PathBuf>> {
if let Ok(path) = env::var("TREE_SITTER_DIR") {
let mut path = PathBuf::from(path);
path.push("config.json");
return Ok(Some(path));
}
let xdg_path = Self::xdg_config_file()?;
if xdg_path.is_file() {
return Ok(Some(xdg_path));
}
let legacy_path = dirs::home_dir()
.ok_or(anyhow!("Cannot determine home directory"))?
.join(".tree-sitter/config.json");
if legacy_path.is_file() {
return Ok(Some(legacy_path));
}
Ok(None)
}
fn xdg_config_file() -> Result<PathBuf> {
let xdg_path = dirs::config_dir()
.ok_or(anyhow!("Cannot determine config directory"))?
.join("tree-sitter/config.json");
Ok(xdg_path)
}
/// Locates and loads in the user's configuration file. We search for the configuration file
/// in the following locations, in order:
///
/// - `$TREE_SITTER_DIR/config.json`, if the `TREE_SITTER_DIR` environment variable is set
/// - `tree-sitter/config.json` in your default user configuration directory, as determined
/// by [`dirs::config_dir`](https://docs.rs/dirs/*/dirs/fn.config_dir.html)
/// - `$HOME/.tree-sitter/config.json` as a fallback from where tree-sitter _used_ to store
/// its configuration
pub fn load() -> Result<Config> {
let location = match Self::find_config_file()? {
Some(location) => location,
None => return Config::initial(),
};
let content = fs::read_to_string(&location)?;
let config = serde_json::from_str(&content)?;
Ok(Config { location, config })
}
/// Creates an empty initial configuration file. You can then use the [`add`][] method to add
/// the component-specific configuration types for any components that want to add content to
/// the default file, and then use [`save`][] to write the configuration to disk.
///
/// (Note that this is typically only done by the `tree-sitter init-config` command.)
pub fn initial() -> Result<Config> {
let location = Self::xdg_config_file()?;
let config = serde_json::json!({});
Ok(Config { location, config })
}
/// Saves this configuration to the file that it was originally loaded from.
pub fn save(&self) -> Result<()> {
let json = serde_json::to_string_pretty(&self.config)?;
fs::create_dir_all(self.location.parent().unwrap())?;
fs::write(&self.location, json)?;
Ok(())
}
/// Parses a component-specific configuration from the configuration file. The type `C` must
/// be [deserializable](https://docs.rs/serde/*/serde/trait.Deserialize.html) from a JSON
/// object, and must only include the fields relevant to that component.
pub fn get<C>(&self) -> Result<C>
where
C: for<'de> Deserialize<'de>,
{
let config = serde_json::from_value(self.config.clone())?;
Ok(config)
}
/// Adds a component-specific configuration to the configuration file. The type `C` must be
/// [serializable](https://docs.rs/serde/*/serde/trait.Serialize.html) into a JSON object, and
/// must only include the fields relevant to that component.
pub fn add<C>(&mut self, config: C) -> Result<()>
where
C: Serialize,
{
let mut config = serde_json::to_value(&config)?;
self.config
.as_object_mut()
.unwrap()
.append(config.as_object_mut().unwrap());
Ok(())
}
}

1
cli/loader/Cargo.toml
View file

@ -13,6 +13,7 @@ repository = "https://github.com/tree-sitter/tree-sitter"
[dependencies]
anyhow = "1.0"
cc = "^1.0.58"
dirs = "3.0"
libloading = "0.7"
once_cell = "1.7"
regex = "1"

43
cli/loader/src/lib.rs
View file

@ -2,7 +2,7 @@ use anyhow::{anyhow, Context, Error, Result};
use libloading::{Library, Symbol};
use once_cell::unsync::OnceCell;
use regex::{Regex, RegexBuilder};
use serde_derive::Deserialize;
use serde_derive::{Deserialize, Serialize};
use std::collections::HashMap;
use std::io::BufReader;
use std::ops::Range;
@ -15,6 +15,26 @@ use tree_sitter::{Language, QueryError};
use tree_sitter_highlight::HighlightConfiguration;
use tree_sitter_tags::{Error as TagsError, TagsConfiguration};
#[derive(Default, Deserialize, Serialize)]
pub struct Config {
#[serde(default)]
#[serde(rename = "parser-directories")]
pub parser_directories: Vec<PathBuf>,
}
impl Config {
pub fn initial() -> Config {
let home_dir = dirs::home_dir().expect("Cannot determine home directory");
Config {
parser_directories: vec![
home_dir.join("github"),
home_dir.join("src"),
home_dir.join("source"),
],
}
}
}
#[cfg(unix)]
const DYLIB_EXTENSION: &'static str = "so";
@ -54,7 +74,14 @@ unsafe impl Send for Loader {}
unsafe impl Sync for Loader {}
impl Loader {
pub fn new(parser_lib_path: PathBuf) -> Self {
pub fn new() -> Result<Self> {
let parser_lib_path = dirs::cache_dir()
.ok_or(anyhow!("Cannot determine cache directory"))?
.join("tree-sitter/lib");
Ok(Self::with_parser_lib_path(parser_lib_path))
}
pub fn with_parser_lib_path(parser_lib_path: PathBuf) -> Self {
Loader {
parser_lib_path,
languages_by_id: Vec::new(),
@ -76,8 +103,15 @@ impl Loader {
self.highlight_names.lock().unwrap().clone()
}
pub fn find_all_languages(&mut self, parser_src_paths: &Vec<PathBuf>) -> Result<()> {
for parser_container_dir in parser_src_paths.iter() {
pub fn find_all_languages(&mut self, config: &Config) -> Result<()> {
if config.parser_directories.is_empty() {
eprintln!("Warning: You have not configured any parser directories!");
eprintln!("Please run `tree-sitter init-config` and edit the resulting");
eprintln!("configuration file to indicate where we should look for");
eprintln!("language grammars.");
eprintln!("");
}
for parser_container_dir in &config.parser_directories {
if let Ok(entries) = fs::read_dir(parser_container_dir) {
for entry in entries {
let entry = entry?;
@ -287,6 +321,7 @@ impl Loader {
.with_context(|| "Failed to compare source and binary timestamps")?;
if recompile {
fs::create_dir_all(&self.parser_lib_path)?;
let mut config = cc::Build::new();
config
.cpp(true)

69
cli/src/config.rs
View file

@ -1,69 +0,0 @@
use super::highlight::Theme;
use serde_derive::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
use std::{env, fs, io};
#[derive(Default, Deserialize, Serialize)]
pub struct Config {
#[serde(skip)]
pub binary_directory: PathBuf,
#[serde(default)]
#[serde(rename = "parser-directories")]
pub parser_directories: Vec<PathBuf>,
#[serde(default)]
pub theme: Theme,
}
impl Config {
pub fn get_path(home_dir: &Path) -> PathBuf {
env::var("TREE_SITTER_DIR")
.map(|p| p.into())
.unwrap_or_else(|_| home_dir.join(".tree-sitter"))
}
pub fn load(home_dir: &Path) -> Self {
let tree_sitter_dir = Self::get_path(home_dir);
let config_path = tree_sitter_dir.join("config.json");
let mut result = fs::read_to_string(&config_path)
.map_err(drop)
.and_then(|json| serde_json::from_str(&json).map_err(drop))
.unwrap_or_else(|_| Self::default());
result.init(home_dir, &tree_sitter_dir);
result
}
pub fn save(&self, home_dir: &Path) -> io::Result<()> {
let tree_sitter_dir = Self::get_path(home_dir);
let config_path = tree_sitter_dir.join("config.json");
let json = serde_json::to_string_pretty(self).expect("Failed to serialize config");
fs::write(config_path, json)
}
pub fn new(home_dir: &Path) -> Self {
let tree_sitter_dir = Self::get_path(home_dir);
let mut result = Self::default();
result.init(home_dir, &tree_sitter_dir);
result
}
fn init(&mut self, home_dir: &Path, tree_sitter_dir: &Path) {
if self.parser_directories.is_empty() {
self.parser_directories = vec![
home_dir.join("github"),
home_dir.join("src"),
home_dir.join("source"),
]
}
let binary_path = tree_sitter_dir.join("bin");
self.binary_directory = binary_path;
fs::create_dir_all(&self.binary_directory).unwrap_or_else(|error| {
panic!(
"Could not find or create parser binary directory {:?}. Error: {}",
self.binary_directory, error
)
});
}
}

7
cli/src/highlight.rs
View file

@ -4,6 +4,7 @@ use anyhow::Result;
use lazy_static::lazy_static;
use serde::ser::SerializeMap;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_derive::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::collections::HashMap;
use std::fmt::Write;
@ -56,6 +57,12 @@ pub struct Theme {
pub highlight_names: Vec<String>,
}
#[derive(Default, Deserialize, Serialize)]
pub struct ThemeConfig {
#[serde(default)]
pub theme: Theme,
}
impl Theme {
pub fn load(path: &path::Path) -> io::Result<Self> {
let json = fs::read_to_string(path)?;

1
cli/src/lib.rs
View file

@ -1,4 +1,3 @@
pub mod config;
pub mod generate;
pub mod highlight;
pub mod logger;

42
cli/src/main.rs
View file

@ -4,9 +4,9 @@ use glob::glob;
use std::path::Path;
use std::{env, fs, u64};
use tree_sitter_cli::{
config, generate, highlight, logger, parse, query, tags, test, test_highlight, util, wasm,
web_ui,
generate, highlight, logger, parse, query, tags, test, test_highlight, util, wasm, web_ui,
};
use tree_sitter_config::Config;
use tree_sitter_loader as loader;
const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION");
@ -174,14 +174,19 @@ fn run() -> Result<()> {
)
.get_matches();
let home_dir = dirs::home_dir().expect("Failed to read home directory");
let current_dir = env::current_dir().unwrap();
let config = config::Config::load(&home_dir);
let mut loader = loader::Loader::new(config.binary_directory.clone());
let config = Config::load()?;
let mut loader = loader::Loader::new()?;
if matches.subcommand_matches("init-config").is_some() {
let config = config::Config::new(&home_dir);
config.save(&home_dir)?;
let mut config = Config::initial()?;
config.add(tree_sitter_loader::Config::initial())?;
config.add(tree_sitter_cli::highlight::ThemeConfig::default())?;
config.save()?;
println!(
"Saved initial configuration to {}",
config.location.display()
);
} else if let Some(matches) = matches.subcommand_matches("generate") {
let grammar_path = matches.value_of("grammar-path");
let report_symbol_name = matches.value_of("report-states-for-rule").or_else(|| {
@ -257,7 +262,8 @@ fn run() -> Result<()> {
let max_path_length = paths.iter().map(|p| p.chars().count()).max().unwrap();
let mut has_error = false;
loader.find_all_languages(&config.parser_directories)?;
let loader_config = config.get()?;
loader.find_all_languages(&loader_config)?;
let should_track_stats = matches.is_present("stat");
let mut stats = parse::Stats::default();
@ -300,7 +306,8 @@ fn run() -> Result<()> {
} else if let Some(matches) = matches.subcommand_matches("query") {
let ordered_captures = matches.values_of("captures").is_some();
let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?;
loader.find_all_languages(&config.parser_directories)?;
let loader_config = config.get()?;
loader.find_all_languages(&loader_config)?;
let language = loader.select_language(
Path::new(&paths[0]),
&current_dir,
@ -321,7 +328,8 @@ fn run() -> Result<()> {
should_test,
)?;
} else if let Some(matches) = matches.subcommand_matches("tags") {
loader.find_all_languages(&config.parser_directories)?;
let loader_config = config.get()?;
loader.find_all_languages(&loader_config)?;
let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?;
tags::generate_tags(
&loader,
@ -331,8 +339,10 @@ fn run() -> Result<()> {
matches.is_present("time"),
)?;
} else if let Some(matches) = matches.subcommand_matches("highlight") {
loader.configure_highlights(&config.theme.highlight_names);
loader.find_all_languages(&config.parser_directories)?;
let theme_config: tree_sitter_cli::highlight::ThemeConfig = config.get()?;
loader.configure_highlights(&theme_config.theme.highlight_names);
let loader_config = config.get()?;
loader.find_all_languages(&loader_config)?;
let time = matches.is_present("time");
let quiet = matches.is_present("quiet");
@ -368,10 +378,11 @@ fn run() -> Result<()> {
if let Some(highlight_config) = language_config.highlight_config(language)? {
let source = fs::read(path)?;
let theme_config = config.get()?;
if html_mode {
highlight::html(
&loader,
&config.theme,
&theme_config,
&source,
highlight_config,
quiet,
@ -380,7 +391,7 @@ fn run() -> Result<()> {
} else {
highlight::ansi(
&loader,
&config.theme,
&theme_config,
&source,
highlight_config,
time,
@ -402,7 +413,8 @@ fn run() -> Result<()> {
let open_in_browser = !matches.is_present("quiet");
web_ui::serve(&current_dir, open_in_browser);
} else if matches.subcommand_matches("dump-languages").is_some() {
loader.find_all_languages(&config.parser_directories)?;
let loader_config = config.get()?;
loader.find_all_languages(&loader_config)?;
for (configuration, language_path) in loader.get_all_language_configurations() {
println!(
concat!(

2
cli/src/tests/helpers/fixtures.rs
View file

@ -8,7 +8,7 @@ use tree_sitter_loader::Loader;
include!("./dirs.rs");
lazy_static! {
static ref TEST_LOADER: Loader = Loader::new(SCRATCH_DIR.clone());
static ref TEST_LOADER: Loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone());
}
pub fn test_loader<'a>() -> &'a Loader {

22
docs/section-4-syntax-highlighting.md
View file

@ -15,20 +15,30 @@ This document explains how the Tree-sitter syntax highlighting system works, usi
All of the files needed to highlight a given language are normally included in the same git repository as the Tree-sitter grammar for that language (for example, [`tree-sitter-javascript`](https://github.com/tree-sitter/tree-sitter-javascript), [`tree-sitter-ruby`](https://github.com/tree-sitter/tree-sitter-ruby)). In order to run syntax highlighting from the command-line, three types of files are needed:
1. Global configuration in `~/.tree-sitter/config.json`
1. Per-user configuration in `~/.config/tree-sitter/config.json`
2. Language configuration in grammar repositories' `package.json` files.
3. Tree queries in the grammars repositories' `queries` folders.
For an example of the language-specific files, see the [`package.json` file](https://github.com/tree-sitter/tree-sitter-ruby/blob/master/package.json) and [`queries` directory](https://github.com/tree-sitter/tree-sitter-ruby/tree/master/queries) in the `tree-sitter-ruby` repository. The following sections describe the behavior of each file.
## Global Configuration
## Per-user Configuration
The Tree-sitter CLI automatically creates a directory in your home folder called `~/.tree-sitter`. This is used to store compiled language binaries, and it can also contain a JSON configuration file. To automatically create a default config file, run this command:
The Tree-sitter CLI automatically creates two directories in your home folder. One holds a JSON configuration file, that lets you customize the behavior of the CLI. The other holds any compiled language parsers that you use.
These directories are created in the "normal" place for your platform:
- On Linux, `~/.config/tree-sitter` and `~/.cache/tree-sitter`
- On Mac, `~/Library/Application Support/tree-sitter` and `~/Library/Caches/tree-sitter`
- On Windows, `C:\Users\[username]\AppData\Roaming\tree-sitter` and `C:\Users\[username]\AppData\Local\tree-sitter`
The CLI will work if there's no config file present, falling back on default values for each configuration option. To create a config file that you can edit, run this command:
```sh
tree-sitter init-config
```
(This will print out the location of the file that it creates so that you can easily find and modify it.)
### Paths
The `tree-sitter highlight` command takes one or more file paths, and tries to automatically determine which language should be used to highlight those files. In order to do this, it needs to know *where* to look for Tree-sitter grammars on your filesystem. You can control this using the `"parser-directories"` key in your configuration file:
@ -42,13 +52,13 @@ The `tree-sitter highlight` command takes one or more file paths, and tries to a
}
```
Currently, any folder within one of these *parser directories* whose name begins with "tree-sitter-" will be treated as a Tree-sitter grammar repository.
Currently, any folder within one of these *parser directories* whose name begins with `tree-sitter-` will be treated as a Tree-sitter grammar repository.
### Theme
The Tree-sitter highlighting system works by annotating ranges of source code with logical "highlight names" like `function.method`, `type.builtin`, `keyword`, etc. In order to decide what *color* should be used for rendering each highlight, a *theme* is needed.
In `~/.tree-sitter/config.json`, the `"theme"` value is an object whose keys are dot-separated highlight names like `function.builtin` or `keyword`, and whose values are JSON expressions that represent text styling parameters.
In your config file, the `"theme"` value is an object whose keys are dot-separated highlight names like `function.builtin` or `keyword`, and whose values are JSON expressions that represent text styling parameters.
#### Highlight Names
@ -187,7 +197,7 @@ We can assign each of these categories a *highlight name* using a query like thi
(function_declaration name: (identifier) @function)
```
Then, in our `~/.tree-sitter/config.json` file, we could map each of these highlight names to a color:
Then, in our config file, we could map each of these highlight names to a color:
```json
{