traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Start work on new tree-sitter-tags crate

Browse source 
Co-Authored-By: Patrick Thomson <patrickt@users.noreply.github.com>
This commit is contained in:
Max Brunsfeld 2020-03-04 14:27:31 -08:00
parent 75a910229b
commit feac368a30
8 changed files with 291 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

15
Cargo.lock generated
View file

@ -608,6 +608,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
name = "serde"
version = "1.0.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "serde_derive"
@ -772,6 +775,7 @@ dependencies = [
"tiny_http 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
"tree-sitter 0.6.3",
"tree-sitter-highlight 0.1.6",
"tree-sitter-tags 0.1.6",
"webbrowser 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -786,6 +790,17 @@ dependencies = [
"tree-sitter 0.6.3",
]
[[package]]
name = "tree-sitter-tags"
version = "0.1.6"
dependencies = [
"regex 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
"tree-sitter 0.6.3",
]
[[package]]
name = "ucd-util"
version = "0.1.3"

4
cli/Cargo.toml
View file

@ -44,6 +44,10 @@ path = "../lib"
version = ">= 0.1.0"
path = "../highlight"
[dependencies.tree-sitter-tags]
version = ">= 0.1.0"
path = "../tags"
[dependencies.serde_json]
version = "1.0"
features = ["preserve_order"]

1
cli/src/lib.rs
View file

@ -6,6 +6,7 @@ pub mod loader;
pub mod logger;
pub mod parse;
pub mod query;
pub mod tags;
pub mod test;
pub mod test_highlight;
pub mod util;

76
cli/src/loader.rs
View file

@ -12,6 +12,7 @@ use std::time::SystemTime;
use std::{fs, mem};
use tree_sitter::Language;
use tree_sitter_highlight::HighlightConfiguration;
use tree_sitter_tags::TagsConfiguration;
#[cfg(unix)]
const DYLIB_EXTENSION: &'static str = "so";
@ -33,6 +34,7 @@ pub struct LanguageConfiguration<'a> {
pub locals_filenames: Option<Vec<String>>,
language_id: usize,
highlight_config: OnceCell<Option<HighlightConfiguration>>,
tags_config: OnceCell<Option<TagsConfiguration>>,
highlight_names: &'a Mutex<Vec<String>>,
use_all_highlight_names: bool,
}
@ -481,6 +483,7 @@ impl Loader {
locals_filenames: config_json.locals.into_vec(),
highlights_filenames: config_json.highlights.into_vec(),
highlight_config: OnceCell::new(),
tags_config: OnceCell::new(),
highlight_names: &*self.highlight_names,
use_all_highlight_names: self.use_all_highlight_names,
};
@ -513,6 +516,7 @@ impl Loader {
locals_filenames: None,
highlights_filenames: None,
highlight_config: OnceCell::new(),
tags_config: OnceCell::new(),
highlight_names: &*self.highlight_names,
use_all_highlight_names: self.use_all_highlight_names,
};
@ -534,32 +538,11 @@ impl<'a> LanguageConfiguration<'a> {
pub fn highlight_config(&self, language: Language) -> Result<Option<&HighlightConfiguration>> {
self.highlight_config
.get_or_try_init(|| {
let queries_path = self.root_path.join("queries");
let read_queries = |paths: &Option<Vec<String>>, default_path: &str| {
if let Some(paths) = paths.as_ref() {
let mut query = String::new();
for path in paths {
let path = self.root_path.join(path);
query += &fs::read_to_string(&path).map_err(Error::wrap(|| {
format!("Failed to read query file {:?}", path)
}))?;
}
Ok(query)
} else {
let path = queries_path.join(default_path);
if path.exists() {
fs::read_to_string(&path).map_err(Error::wrap(|| {
format!("Failed to read query file {:?}", path)
}))
} else {
Ok(String::new())
}
}
};
let highlights_query = read_queries(&self.highlights_filenames, "highlights.scm")?;
let injections_query = read_queries(&self.injections_filenames, "injections.scm")?;
let locals_query = read_queries(&self.locals_filenames, "locals.scm")?;
let highlights_query =
self.read_queries(&self.highlights_filenames, "highlights.scm")?;
let injections_query =
self.read_queries(&self.injections_filenames, "injections.scm")?;
let locals_query = self.read_queries(&self.locals_filenames, "locals.scm")?;
if highlights_query.is_empty() {
Ok(None)
@ -587,6 +570,47 @@ impl<'a> LanguageConfiguration<'a> {
})
.map(Option::as_ref)
}
pub fn tags_config(&self, language: Language) -> Result<Option<&TagsConfiguration>> {
self.tags_config
.get_or_try_init(|| {
let tags_query = self.read_queries(&self.highlights_filenames, "tags.scm")?;
let locals_query = self.read_queries(&self.locals_filenames, "locals.scm")?;
if tags_query.is_empty() {
Ok(None)
} else {
TagsConfiguration::new(language, &tags_query, &locals_query)
.map_err(Error::wrap(|| {
format!("Failed to load queries in {:?}", self.root_path)
}))
.map(|config| Some(config))
}
})
.map(Option::as_ref)
}
fn read_queries(&self, paths: &Option<Vec<String>>, default_path: &str) -> Result<String> {
if let Some(paths) = paths.as_ref() {
let mut query = String::new();
for path in paths {
let path = self.root_path.join(path);
query += &fs::read_to_string(&path).map_err(Error::wrap(|| {
format!("Failed to read query file {:?}", path)
}))?;
}
Ok(query)
} else {
let queries_path = self.root_path.join("queries");
let path = queries_path.join(default_path);
if path.exists() {
fs::read_to_string(&path).map_err(Error::wrap(|| {
format!("Failed to read query file {:?}", path)
}))
} else {
Ok(String::new())
}
}
}
}
fn needs_recompile(

76
cli/src/main.rs
View file

@ -6,8 +6,8 @@ use std::process::exit;
use std::{env, fs, u64};
use tree_sitter::Language;
use tree_sitter_cli::{
config, error, generate, highlight, loader, logger, parse, query, test, test_highlight, wasm,
web_ui,
config, error, generate, highlight, loader, logger, parse, query, tags, test, test_highlight,
wasm, web_ui,
};
const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION");
@ -88,6 +88,30 @@ fn run() -> error::Result<()> {
.arg(Arg::with_name("scope").long("scope").takes_value(true))
.arg(Arg::with_name("captures").long("captures").short("c")),
)
.subcommand(
SubCommand::with_name("tags")
.arg(
Arg::with_name("format")
.short("f")
.long("format")
.value_name("json|protobuf")
.help("Determine output format (default: json)"),
)
.arg(Arg::with_name("scope").long("scope").takes_value(true))
.arg(
Arg::with_name("inputs")
.help("The source file to use")
.index(1)
.required(true)
.multiple(true),
)
.arg(
Arg::with_name("v")
.short("v")
.multiple(true)
.help("Sets the level of verbosity"),
),
)
.subcommand(
SubCommand::with_name("test")
.about("Run a parser's tests")
@ -240,6 +264,38 @@ fn run() -> error::Result<()> {
)?;
let query_path = Path::new(matches.value_of("query-path").unwrap());
query::query_files_at_paths(language, paths, query_path, ordered_captures)?;
} else if let Some(matches) = matches.subcommand_matches("tags") {
loader.find_all_languages(&config.parser_directories)?;
let paths = collect_paths(matches.values_of("inputs").unwrap())?;
let mut lang = None;
if let Some(scope) = matches.value_of("scope") {
lang = loader.language_configuration_for_scope(scope)?;
if lang.is_none() {
return Error::err(format!("Unknown scope '{}'", scope));
}
}
for path in paths {
let path = Path::new(&path);
let (language, language_config) = match lang {
Some(v) => v,
None => match loader.language_configuration_for_file_name(path)? {
Some(v) => v,
None => {
eprintln!("No language found for path {:?}", path);
continue;
}
},
};
if let Some(tags_config) = language_config.tags_config(language)? {
let source = fs::read(path)?;
tags::generate_tags(tags_config, &source)?;
} else {
eprintln!("No tags config found for path {:?}", path);
}
}
} else if let Some(matches) = matches.subcommand_matches("highlight") {
loader.configure_highlights(&config.theme.highlight_names);
loader.find_all_languages(&config.parser_directories)?;
@ -251,19 +307,17 @@ fn run() -> error::Result<()> {
println!("{}", highlight::HTML_HEADER);
}
let language_config;
let mut lang = None;
if let Some(scope) = matches.value_of("scope") {
language_config = loader.language_configuration_for_scope(scope)?;
if language_config.is_none() {
lang = loader.language_configuration_for_scope(scope)?;
if lang.is_none() {
return Error::err(format!("Unknown scope '{}'", scope));
}
} else {
language_config = None;
}
for path in paths {
let path = Path::new(&path);
let (language, language_config) = match language_config {
let (language, language_config) = match lang {
Some(v) => v,
None => match loader.language_configuration_for_file_name(path)? {
Some(v) => v,
@ -274,23 +328,21 @@ fn run() -> error::Result<()> {
},
};
let source = fs::read(path)?;
if let Some(highlight_config) = language_config.highlight_config(language)? {
let source = fs::read(path)?;
if html_mode {
highlight::html(&loader, &config.theme, &source, highlight_config, time)?;
} else {
highlight::ansi(&loader, &config.theme, &source, highlight_config, time)?;
}
} else {
return Error::err(format!("No syntax highlighting query found"));
eprintln!("No syntax highlighting config found for path {:?}", path);
}
}
if html_mode {
println!("{}", highlight::HTML_FOOTER);
}
} else if let Some(matches) = matches.subcommand_matches("build-wasm") {
let grammar_path = current_dir.join(matches.value_of("path").unwrap_or(""));
wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?;

16
cli/src/tags.rs Normal file
View file

@ -0,0 +1,16 @@
use crate::error::Result;
use std::io;
use tree_sitter_tags::{TagsConfiguration, TagsContext};
pub fn generate_tags(config: &TagsConfiguration, source: &[u8]) -> Result<()> {
let mut context = TagsContext::new();
let stdout = io::stdout();
let mut stdout = stdout.lock();
for tag in context.generate_tags(config, source) {
serde_json::to_writer(&mut stdout, &tag)?;
}
Ok(())
}

30
tags/Cargo.toml Normal file
View file

@ -0,0 +1,30 @@
[package]
name = "tree-sitter-tags"
description = "Library for extracting tag information"
version = "0.1.6"
authors = [
"Max Brunsfeld <maxbrunsfeld@gmail.com>",
"Patrick Thomson <patrickt@github.com>"
]
license = "MIT"
readme = "README.md"
edition = "2018"
keywords = ["incremental", "parsing", "syntax", "highlighting"]
categories = ["parsing", "text-editors"]
repository = "https://github.com/tree-sitter/tree-sitter"
[lib]
crate-type = ["lib", "staticlib"]
[dependencies]
regex = "1"
serde_json = "1.0"
serde_derive = "1.0"
[dependencies.serde]
version = "1.0"
features = ["derive"]
[dependencies.tree-sitter]
version = ">= 0.3.7"
path = "../lib"

111
tags/src/lib.rs Normal file
View file

@ -0,0 +1,111 @@
use serde::{Serialize, Serializer};
use tree_sitter::{Language, Parser, Query, QueryCursor, QueryError};
/// Contains the data neeeded to compute tags for code written in a
/// particular language.
pub struct TagsConfiguration {
pub language: Language,
pub query: Query,
locals_pattern_index: usize,
}
pub struct TagsContext {
parser: Parser,
cursor: QueryCursor,
}
#[derive(Serialize)]
pub struct Range {
pub start: i64,
pub end: i64,
}
#[derive(Serialize)]
pub struct Loc {
pub byte_range: Range,
pub span: Span,
}
#[derive(Serialize)]
pub struct Span {
pub start: Pos,
pub end: Pos,
}
#[derive(Serialize)]
pub struct Pos {
pub line: i64,
pub column: i64,
}
pub enum TagKind {
Function,
Method,
Class,
Module,
Call,
}
#[derive(Serialize)]
pub struct Tag<'a> {
pub kind: TagKind,
pub loc: Loc,
pub name: &'a str,
pub line: &'a str,
pub docs: Option<&'a str>,
}
impl Serialize for TagKind {
fn serialize<S>(&self, s: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
match self {
TagKind::Call => "Call".serialize(s),
TagKind::Module => "Module".serialize(s),
TagKind::Class => "Class".serialize(s),
TagKind::Method => "Method".serialize(s),
TagKind::Function => "Function".serialize(s),
}
}
}
impl TagsConfiguration {
pub fn new(
language: Language,
tags_query: &str,
locals_query: &str,
) -> Result<Self, QueryError> {
let query = Query::new(language, &format!("{}{}", tags_query, locals_query))?;
let locals_query_offset = tags_query.len();
let mut locals_pattern_index = 0;
for i in 0..(query.pattern_count()) {
let pattern_offset = query.start_byte_for_pattern(i);
if pattern_offset < locals_query_offset {
locals_pattern_index += 1;
}
}
query.pattern_count();
query.start_byte_for_pattern(5);
Ok(TagsConfiguration {
language,
query,
locals_pattern_index,
})
}
}
impl TagsContext {
pub fn new() -> Self {
TagsContext {
parser: Parser::new(),
cursor: QueryCursor::new(),
}
}
pub fn generate_tags(&mut self, config: &TagsConfiguration, source: &[u8]) -> Vec<Tag> {
Vec::new()
}
}