diff --git a/Cargo.lock b/Cargo.lock index 172e2d78..76a9973a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -608,6 +608,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "serde" version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "serde_derive" @@ -772,6 +775,7 @@ dependencies = [ "tiny_http 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "tree-sitter 0.6.3", "tree-sitter-highlight 0.1.6", + "tree-sitter-tags 0.1.6", "webbrowser 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -786,6 +790,17 @@ dependencies = [ "tree-sitter 0.6.3", ] +[[package]] +name = "tree-sitter-tags" +version = "0.1.6" +dependencies = [ + "regex 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)", + "tree-sitter 0.6.3", +] + [[package]] name = "ucd-util" version = "0.1.3" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 4304d1b6..27706945 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -44,6 +44,10 @@ path = "../lib" version = ">= 0.1.0" path = "../highlight" +[dependencies.tree-sitter-tags] +version = ">= 0.1.0" +path = "../tags" + [dependencies.serde_json] version = "1.0" features = ["preserve_order"] diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 945fe339..97c288a1 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -6,6 +6,7 @@ pub mod loader; pub mod logger; pub mod parse; pub mod query; +pub mod tags; pub mod test; pub mod test_highlight; pub mod util; diff --git a/cli/src/loader.rs b/cli/src/loader.rs index 1f9a1978..b761c137 100644 --- a/cli/src/loader.rs +++ b/cli/src/loader.rs @@ -12,6 +12,7 @@ use std::time::SystemTime; use std::{fs, mem}; use tree_sitter::Language; use tree_sitter_highlight::HighlightConfiguration; +use tree_sitter_tags::TagsConfiguration; #[cfg(unix)] const DYLIB_EXTENSION: &'static str = "so"; @@ -33,6 +34,7 @@ pub struct LanguageConfiguration<'a> { pub locals_filenames: Option>, language_id: usize, highlight_config: OnceCell>, + tags_config: OnceCell>, highlight_names: &'a Mutex>, use_all_highlight_names: bool, } @@ -481,6 +483,7 @@ impl Loader { locals_filenames: config_json.locals.into_vec(), highlights_filenames: config_json.highlights.into_vec(), highlight_config: OnceCell::new(), + tags_config: OnceCell::new(), highlight_names: &*self.highlight_names, use_all_highlight_names: self.use_all_highlight_names, }; @@ -513,6 +516,7 @@ impl Loader { locals_filenames: None, highlights_filenames: None, highlight_config: OnceCell::new(), + tags_config: OnceCell::new(), highlight_names: &*self.highlight_names, use_all_highlight_names: self.use_all_highlight_names, }; @@ -534,32 +538,11 @@ impl<'a> LanguageConfiguration<'a> { pub fn highlight_config(&self, language: Language) -> Result> { self.highlight_config .get_or_try_init(|| { - let queries_path = self.root_path.join("queries"); - let read_queries = |paths: &Option>, default_path: &str| { - if let Some(paths) = paths.as_ref() { - let mut query = String::new(); - for path in paths { - let path = self.root_path.join(path); - query += &fs::read_to_string(&path).map_err(Error::wrap(|| { - format!("Failed to read query file {:?}", path) - }))?; - } - Ok(query) - } else { - let path = queries_path.join(default_path); - if path.exists() { - fs::read_to_string(&path).map_err(Error::wrap(|| { - format!("Failed to read query file {:?}", path) - })) - } else { - Ok(String::new()) - } - } - }; - - let highlights_query = read_queries(&self.highlights_filenames, "highlights.scm")?; - let injections_query = read_queries(&self.injections_filenames, "injections.scm")?; - let locals_query = read_queries(&self.locals_filenames, "locals.scm")?; + let highlights_query = + self.read_queries(&self.highlights_filenames, "highlights.scm")?; + let injections_query = + self.read_queries(&self.injections_filenames, "injections.scm")?; + let locals_query = self.read_queries(&self.locals_filenames, "locals.scm")?; if highlights_query.is_empty() { Ok(None) @@ -587,6 +570,47 @@ impl<'a> LanguageConfiguration<'a> { }) .map(Option::as_ref) } + + pub fn tags_config(&self, language: Language) -> Result> { + self.tags_config + .get_or_try_init(|| { + let tags_query = self.read_queries(&self.highlights_filenames, "tags.scm")?; + let locals_query = self.read_queries(&self.locals_filenames, "locals.scm")?; + if tags_query.is_empty() { + Ok(None) + } else { + TagsConfiguration::new(language, &tags_query, &locals_query) + .map_err(Error::wrap(|| { + format!("Failed to load queries in {:?}", self.root_path) + })) + .map(|config| Some(config)) + } + }) + .map(Option::as_ref) + } + + fn read_queries(&self, paths: &Option>, default_path: &str) -> Result { + if let Some(paths) = paths.as_ref() { + let mut query = String::new(); + for path in paths { + let path = self.root_path.join(path); + query += &fs::read_to_string(&path).map_err(Error::wrap(|| { + format!("Failed to read query file {:?}", path) + }))?; + } + Ok(query) + } else { + let queries_path = self.root_path.join("queries"); + let path = queries_path.join(default_path); + if path.exists() { + fs::read_to_string(&path).map_err(Error::wrap(|| { + format!("Failed to read query file {:?}", path) + })) + } else { + Ok(String::new()) + } + } + } } fn needs_recompile( diff --git a/cli/src/main.rs b/cli/src/main.rs index 79d310fe..0bbf6b25 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -6,8 +6,8 @@ use std::process::exit; use std::{env, fs, u64}; use tree_sitter::Language; use tree_sitter_cli::{ - config, error, generate, highlight, loader, logger, parse, query, test, test_highlight, wasm, - web_ui, + config, error, generate, highlight, loader, logger, parse, query, tags, test, test_highlight, + wasm, web_ui, }; const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION"); @@ -88,6 +88,30 @@ fn run() -> error::Result<()> { .arg(Arg::with_name("scope").long("scope").takes_value(true)) .arg(Arg::with_name("captures").long("captures").short("c")), ) + .subcommand( + SubCommand::with_name("tags") + .arg( + Arg::with_name("format") + .short("f") + .long("format") + .value_name("json|protobuf") + .help("Determine output format (default: json)"), + ) + .arg(Arg::with_name("scope").long("scope").takes_value(true)) + .arg( + Arg::with_name("inputs") + .help("The source file to use") + .index(1) + .required(true) + .multiple(true), + ) + .arg( + Arg::with_name("v") + .short("v") + .multiple(true) + .help("Sets the level of verbosity"), + ), + ) .subcommand( SubCommand::with_name("test") .about("Run a parser's tests") @@ -240,6 +264,38 @@ fn run() -> error::Result<()> { )?; let query_path = Path::new(matches.value_of("query-path").unwrap()); query::query_files_at_paths(language, paths, query_path, ordered_captures)?; + } else if let Some(matches) = matches.subcommand_matches("tags") { + loader.find_all_languages(&config.parser_directories)?; + let paths = collect_paths(matches.values_of("inputs").unwrap())?; + + let mut lang = None; + if let Some(scope) = matches.value_of("scope") { + lang = loader.language_configuration_for_scope(scope)?; + if lang.is_none() { + return Error::err(format!("Unknown scope '{}'", scope)); + } + } + + for path in paths { + let path = Path::new(&path); + let (language, language_config) = match lang { + Some(v) => v, + None => match loader.language_configuration_for_file_name(path)? { + Some(v) => v, + None => { + eprintln!("No language found for path {:?}", path); + continue; + } + }, + }; + + if let Some(tags_config) = language_config.tags_config(language)? { + let source = fs::read(path)?; + tags::generate_tags(tags_config, &source)?; + } else { + eprintln!("No tags config found for path {:?}", path); + } + } } else if let Some(matches) = matches.subcommand_matches("highlight") { loader.configure_highlights(&config.theme.highlight_names); loader.find_all_languages(&config.parser_directories)?; @@ -251,19 +307,17 @@ fn run() -> error::Result<()> { println!("{}", highlight::HTML_HEADER); } - let language_config; + let mut lang = None; if let Some(scope) = matches.value_of("scope") { - language_config = loader.language_configuration_for_scope(scope)?; - if language_config.is_none() { + lang = loader.language_configuration_for_scope(scope)?; + if lang.is_none() { return Error::err(format!("Unknown scope '{}'", scope)); } - } else { - language_config = None; } for path in paths { let path = Path::new(&path); - let (language, language_config) = match language_config { + let (language, language_config) = match lang { Some(v) => v, None => match loader.language_configuration_for_file_name(path)? { Some(v) => v, @@ -274,23 +328,21 @@ fn run() -> error::Result<()> { }, }; - let source = fs::read(path)?; - if let Some(highlight_config) = language_config.highlight_config(language)? { + let source = fs::read(path)?; if html_mode { highlight::html(&loader, &config.theme, &source, highlight_config, time)?; } else { highlight::ansi(&loader, &config.theme, &source, highlight_config, time)?; } } else { - return Error::err(format!("No syntax highlighting query found")); + eprintln!("No syntax highlighting config found for path {:?}", path); } } if html_mode { println!("{}", highlight::HTML_FOOTER); } - } else if let Some(matches) = matches.subcommand_matches("build-wasm") { let grammar_path = current_dir.join(matches.value_of("path").unwrap_or("")); wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?; diff --git a/cli/src/tags.rs b/cli/src/tags.rs new file mode 100644 index 00000000..23d448fc --- /dev/null +++ b/cli/src/tags.rs @@ -0,0 +1,16 @@ +use crate::error::Result; +use std::io; +use tree_sitter_tags::{TagsConfiguration, TagsContext}; + +pub fn generate_tags(config: &TagsConfiguration, source: &[u8]) -> Result<()> { + let mut context = TagsContext::new(); + + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + + for tag in context.generate_tags(config, source) { + serde_json::to_writer(&mut stdout, &tag)?; + } + + Ok(()) +} diff --git a/tags/Cargo.toml b/tags/Cargo.toml new file mode 100644 index 00000000..1d47c951 --- /dev/null +++ b/tags/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "tree-sitter-tags" +description = "Library for extracting tag information" +version = "0.1.6" +authors = [ + "Max Brunsfeld ", + "Patrick Thomson " +] +license = "MIT" +readme = "README.md" +edition = "2018" +keywords = ["incremental", "parsing", "syntax", "highlighting"] +categories = ["parsing", "text-editors"] +repository = "https://github.com/tree-sitter/tree-sitter" + +[lib] +crate-type = ["lib", "staticlib"] + +[dependencies] +regex = "1" +serde_json = "1.0" +serde_derive = "1.0" + +[dependencies.serde] +version = "1.0" +features = ["derive"] + +[dependencies.tree-sitter] +version = ">= 0.3.7" +path = "../lib" diff --git a/tags/src/lib.rs b/tags/src/lib.rs new file mode 100644 index 00000000..3cad20b5 --- /dev/null +++ b/tags/src/lib.rs @@ -0,0 +1,111 @@ +use serde::{Serialize, Serializer}; +use tree_sitter::{Language, Parser, Query, QueryCursor, QueryError}; + +/// Contains the data neeeded to compute tags for code written in a +/// particular language. +pub struct TagsConfiguration { + pub language: Language, + pub query: Query, + locals_pattern_index: usize, +} + +pub struct TagsContext { + parser: Parser, + cursor: QueryCursor, +} + +#[derive(Serialize)] +pub struct Range { + pub start: i64, + pub end: i64, +} + +#[derive(Serialize)] +pub struct Loc { + pub byte_range: Range, + pub span: Span, +} + +#[derive(Serialize)] +pub struct Span { + pub start: Pos, + pub end: Pos, +} + +#[derive(Serialize)] +pub struct Pos { + pub line: i64, + pub column: i64, +} + +pub enum TagKind { + Function, + Method, + Class, + Module, + Call, +} + +#[derive(Serialize)] +pub struct Tag<'a> { + pub kind: TagKind, + pub loc: Loc, + pub name: &'a str, + pub line: &'a str, + pub docs: Option<&'a str>, +} + +impl Serialize for TagKind { + fn serialize(&self, s: S) -> Result + where + S: Serializer, + { + match self { + TagKind::Call => "Call".serialize(s), + TagKind::Module => "Module".serialize(s), + TagKind::Class => "Class".serialize(s), + TagKind::Method => "Method".serialize(s), + TagKind::Function => "Function".serialize(s), + } + } +} + +impl TagsConfiguration { + pub fn new( + language: Language, + tags_query: &str, + locals_query: &str, + ) -> Result { + let query = Query::new(language, &format!("{}{}", tags_query, locals_query))?; + + let locals_query_offset = tags_query.len(); + let mut locals_pattern_index = 0; + for i in 0..(query.pattern_count()) { + let pattern_offset = query.start_byte_for_pattern(i); + if pattern_offset < locals_query_offset { + locals_pattern_index += 1; + } + } + + query.pattern_count(); + query.start_byte_for_pattern(5); + Ok(TagsConfiguration { + language, + query, + locals_pattern_index, + }) + } +} + +impl TagsContext { + pub fn new() -> Self { + TagsContext { + parser: Parser::new(), + cursor: QueryCursor::new(), + } + } + + pub fn generate_tags(&mut self, config: &TagsConfiguration, source: &[u8]) -> Vec { + Vec::new() + } +}