traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tags: Implement select-adjacent! predicate

Browse source
This commit is contained in:
Max Brunsfeld 2020-03-13 13:02:56 -07:00
parent 0457736766
commit b5f2ed83fe
2 changed files with 140 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

43
cli/src/tests/tags_test.rs
View file

@ -8,19 +8,19 @@ fn test_tags_python() {
language, language,
r#" r#"
((function_definition ((function_definition
name: (identifier) @name name: (identifier) @name
body: (block . (expression_statement (string) @doc))) @function body: (block . (expression_statement (string) @doc))) @function
(set! strip @doc "(^['\"\\s]*)|(['\"\\s]*$)")) (strip! @doc "(^['\"\\s]*)|(['\"\\s]*$)"))
(function_definition (function_definition
name: (identifier) @name) @function name: (identifier) @name) @function
((class_definition ((class_definition
name: (identifier) @name name: (identifier) @name
body: (block . (expression_statement (string) @doc))) @class body: (block . (expression_statement (string) @doc))) @class
(set! strip @doc "(^['\"\\s]*)|(['\"\\s]*$)")) (strip! @doc "(^['\"\\s]*)|(['\"\\s]*$)"))
(class_definition (class_definition
name: (identifier) @name) @class name: (identifier) @name) @class
(call (call
function: (identifier) @name) @call function: (identifier) @name) @call
"#, "#,
"", "",
) )
@ -67,17 +67,19 @@ fn test_tags_javascript() {
language, language,
r#" r#"
((* ((*
(comment)+ @doc (comment)+ @doc
. .
(class_declaration (class_declaration
name: (identifier) @name) @class) name: (identifier) @name) @class)
(set! strip @doc "(^[/\\*\\s]*)|([/\\*\\s]*$)")) (select-adjacent! @doc @class)
(strip! @doc "(^[/\\*\\s]*)|([/\\*\\s]*$)"))
((* ((*
(comment)+ @doc (comment)+ @doc
. .
(method_definition (method_definition
name: (property_identifier) @name) @method) name: (property_identifier) @name) @method)
(set! strip @doc "(^[/\\*\\s]*)|([/\\*\\s]*$)")) (select-adjacent! @doc @method)
(strip! @doc "(^[/\\*\\s]*)|([/\\*\\s]*$)"))
"#, "#,
"", "",
) )
@ -88,6 +90,8 @@ fn test_tags_javascript() {
.generate_tags( .generate_tags(
&tags_config, &tags_config,
br#" br#"
// hi
// Data about a customer. // Data about a customer.
// bla bla bla // bla bla bla
class Customer { class Customer {
@ -95,7 +99,6 @@ fn test_tags_javascript() {
* Get the customer's age * Get the customer's age
*/ */
getAge() { getAge() {
} }
} }
"#, "#,

201
tags/src/lib.rs
View file

@ -1,7 +1,9 @@
use regex::Regex; use regex::Regex;
use serde::{Serialize, Serializer}; use serde::{Serialize, Serializer};
use std::{mem, ops, str}; use std::{mem, ops, str};
use tree_sitter::{Language, Node, Parser, Query, QueryCursor, QueryError, Tree}; use tree_sitter::{
Language, Node, Parser, Query, QueryCursor, QueryError, QueryPredicateArg, Tree,
};
/// Contains the data neeeded to compute tags for code written in a /// Contains the data neeeded to compute tags for code written in a
/// particular language. /// particular language.
@ -16,7 +18,7 @@ pub struct TagsConfiguration {
method_capture_index: Option<u32>, method_capture_index: Option<u32>,
module_capture_index: Option<u32>, module_capture_index: Option<u32>,
name_capture_index: Option<u32>, name_capture_index: Option<u32>,
doc_strip_regexes: Vec<Option<Regex>>, pattern_info: Vec<PatternInfo>,
} }
pub struct TagsContext { pub struct TagsContext {
@ -24,6 +26,12 @@ pub struct TagsContext {
cursor: QueryCursor, cursor: QueryCursor,
} }
#[derive(Default)]
struct PatternInfo {
docs_adjacent_capture: Option<u32>,
doc_strip_regex: Option<Regex>,
}
struct TagsIter<'a, I> struct TagsIter<'a, I>
where where
I: Iterator<Item = tree_sitter::QueryMatch<'a>>, I: Iterator<Item = tree_sitter::QueryMatch<'a>>,
@ -105,20 +113,28 @@ impl TagsConfiguration {
*index = Some(i as u32); *index = Some(i as u32);
} }
let doc_strip_regexes = (0..query.pattern_count()) let pattern_info = (0..query.pattern_count())
.map(|pattern_index| { .map(|pattern_index| {
let properties = query.property_settings(pattern_index); let mut info = PatternInfo::default();
for property in properties { if let Some(doc_capture_index) = doc_capture_index {
if property.key.as_ref() == "strip" for predicate in query.general_predicates(pattern_index) {
&& property.capture_id.map(|id| id as u32) == doc_capture_index if predicate.args.get(0)
{ == Some(&QueryPredicateArg::Capture(doc_capture_index))
if let Some(value) = &property.value { {
let regex = Regex::new(value.as_ref())?; match (predicate.operator.as_ref(), predicate.args.get(1)) {
return Ok(Some(regex)); ("select-adjacent!", Some(QueryPredicateArg::Capture(index))) => {
info.docs_adjacent_capture = Some(*index);
}
("strip!", Some(QueryPredicateArg::String(pattern))) => {
let regex = Regex::new(pattern.as_ref())?;
info.doc_strip_regex = Some(regex);
}
_ => {}
}
} }
} }
} }
return Ok(None); return Ok(info);
}) })
.collect::<Result<Vec<_>, Error>>()?; .collect::<Result<Vec<_>, Error>>()?;
@ -133,7 +149,7 @@ impl TagsConfiguration {
doc_capture_index, doc_capture_index,
call_capture_index, call_capture_index,
name_capture_index, name_capture_index,
doc_strip_regexes, pattern_info,
}) })
} }
} }
@ -199,26 +215,68 @@ where
// If there is another match, then compute its tag and add it to the // If there is another match, then compute its tag and add it to the
// tag queue. // tag queue.
if let Some(mat) = self.matches.next() { if let Some(mat) = self.matches.next() {
let mut docs = None; let mut name = None;
let mut call_node = None; let mut doc_nodes = Vec::new();
let mut class_node = None; let mut tag_node = None;
let mut function_node = None; let mut tag_kind = TagKind::Call;
let mut method_node = None; let mut docs_adjacent_node = None;
let mut module_node = None;
let mut name_node = None;
for capture in mat.captures { for capture in mat.captures {
let index = Some(capture.index); let index = Some(capture.index);
let node = Some(capture.node);
if index == self.config.call_capture_index { if index == self.config.pattern_info[mat.pattern_index].docs_adjacent_capture {
call_node = node; docs_adjacent_node = Some(capture.node);
} else if index == self.config.class_capture_index { }
class_node = node;
if index == self.config.name_capture_index {
name = str::from_utf8(&self.source[Some(capture.node)?.byte_range()]).ok();
} else if index == self.config.doc_capture_index { } else if index == self.config.doc_capture_index {
if let Ok(content) = str::from_utf8(&self.source[capture.node.byte_range()]) doc_nodes.push(capture.node);
{ } else if index == self.config.call_capture_index {
tag_node = Some(capture.node);
tag_kind = TagKind::Call;
} else if index == self.config.class_capture_index {
tag_node = Some(capture.node);
tag_kind = TagKind::Class;
} else if index == self.config.function_capture_index {
tag_node = Some(capture.node);
tag_kind = TagKind::Function;
} else if index == self.config.method_capture_index {
tag_node = Some(capture.node);
tag_kind = TagKind::Method;
} else if index == self.config.module_capture_index {
tag_node = Some(capture.node);
tag_kind = TagKind::Module;
}
}
if let (Some(tag_node), Some(name)) = (tag_node, name) {
// If needed, filter the doc nodes based on their ranges, selecting
// only the slice that are adjacent to some specified node.
let mut docs_start_index = 0;
if let (Some(docs_adjacent_node), false) =
(docs_adjacent_node, doc_nodes.is_empty())
{
docs_start_index = doc_nodes.len();
let mut start_row = docs_adjacent_node.start_position().row;
while docs_start_index > 0 {
let doc_node = &doc_nodes[docs_start_index - 1];
let prev_doc_end_row = doc_node.end_position().row;
if prev_doc_end_row + 1 >= start_row {
docs_start_index -= 1;
start_row = doc_node.start_position().row;
} else {
break;
}
}
}
// Generate a doc string from all of the doc nodes, applying any strip regexes.
let mut docs = None;
for doc_node in &doc_nodes[docs_start_index..] {
if let Ok(content) = str::from_utf8(&self.source[doc_node.byte_range()]) {
let content = if let Some(regex) = let content = if let Some(regex) =
&self.config.doc_strip_regexes[mat.pattern_index] &self.config.pattern_info[mat.pattern_index].doc_strip_regex
{ {
regex.replace_all(content, "").to_string() regex.replace_all(content, "").to_string()
} else { } else {
@ -232,68 +290,43 @@ where
} }
} }
} }
} else if index == self.config.function_capture_index {
function_node = node;
} else if index == self.config.method_capture_index {
method_node = node;
} else if index == self.config.module_capture_index {
module_node = node;
} else if index == self.config.name_capture_index {
name_node = node;
} }
}
let source = &self.source; let source = &self.source;
let tag_from_node = |node: Node, kind: TagKind| -> Option<Tag> { let tag_from_node = |node: Node, kind: TagKind| -> Option<Tag> {
let name = str::from_utf8(&source[name_node?.byte_range()]).ok()?; // Slice out the first line of the text corresponding to the node in question.
let mut line_range = node.byte_range();
line_range.end = line_range.end.min(line_range.start + 180);
let line = str::from_utf8(&source[line_range]).ok()?.lines().next()?;
Some(Tag {
name,
line,
kind,
docs,
loc: loc_for_node(node),
})
};
// Slice out the first line of the text corresponding to the node in question. // Only create one tag per node. The tag queue is sorted by node position
let mut line_range = node.byte_range(); // to allow for fast lookup.
line_range.end = line_range.end.min(line_range.start + 180); match self.tag_queue.binary_search_by_key(
let line = str::from_utf8(&source[line_range]).ok()?.lines().next()?; &(tag_node.end_byte(), tag_node.start_byte(), tag_node.id()),
|(node, _, _)| (node.end_byte(), node.start_byte(), node.id()),
Some(Tag { ) {
name, Ok(i) => {
line, let (_, old_idx, tag) = &mut self.tag_queue[i];
loc: loc_for_node(node), if *old_idx > mat.pattern_index {
kind: kind, if let Some(new_tag) = tag_from_node(tag_node, tag_kind) {
docs, *tag = new_tag;
}) *old_idx = mat.pattern_index;
};
for (tag_node, tag_kind) in [
(call_node, TagKind::Call),
(class_node, TagKind::Class),
(function_node, TagKind::Function),
(method_node, TagKind::Method),
(module_node, TagKind::Module),
]
.iter()
.cloned()
{
if let Some(found) = tag_node {
// Only create one tag per node. The tag queue is sorted by node position
// to allow for fast lookup.
match self.tag_queue.binary_search_by_key(
&(found.end_byte(), found.start_byte(), found.id()),
|(node, _, _)| (node.end_byte(), node.start_byte(), node.id()),
) {
Ok(i) => {
let (_, old_idx, tag) = &mut self.tag_queue[i];
if *old_idx > mat.pattern_index {
if let Some(new_tag) = tag_from_node(found, tag_kind) {
*tag = new_tag;
*old_idx = mat.pattern_index;
}
}
}
Err(i) => {
if let Some(tag) = tag_from_node(found, tag_kind) {
self.tag_queue.insert(i, (found, mat.pattern_index, tag))
} }
} }
} }
break; Err(i) => {
if let Some(tag) = tag_from_node(tag_node, tag_kind) {
self.tag_queue.insert(i, (tag_node, mat.pattern_index, tag))
}
}
} }
} }
} }