traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement include-children directive in injection queries

Browse source
This commit is contained in:
Max Brunsfeld 2019-10-14 16:55:14 -07:00
parent b3809274f0
commit 060e00463d
6 changed files with 86 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

8
cli/src/tests/highlight_test.rs
View file

@ -124,9 +124,6 @@ fn test_highlighting_injected_html_in_javascript() {
fn test_highlighting_injected_javascript_in_html_mini() {
let source = "<script>const x = new Thing();</script>";
eprintln!("HTML {:?}", HTML_HIGHLIGHT.language);
eprintln!("JavaScript {:?}", JS_HIGHLIGHT.language);
assert_eq!(
&to_token_vector(source, &HTML_HIGHLIGHT).unwrap(),
&[vec![
@ -377,7 +374,10 @@ fn test_highlighting_with_content_children_included() {
("(", vec!["punctuation.bracket"]),
(")", vec!["punctuation.bracket"]),
],
vec![(")", vec!["punctuation.bracket"]), (";", vec![]),]
vec![
(")", vec!["punctuation.bracket"]),
(";", vec!["punctuation.delimiter"]),
]
],
);
}

38
cli/src/tests/parser_test.rs
View file

@ -7,7 +7,7 @@ use std::{thread, time};
use tree_sitter::{InputEdit, LogType, Parser, Point, Range};
#[test]
fn test_basic_parsing() {
fn test_parsing_simple_string() {
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
@ -26,7 +26,11 @@ fn test_basic_parsing() {
assert_eq!(
root_node.to_sexp(),
"(source_file (struct_item (type_identifier) (field_declaration_list)) (function_item (identifier) (parameters) (block)))"
concat!(
"(source_file ",
"(struct_item name: (type_identifier) body: (field_declaration_list)) ",
"(function_item name: (identifier) parameters: (parameters) body: (block)))"
)
);
let struct_node = root_node.child(0).unwrap();
@ -118,7 +122,17 @@ fn test_parsing_with_custom_utf8_input() {
.unwrap();
let root = tree.root_node();
assert_eq!(root.to_sexp(), "(source_file (function_item (visibility_modifier) (identifier) (parameters) (block (integer_literal))))");
assert_eq!(
root.to_sexp(),
concat!(
"(source_file ",
"(function_item ",
"(visibility_modifier) ",
"name: (identifier) ",
"parameters: (parameters) ",
"body: (block (integer_literal))))"
)
);
assert_eq!(root.kind(), "source_file");
assert_eq!(root.has_error(), false);
assert_eq!(root.child(0).unwrap().kind(), "function_item");
@ -154,7 +168,10 @@ fn test_parsing_with_custom_utf16_input() {
.unwrap();
let root = tree.root_node();
assert_eq!(root.to_sexp(), "(source_file (function_item (visibility_modifier) (identifier) (parameters) (block (integer_literal))))");
assert_eq!(
root.to_sexp(),
"(source_file (function_item (visibility_modifier) name: (identifier) parameters: (parameters) body: (block (integer_literal))))"
);
assert_eq!(root.kind(), "source_file");
assert_eq!(root.has_error(), false);
assert_eq!(root.child(0).unwrap().kind(), "function_item");
@ -175,7 +192,10 @@ fn test_parsing_with_callback_returning_owned_strings() {
.unwrap();
let root = tree.root_node();
assert_eq!(root.to_sexp(), "(source_file (function_item (visibility_modifier) (identifier) (parameters) (block (integer_literal))))");
assert_eq!(
root.to_sexp(),
"(source_file (function_item (visibility_modifier) name: (identifier) parameters: (parameters) body: (block (integer_literal))))"
);
}
#[test]
@ -192,7 +212,7 @@ fn test_parsing_text_with_byte_order_mark() {
.unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(source_file (function_item (identifier) (parameters) (block)))"
"(source_file (function_item name: (identifier) parameters: (parameters) body: (block)))"
);
assert_eq!(tree.root_node().start_byte(), 2);
@ -200,7 +220,7 @@ fn test_parsing_text_with_byte_order_mark() {
let mut tree = parser.parse("\u{FEFF}fn a() {}", None).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(source_file (function_item (identifier) (parameters) (block)))"
"(source_file (function_item name: (identifier) parameters: (parameters) body: (block)))"
);
assert_eq!(tree.root_node().start_byte(), 3);
@ -216,7 +236,7 @@ fn test_parsing_text_with_byte_order_mark() {
let mut tree = parser.parse(" \u{FEFF}fn a() {}", Some(&tree)).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(source_file (ERROR (UNEXPECTED 65279)) (function_item (identifier) (parameters) (block)))"
"(source_file (ERROR (UNEXPECTED 65279)) (function_item name: (identifier) parameters: (parameters) body: (block)))"
);
assert_eq!(tree.root_node().start_byte(), 1);
@ -232,7 +252,7 @@ fn test_parsing_text_with_byte_order_mark() {
let tree = parser.parse("\u{FEFF}fn a() {}", Some(&tree)).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(source_file (function_item (identifier) (parameters) (block)))"
"(source_file (function_item name: (identifier) parameters: (parameters) body: (block)))"
);
assert_eq!(tree.root_node().start_byte(), 3);
}

8
cli/src/tests/query_test.rs
View file

@ -16,7 +16,13 @@ fn test_query_errors_on_invalid_syntax() {
// Mismatched parens
assert_eq!(
Query::new(language, "(if_statement"),
Err(QueryError::Syntax("Unexpected EOF".to_string()))
Err(QueryError::Syntax(
[
"(if_statement", //
" ^",
]
.join("\n")
))
);
assert_eq!(
Query::new(language, "; comment 1\n; comment 2\n (if_statement))"),

68
highlight/src/lib.rs
View file

@ -73,6 +73,7 @@ where
layers: Vec<HighlightIterLayer<'a>>,
iter_count: usize,
next_event: Option<HighlightEvent>,
last_highlight_range: Option<(usize, usize, usize)>,
}
struct HighlightIterLayer<'a> {
@ -252,6 +253,7 @@ impl Highlighter {
iter_count: 0,
layers: vec![layer],
next_event: None,
last_highlight_range: None,
})
}
}
@ -397,7 +399,8 @@ impl<'a> HighlightIterLayer<'a> {
// First, sort scope boundaries by their byte offset in the document. At a
// given position, emit scope endings before scope beginnings. Finally, emit
// scope boundaries from outer layers first.
fn sort_key(&mut self) -> Option<(usize, bool, usize)> {
fn sort_key(&mut self) -> Option<(usize, bool, isize)> {
let depth = -(self.depth as isize);
let next_start = self
.captures
.peek()
@ -406,13 +409,13 @@ impl<'a> HighlightIterLayer<'a> {
match (next_start, next_end) {
(Some(start), Some(end)) => {
if start < end {
Some((start, true, self.depth))
Some((start, true, depth))
} else {
Some((end, false, self.depth))
Some((end, false, depth))
}
}
(Some(i), None) => Some((i, true, self.depth)),
(None, Some(j)) => Some((j, false, self.depth)),
(Some(i), None) => Some((i, true, depth)),
(None, Some(j)) => Some((j, false, depth)),
_ => None,
}
}
@ -589,7 +592,7 @@ where
// captures being intermixed with other captures related to local variables
// and syntax highlighting.
let source = self.source;
let mut injections = Vec::<(usize, Option<&str>, Vec<Node>)>::new();
let mut injections = Vec::<(usize, Option<&str>, Vec<Node>, bool)>::new();
for mat in self.injections_cursor.matches(
&layer.config.injections_query,
site_node,
@ -600,7 +603,7 @@ where
{
entry
} else {
injections.push((mat.pattern_index, None, Vec::new()));
injections.push((mat.pattern_index, None, Vec::new(), false));
injections.last_mut().unwrap()
};
@ -618,27 +621,29 @@ where
}
}
for (pattern_index, language, _) in injections.iter_mut() {
// In addition to specifying the language name via the text of a captured node,
// it can also be hard-coded via a `(set! injection.language <language>)`
// predicate.
if language.is_none() {
*language = layer
.config
.query
.property_settings(*pattern_index)
.iter()
.find_map(|prop| {
if prop.key.as_ref() == "injection.language" {
prop.value.as_ref().map(|s| s.as_ref())
} else {
None
for (pattern_index, language, _, include_children) in injections.iter_mut() {
for prop in layer.config.query.property_settings(*pattern_index) {
match prop.key.as_ref() {
// In addition to specifying the language name via the text of a
// captured node, it can also be hard-coded via a `set!` predicate
// that sets the injection.language key.
"injection.language" => {
if language.is_none() {
*language = prop.value.as_ref().map(|s| s.as_ref())
}
});
}
// By default, injections do not include the *children* of an
// `injection.content` node - only the ranges that belong to the
// node itself. This can be changed using a `set!` predicate that
// sets the `injection.include-children` key.
"injection.include-children" => *include_children = true,
_ => {}
}
}
}
for (_, language, content_nodes) in injections {
for (_, language, content_nodes, include_children) in injections {
// If a language is found with the given name, then add a new language layer
// to the highlighted document.
if let Some(config) = language.and_then(&self.injection_callback) {
@ -649,7 +654,8 @@ where
self.context,
self.cancellation_flag,
self.layers[0].depth + 1,
self.layers[0].intersect_ranges(&content_nodes, false),
self.layers[0]
.intersect_ranges(&content_nodes, include_children),
) {
Ok(layer) => self.insert_layer(layer),
Err(e) => return Some(Err(e)),
@ -736,10 +742,17 @@ where
break;
}
let mut has_highlight = true;
if let Some((last_start, last_end, last_depth)) = self.last_highlight_range {
if range.start == last_start && range.end == last_end && layer.depth < last_depth {
has_highlight = false;
}
}
// If the current node was found to be a local variable, then skip over any
// highlighting patterns that are disabled for local variables.
let mut has_highlight = true;
while (definition_highlight.is_some() || reference_highlight.is_some())
while has_highlight
&& (definition_highlight.is_some() || reference_highlight.is_some())
&& layer.config.non_local_variable_patterns[pattern_index]
{
has_highlight = false;
@ -780,6 +793,7 @@ where
// Emit a scope start event and push the node's end position to the stack.
if let Some(highlight) = reference_highlight.or(current_highlight) {
self.last_highlight_range = Some((range.start, range.end, layer.depth));
layer.highlight_end_stack.push(range.end);
return self
.emit_event(range.start, Some(HighlightEvent::HighlightStart(highlight)));

5
lib/binding_web/test/parser-test.js
View file

@ -185,7 +185,10 @@ describe("Parser", () => {
tree = parser.parse("const x: &'static str = r###\"hello\"###;");
assert.equal(
tree.rootNode.toString(),
'(source_file (const_item (identifier) (reference_type (lifetime (identifier)) (primitive_type)) (raw_string_literal)))'
'(source_file (const_item ' +
'name: (identifier) ' +
'type: (reference_type (lifetime (identifier)) type: (primitive_type)) ' +
'value: (raw_string_literal)))'
);
}).timeout(5000);

2
script/fetch-fixtures
View file

@ -31,5 +31,5 @@ fetch_grammar javascript highlight-queries
fetch_grammar json master
fetch_grammar python master
fetch_grammar ruby master
fetch_grammar rust master
fetch_grammar rust highlight-queries
fetch_grammar typescript master