traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into query-pattern-is-definite

Browse source
This commit is contained in:
Max Brunsfeld 2020-08-14 09:31:55 -07:00
commit 1ea29053e1
33 changed files with 2004 additions and 763 deletions
Download patch file Download diff file Expand all files Collapse all files

1
cli/src/tests/mod.rs
View file

@ -3,6 +3,7 @@ mod helpers;
mod highlight_test;
mod node_test;
mod parser_test;
mod pathological_test;
mod query_test;
mod tags_test;
mod test_highlight_test;

15
cli/src/tests/pathological_test.rs Normal file
View file

@ -0,0 +1,15 @@
use super::helpers::allocations;
use super::helpers::fixtures::get_language;
use tree_sitter::Parser;
#[test]
fn test_pathological_example_1() {
let language = "cpp";
let source = r#"*ss<s"ss<sqXqss<s._<s<sq<(qqX<sqss<s.ss<sqsssq<(qss<qssqXqss<s._<s<sq<(qqX<sqss<s.ss<sqsssq<(qss<sqss<sqss<s._<s<sq>(qqX<sqss<s.ss<sqsssq<(qss<sq&=ss<s<sqss<s._<s<sq<(qqX<sqss<s.ss<sqs"#;
allocations::record(|| {
let mut parser = Parser::new();
parser.set_language(get_language(language)).unwrap();
parser.parse(source, None).unwrap();
});
}

195
cli/src/tests/query_test.rs
View file

@ -408,7 +408,7 @@ fn test_query_matches_with_many_overlapping_results() {
)
.unwrap();
let count = 80;
let count = 1024;
// Deeply nested chained function calls:
// a
@ -573,8 +573,8 @@ fn test_query_matches_with_immediate_siblings() {
&[
(0, vec![("parent", "a"), ("child", "b")]),
(0, vec![("parent", "b"), ("child", "c")]),
(1, vec![("last-child", "d")]),
(0, vec![("parent", "c"), ("child", "d")]),
(1, vec![("last-child", "d")]),
(2, vec![("first-element", "w")]),
(2, vec![("first-element", "1")]),
],
@ -758,6 +758,55 @@ fn test_query_matches_with_nested_repetitions() {
});
}
#[test]
fn test_query_matches_with_multiple_repetition_patterns_that_intersect_other_pattern() {
allocations::record(|| {
let language = get_language("javascript");
// When this query sees a comment, it must keep track of several potential
// matches: up to two for each pattern that begins with a comment.
let query = Query::new(
language,
r#"
(call_expression
function: (member_expression
property: (property_identifier) @name)) @ref.method
((comment)* @doc (function_declaration))
((comment)* @doc (generator_function_declaration))
((comment)* @doc (class_declaration))
((comment)* @doc (lexical_declaration))
((comment)* @doc (variable_declaration))
((comment)* @doc (method_definition))
(comment) @comment
"#,
)
.unwrap();
// Here, a series of comments occurs in the middle of a match of the first
// pattern. To avoid exceeding the storage limits and discarding that outer
// match, the comment-related matches need to be managed efficiently.
let source = format!(
"theObject\n{}\n.theMethod()",
" // the comment\n".repeat(64)
);
assert_query_matches(
language,
&query,
&source,
&vec![(7, vec![("comment", "// the comment")]); 64]
.into_iter()
.chain(vec![(
0,
vec![("ref.method", source.as_str()), ("name", "theMethod")],
)])
.collect::<Vec<_>>(),
);
});
}
#[test]
fn test_query_matches_with_leading_zero_or_more_repeated_leaf_nodes() {
allocations::record(|| {
@ -1161,6 +1210,43 @@ fn test_query_matches_with_too_many_permutations_to_track() {
});
}
#[test]
fn test_query_matches_with_alternatives_and_too_many_permutations_to_track() {
allocations::record(|| {
let language = get_language("javascript");
let query = Query::new(
language,
"
(
(comment) @doc
; not immediate
(class_declaration) @class
)
(call_expression
function: [
(identifier) @function
(member_expression property: (property_identifier) @method)
])
",
)
.unwrap();
let source = "/* hi */ a.b(); ".repeat(50);
let mut parser = Parser::new();
parser.set_language(language).unwrap();
let tree = parser.parse(&source, None).unwrap();
let mut cursor = QueryCursor::new();
let matches = cursor.matches(&query, tree.root_node(), to_callback(&source));
assert_eq!(
collect_matches(matches, &query, source.as_str()),
vec![(1, vec![("method", "b")]); 50],
);
});
}
#[test]
fn test_query_matches_with_anonymous_tokens() {
allocations::record(|| {
@ -1215,6 +1301,45 @@ fn test_query_matches_within_byte_range() {
});
}
#[test]
fn test_query_captures_within_byte_range() {
allocations::record(|| {
let language = get_language("c");
let query = Query::new(
language,
"
(call_expression
function: (identifier) @function
arguments: (argument_list (string_literal) @string.arg))
(string_literal) @string
",
)
.unwrap();
let source = r#"DEFUN ("safe-length", Fsafe_length, Ssafe_length, 1, 1, 0)"#;
let mut parser = Parser::new();
parser.set_language(language).unwrap();
let tree = parser.parse(&source, None).unwrap();
let mut cursor = QueryCursor::new();
let captures =
cursor
.set_byte_range(3, 27)
.captures(&query, tree.root_node(), to_callback(source));
assert_eq!(
collect_captures(captures, &query, source),
&[
("function", "DEFUN"),
("string.arg", "\"safe-length\""),
("string", "\"safe-length\""),
]
);
});
}
#[test]
fn test_query_matches_different_queries_same_cursor() {
allocations::record(|| {
@ -1420,12 +1545,17 @@ fn test_query_captures_with_text_conditions() {
((identifier) @function.builtin
(#eq? @function.builtin "require"))
(identifier) @variable
((identifier) @variable
(#not-match? @variable "^(lambda|load)$"))
"#,
)
.unwrap();
let source = "
toad
load
panda
lambda
const ab = require('./ab');
new Cd(EF);
";
@ -1439,6 +1569,8 @@ fn test_query_captures_with_text_conditions() {
assert_eq!(
collect_captures(captures, &query, source),
&[
("variable", "toad"),
("variable", "panda"),
("variable", "ab"),
("function.builtin", "require"),
("variable", "require"),
@ -2074,6 +2206,39 @@ fn test_query_disable_pattern() {
});
}
#[test]
fn test_query_alternative_predicate_prefix() {
allocations::record(|| {
let language = get_language("c");
let query = Query::new(
language,
r#"
((call_expression
function: (identifier) @keyword
arguments: (argument_list
(string_literal) @function))
(.eq? @keyword "DEFUN"))
"#,
)
.unwrap();
let source = r#"
DEFUN ("identity", Fidentity, Sidentity, 1, 1, 0,
doc: /* Return the argument unchanged. */
attributes: const)
(Lisp_Object arg)
{
return arg;
}
"#;
assert_query_matches(
language,
&query,
source,
&[(0, vec![("keyword", "DEFUN"), ("function", "\"identity\"")])],
);
});
}
#[test]
fn test_query_is_definite() {
struct Row {
@ -2086,10 +2251,7 @@ fn test_query_is_definite() {
Row {
language: get_language("python"),
pattern: r#"(expression_statement (string))"#,
results_by_symbol: &[
("expression_statement", false),
("string", false),
],
results_by_symbol: &[("expression_statement", false), ("string", false)],
},
Row {
language: get_language("javascript"),
@ -2102,30 +2264,17 @@ fn test_query_is_definite() {
Row {
language: get_language("javascript"),
pattern: r#"(object "{" "}")"#,
results_by_symbol: &[
("object", false),
("{", true),
("}", true),
],
results_by_symbol: &[("object", false), ("{", true), ("}", true)],
},
Row {
language: get_language("javascript"),
pattern: r#"(pair (property_identifier) ":")"#,
results_by_symbol: &[
("pair", false),
("property_identifier", false),
(":", true),
],
results_by_symbol: &[("pair", false), ("property_identifier", false), (":", true)],
},
Row {
language: get_language("javascript"),
pattern: r#"(object "{" (_) "}")"#,
results_by_symbol: &[
("object", false),
("{", false),
("", false),
("}", true),
],
results_by_symbol: &[("object", false), ("{", false), ("", false), ("}", true)],
},
Row {
language: get_language("javascript"),

197
cli/src/tests/tags_test.rs
View file

@ -1,73 +1,81 @@
use super::helpers::allocations;
use super::helpers::fixtures::{get_language, get_language_queries_path};
use std::ffi::CStr;
use std::ffi::CString;
use std::{fs, ptr, slice, str};
use tree_sitter::Point;
use tree_sitter_tags::c_lib as c;
use tree_sitter_tags::{Error, TagKind, TagsConfiguration, TagsContext};
use tree_sitter_tags::{Error, TagsConfiguration, TagsContext};
const PYTHON_TAG_QUERY: &'static str = r#"
(
(function_definition
name: (identifier) @name
body: (block . (expression_statement (string) @doc))) @function
(#strip! @doc "(^['\"\\s]*)|(['\"\\s]*$)")
(function_definition
name: (identifier) @name
body: (block . (expression_statement (string) @doc))) @definition.function
(#strip! @doc "(^['\"\\s]*)|(['\"\\s]*$)")
)
(function_definition
name: (identifier) @name) @function
name: (identifier) @name) @definition.function
(
(class_definition
name: (identifier) @name
body: (block
. (expression_statement (string) @doc))) @class
(#strip! @doc "(^['\"\\s]*)|(['\"\\s]*$)")
(class_definition
name: (identifier) @name
body: (block
. (expression_statement (string) @doc))) @definition.class
(#strip! @doc "(^['\"\\s]*)|(['\"\\s]*$)")
)
(class_definition
name: (identifier) @name) @class
name: (identifier) @name) @definition.class
(call
function: (identifier) @name) @call
function: (identifier) @name) @reference.call
(call
function: (attribute
attribute: (identifier) @name)) @reference.call
"#;
const JS_TAG_QUERY: &'static str = r#"
(
(comment)* @doc .
(class_declaration
name: (identifier) @name) @class
(#select-adjacent! @doc @class)
name: (identifier) @name) @definition.class
(#select-adjacent! @doc @definition.class)
(#strip! @doc "(^[/\\*\\s]*)|([/\\*\\s]*$)")
)
(
(comment)* @doc .
(method_definition
name: (property_identifier) @name) @method
(#select-adjacent! @doc @method)
name: (property_identifier) @name) @definition.method
(#select-adjacent! @doc @definition.method)
(#strip! @doc "(^[/\\*\\s]*)|([/\\*\\s]*$)")
)
(
(comment)* @doc .
(function_declaration
name: (identifier) @name) @function
(#select-adjacent! @doc @function)
name: (identifier) @name) @definition.function
(#select-adjacent! @doc @definition.function)
(#strip! @doc "(^[/\\*\\s]*)|([/\\*\\s]*$)")
)
(call_expression
function: (identifier) @name) @call
function: (identifier) @name) @reference.call
"#;
const RUBY_TAG_QUERY: &'static str = r#"
(method
name: (identifier) @name) @method
name: (_) @name) @definition.method
(method_call
method: (identifier) @name) @call
method: (identifier) @name) @reference.call
((identifier) @name @call
(setter (identifier) @ignore)
((identifier) @name @reference.call
(#is-not? local))
"#;
@ -94,25 +102,26 @@ fn test_tags_python() {
let tags = tag_context
.generate_tags(&tags_config, source, None)
.unwrap()
.0
.collect::<Result<Vec<_>, _>>()
.unwrap();
assert_eq!(
tags.iter()
.map(|t| (substr(source, &t.name_range), t.kind))
.map(|t| (
substr(source, &t.name_range),
tags_config.syntax_type_name(t.syntax_type_id)
))
.collect::<Vec<_>>(),
&[
("Customer", TagKind::Class),
("age", TagKind::Function),
("compute_age", TagKind::Call),
("Customer", "class"),
("age", "function"),
("compute_age", "call"),
]
);
assert_eq!(substr(source, &tags[0].line_range), " class Customer:");
assert_eq!(
substr(source, &tags[1].line_range),
" def age(self):"
);
assert_eq!(substr(source, &tags[0].line_range), "class Customer:");
assert_eq!(substr(source, &tags[1].line_range), "def age(self):");
assert_eq!(tags[0].docs.as_ref().unwrap(), "Data about a customer");
assert_eq!(tags[1].docs.as_ref().unwrap(), "Get the customer's age");
}
@ -145,17 +154,22 @@ fn test_tags_javascript() {
let tags = tag_context
.generate_tags(&tags_config, source, None)
.unwrap()
.0
.collect::<Result<Vec<_>, _>>()
.unwrap();
assert_eq!(
tags.iter()
.map(|t| (substr(source, &t.name_range), t.kind))
.map(|t| (
substr(source, &t.name_range),
t.span.clone(),
tags_config.syntax_type_name(t.syntax_type_id)
))
.collect::<Vec<_>>(),
&[
("Customer", TagKind::Class),
("getAge", TagKind::Method),
("Agent", TagKind::Class)
("Customer", Point::new(5, 10)..Point::new(5, 18), "class",),
("getAge", Point::new(9, 8)..Point::new(9, 14), "method",),
("Agent", Point::new(15, 10)..Point::new(15, 15), "class",)
]
);
assert_eq!(
@ -166,6 +180,27 @@ fn test_tags_javascript() {
assert_eq!(tags[2].docs, None);
}
#[test]
fn test_tags_columns_measured_in_utf16_code_units() {
let language = get_language("python");
let tags_config = TagsConfiguration::new(language, PYTHON_TAG_QUERY, "").unwrap();
let mut tag_context = TagsContext::new();
let source = r#""❤️❤️❤️".hello_α_ω()"#.as_bytes();
let tag = tag_context
.generate_tags(&tags_config, source, None)
.unwrap()
.0
.next()
.unwrap()
.unwrap();
assert_eq!(substr(source, &tag.name_range), "hello_α_ω");
assert_eq!(tag.span, Point::new(0, 21)..Point::new(0, 32));
assert_eq!(tag.utf16_column_range, 9..18);
}
#[test]
fn test_tags_ruby() {
let language = get_language("ruby");
@ -177,7 +212,7 @@ fn test_tags_ruby() {
"
b = 1
def foo()
def foo=()
c = 1
# a is a method because it is not in scope
@ -197,6 +232,7 @@ fn test_tags_ruby() {
let tags = tag_context
.generate_tags(&tags_config, source.as_bytes(), None)
.unwrap()
.0
.collect::<Result<Vec<_>, _>>()
.unwrap();
@ -204,18 +240,18 @@ fn test_tags_ruby() {
tags.iter()
.map(|t| (
substr(source.as_bytes(), &t.name_range),
t.kind,
tags_config.syntax_type_name(t.syntax_type_id),
(t.span.start.row, t.span.start.column),
))
.collect::<Vec<_>>(),
&[
("foo", TagKind::Method, (2, 0)),
("bar", TagKind::Call, (7, 4)),
("a", TagKind::Call, (7, 8)),
("b", TagKind::Call, (7, 11)),
("each", TagKind::Call, (9, 14)),
("baz", TagKind::Call, (13, 8)),
("b", TagKind::Call, (13, 15),),
("foo=", "method", (2, 4)),
("bar", "call", (7, 4)),
("a", "call", (7, 8)),
("b", "call", (7, 11)),
("each", "call", (9, 14)),
("baz", "call", (13, 8)),
("b", "call", (13, 15),),
]
);
}
@ -239,7 +275,7 @@ fn test_tags_cancellation() {
.generate_tags(&tags_config, source.as_bytes(), Some(&cancellation_flag))
.unwrap();
for (i, tag) in tags.enumerate() {
for (i, tag) in tags.0.enumerate() {
if i == 150 {
cancellation_flag.store(1, Ordering::SeqCst);
}
@ -253,6 +289,47 @@ fn test_tags_cancellation() {
});
}
#[test]
fn test_invalid_capture() {
let language = get_language("python");
let e = TagsConfiguration::new(language, "(identifier) @method", "")
.expect_err("expected InvalidCapture error");
assert_eq!(e, Error::InvalidCapture("method".to_string()));
}
#[test]
fn test_tags_with_parse_error() {
let language = get_language("python");
let tags_config = TagsConfiguration::new(language, PYTHON_TAG_QUERY, "").unwrap();
let mut tag_context = TagsContext::new();
let source = br#"
class Fine: pass
class Bad
"#;
let (tags, failed) = tag_context
.generate_tags(&tags_config, source, None)
.unwrap();
let newtags = tags.collect::<Result<Vec<_>, _>>().unwrap();
assert!(failed, "syntax error should have been detected");
assert_eq!(
newtags.iter()
.map(|t| (
substr(source, &t.name_range),
tags_config.syntax_type_name(t.syntax_type_id)
))
.collect::<Vec<_>>(),
&[
("Fine", "class"),
]
);
}
#[test]
fn test_tags_via_c_api() {
allocations::record(|| {
@ -316,29 +393,29 @@ fn test_tags_via_c_api() {
})
.unwrap();
let syntax_types: Vec<&str> = unsafe {
let mut len: u32 = 0;
let ptr =
c::ts_tagger_syntax_kinds_for_scope_name(tagger, c_scope_name.as_ptr(), &mut len);
slice::from_raw_parts(ptr, len as usize)
.iter()
.map(|i| CStr::from_ptr(*i).to_str().unwrap())
.collect()
};
assert_eq!(
tags.iter()
.map(|tag| (
tag.kind,
syntax_types[tag.syntax_type_id as usize],
&source_code[tag.name_start_byte as usize..tag.name_end_byte as usize],
&source_code[tag.line_start_byte as usize..tag.line_end_byte as usize],
&docs[tag.docs_start_byte as usize..tag.docs_end_byte as usize],
))
.collect::<Vec<_>>(),
&[
(
c::TSTagKind::Function,
"b",
"function b() {",
"one\ntwo\nthree"
),
(
c::TSTagKind::Class,
"C",
"class C extends D {",
"four\nfive"
),
(c::TSTagKind::Call, "b", "b(a);", "")
("function", "b", "function b() {", "one\ntwo\nthree"),
("class", "C", "class C extends D {", "four\nfive"),
("call", "b", "b(a);", "")
]
);