From a1a241b01328c03b48eb61af61ea4e5bf83bf251 Mon Sep 17 00:00:00 2001 From: Hendrik van Antwerpen Date: Tue, 7 Dec 2021 16:17:22 +0100 Subject: [PATCH] Expose quantifiers per pattern, instead of merging for all patterns in a query --- cli/src/tests/query_test.rs | 94 +++++++++++++++++++++++------------ lib/binding_rust/bindings.rs | 6 ++- lib/binding_rust/lib.rs | 32 ++++++++---- lib/include/tree_sitter/api.h | 4 +- lib/src/query.c | 23 ++++++--- 5 files changed, 108 insertions(+), 51 deletions(-) diff --git a/cli/src/tests/query_test.rs b/cli/src/tests/query_test.rs index fcb4179e..0513f3a1 100644 --- a/cli/src/tests/query_test.rs +++ b/cli/src/tests/query_test.rs @@ -3824,7 +3824,7 @@ fn test_capture_quantifiers() { description: &'static str, language: Language, pattern: &'static str, - capture_quantifiers: &'static [(&'static str, CaptureQuantifier)], + capture_quantifiers: &'static [(usize, &'static str, CaptureQuantifier)], } let rows = &[ @@ -3835,7 +3835,7 @@ fn test_capture_quantifiers() { pattern: r#" (module) @mod "#, - capture_quantifiers: &[("mod", CaptureQuantifier::One)], + capture_quantifiers: &[(0, "mod", CaptureQuantifier::One)], }, Row { description: "Nested list capture capture", @@ -3844,8 +3844,8 @@ fn test_capture_quantifiers() { (array (_)* @elems) @array "#, capture_quantifiers: &[ - ("array", CaptureQuantifier::One), - ("elems", CaptureQuantifier::ZeroOrMore), + (0, "array", CaptureQuantifier::One), + (0, "elems", CaptureQuantifier::ZeroOrMore), ], }, Row { @@ -3855,8 +3855,8 @@ fn test_capture_quantifiers() { (array (_)+ @elems) @array "#, capture_quantifiers: &[ - ("array", CaptureQuantifier::One), - ("elems", CaptureQuantifier::OneOrMore), + (0, "array", CaptureQuantifier::One), + (0, "elems", CaptureQuantifier::OneOrMore), ], }, // Nested quantifiers @@ -3867,9 +3867,9 @@ fn test_capture_quantifiers() { (array (call_expression (arguments (_) @arg))? @call) @array "#, capture_quantifiers: &[ - ("array", CaptureQuantifier::One), - ("call", CaptureQuantifier::ZeroOrOne), - ("arg", CaptureQuantifier::ZeroOrOne), + (0, "array", CaptureQuantifier::One), + (0, "call", CaptureQuantifier::ZeroOrOne), + (0, "arg", CaptureQuantifier::ZeroOrOne), ], }, Row { @@ -3879,9 +3879,9 @@ fn test_capture_quantifiers() { (array (call_expression (arguments (_)? @arg))+ @call) @array "#, capture_quantifiers: &[ - ("array", CaptureQuantifier::One), - ("call", CaptureQuantifier::OneOrMore), - ("arg", CaptureQuantifier::ZeroOrMore), + (0, "array", CaptureQuantifier::One), + (0, "call", CaptureQuantifier::OneOrMore), + (0, "arg", CaptureQuantifier::ZeroOrMore), ], }, Row { @@ -3891,9 +3891,9 @@ fn test_capture_quantifiers() { (array (call_expression (arguments (_)+ @args))? @call) @array "#, capture_quantifiers: &[ - ("array", CaptureQuantifier::One), - ("call", CaptureQuantifier::ZeroOrOne), - ("args", CaptureQuantifier::ZeroOrMore), + (0, "array", CaptureQuantifier::One), + (0, "call", CaptureQuantifier::ZeroOrOne), + (0, "args", CaptureQuantifier::ZeroOrMore), ], }, // Quantifiers in alternations @@ -3904,7 +3904,7 @@ fn test_capture_quantifiers() { (function_declaration name:(identifier) @name) (call_expression function:(identifier) @name) ]"#, - capture_quantifiers: &[("name", CaptureQuantifier::One)], + capture_quantifiers: &[(0, "name", CaptureQuantifier::One)], }, Row { description: "capture appears in some alternatives", @@ -3914,8 +3914,8 @@ fn test_capture_quantifiers() { (function) ] @fun"#, capture_quantifiers: &[ - ("fun", CaptureQuantifier::One), - ("name", CaptureQuantifier::ZeroOrOne), + (0, "fun", CaptureQuantifier::One), + (0, "name", CaptureQuantifier::ZeroOrOne), ], }, Row { @@ -3926,8 +3926,8 @@ fn test_capture_quantifiers() { (new_expression arguments:(arguments (_)? @args)) ] @call"#, capture_quantifiers: &[ - ("call", CaptureQuantifier::One), - ("args", CaptureQuantifier::ZeroOrMore), + (0, "call", CaptureQuantifier::One), + (0, "args", CaptureQuantifier::ZeroOrMore), ], }, // Quantifiers in siblings @@ -3938,9 +3938,9 @@ fn test_capture_quantifiers() { (call_expression (arguments (identifier)? @self (_)* @args)) @call "#, capture_quantifiers: &[ - ("call", CaptureQuantifier::One), - ("self", CaptureQuantifier::ZeroOrOne), - ("args", CaptureQuantifier::ZeroOrMore), + (0, "call", CaptureQuantifier::One), + (0, "self", CaptureQuantifier::ZeroOrOne), + (0, "args", CaptureQuantifier::ZeroOrMore), ], }, Row { @@ -3950,13 +3950,13 @@ fn test_capture_quantifiers() { (call_expression (arguments (identifier) @args (_)* @args)) @call "#, capture_quantifiers: &[ - ("call", CaptureQuantifier::One), - ("args", CaptureQuantifier::OneOrMore), + (0, "call", CaptureQuantifier::One), + (0, "args", CaptureQuantifier::OneOrMore), ], }, - // Combined nesting, + // Combined scenarios Row { - description: "combined nesting, alterantives, and siblings", + description: "combined nesting, alternatives, and siblings", language: get_language("javascript"), pattern: r#" (array @@ -3969,10 +3969,38 @@ fn test_capture_quantifiers() { ) @array "#, capture_quantifiers: &[ - ("array", CaptureQuantifier::One), - ("call", CaptureQuantifier::OneOrMore), - ("self", CaptureQuantifier::ZeroOrMore), - ("args", CaptureQuantifier::ZeroOrMore), + (0, "array", CaptureQuantifier::One), + (0, "call", CaptureQuantifier::OneOrMore), + (0, "self", CaptureQuantifier::ZeroOrMore), + (0, "args", CaptureQuantifier::ZeroOrMore), + ], + }, + // Multiple patterns + Row { + description: "multiple patterns", + language: get_language("javascript"), + pattern: r#" + (function_declaration name: (identifier) @x) + (statement_identifier) @y + (property_identifier)+ @z + (array (identifier)* @x) + "#, + capture_quantifiers: &[ + // x + (0, "x", CaptureQuantifier::One), + (1, "x", CaptureQuantifier::Zero), + (2, "x", CaptureQuantifier::Zero), + (3, "x", CaptureQuantifier::ZeroOrMore), + // y + (0, "y", CaptureQuantifier::Zero), + (1, "y", CaptureQuantifier::One), + (2, "y", CaptureQuantifier::Zero), + (3, "y", CaptureQuantifier::Zero), + // z + (0, "z", CaptureQuantifier::Zero), + (1, "z", CaptureQuantifier::Zero), + (2, "z", CaptureQuantifier::OneOrMore), + (3, "z", CaptureQuantifier::Zero), ], }, ]; @@ -3988,9 +4016,9 @@ fn test_capture_quantifiers() { } eprintln!(" query example: {:?}", row.description); let query = Query::new(row.language, row.pattern).unwrap(); - for (capture, expected_quantifier) in row.capture_quantifiers { + for (pattern, capture, expected_quantifier) in row.capture_quantifiers { let index = query.capture_index_for_name(capture).unwrap(); - let actual_quantifier = query.capture_quantifiers()[index as usize]; + let actual_quantifier = query.capture_quantifiers(*pattern)[index as usize]; assert_eq!( actual_quantifier, *expected_quantifier, diff --git a/lib/binding_rust/bindings.rs b/lib/binding_rust/bindings.rs index 7d5a6b6b..686fec06 100644 --- a/lib/binding_rust/bindings.rs +++ b/lib/binding_rust/bindings.rs @@ -675,7 +675,11 @@ extern "C" { #[doc = " Get the quantifier of the query's captures, or one of the query's string"] #[doc = " literals. Each capture and string is associated with a numeric id based"] #[doc = " on the order that it appeared in the query's source."] - pub fn ts_query_capture_quantifier_for_id(arg1: *const TSQuery, id: u32) -> TSQuantifier; + pub fn ts_query_capture_quantifier_for_id( + arg1: *const TSQuery, + pattern_id: u32, + capture_id: u32, + ) -> TSQuantifier; } extern "C" { pub fn ts_query_string_value_for_id( diff --git a/lib/binding_rust/lib.rs b/lib/binding_rust/lib.rs index a66cba2f..cada66c6 100644 --- a/lib/binding_rust/lib.rs +++ b/lib/binding_rust/lib.rs @@ -98,7 +98,7 @@ pub struct TreeCursor<'a>(ffi::TSTreeCursor, PhantomData<&'a ()>); pub struct Query { ptr: NonNull, capture_names: Vec, - capture_quantifiers: Vec, + capture_quantifiers: Vec>, text_predicates: Vec>, property_settings: Vec>, property_predicates: Vec>, @@ -108,19 +108,21 @@ pub struct Query { /// A quantifier for captures #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum CaptureQuantifier { - One, - OneOrMore, + Zero, ZeroOrOne, ZeroOrMore, + One, + OneOrMore, } impl From for CaptureQuantifier { fn from(value: ffi::TSQuantifier) -> Self { match value { - ffi::TSQuantifier_One => CaptureQuantifier::One, - ffi::TSQuantifier_OneOrMore => CaptureQuantifier::OneOrMore, + ffi::TSQuantifier_Zero => CaptureQuantifier::Zero, ffi::TSQuantifier_ZeroOrOne => CaptureQuantifier::ZeroOrOne, ffi::TSQuantifier_ZeroOrMore => CaptureQuantifier::ZeroOrMore, + ffi::TSQuantifier_One => CaptureQuantifier::One, + ffi::TSQuantifier_OneOrMore => CaptureQuantifier::OneOrMore, _ => panic!("Unrecognized quantifier: {}", value), } } @@ -1328,7 +1330,7 @@ impl Query { let mut result = Query { ptr: unsafe { NonNull::new_unchecked(ptr) }, capture_names: Vec::with_capacity(capture_count as usize), - capture_quantifiers: Vec::with_capacity(capture_count as usize), + capture_quantifiers: Vec::with_capacity(pattern_count as usize), text_predicates: Vec::with_capacity(pattern_count), property_predicates: Vec::with_capacity(pattern_count), property_settings: Vec::with_capacity(pattern_count), @@ -1344,11 +1346,21 @@ impl Query { let name = slice::from_raw_parts(name, length as usize); let name = str::from_utf8_unchecked(name); result.capture_names.push(name.to_string()); - let quantifier = ffi::ts_query_capture_quantifier_for_id(ptr, i); - result.capture_quantifiers.push(quantifier.into()); } } + // Build + for i in 0..pattern_count { + let mut capture_quantifiers = Vec::with_capacity(capture_count as usize); + for j in 0..capture_count { + unsafe { + let quantifier = ffi::ts_query_capture_quantifier_for_id(ptr, i as u32, j); + capture_quantifiers.push(quantifier.into()); + } + } + result.capture_quantifiers.push(capture_quantifiers); + } + // Build a vector of strings to represent literal values used in predicates. let string_values = (0..string_count) .map(|i| unsafe { @@ -1550,8 +1562,8 @@ impl Query { } /// Get the quantifiers of the captures used in the query. - pub fn capture_quantifiers(&self) -> &[CaptureQuantifier] { - &self.capture_quantifiers + pub fn capture_quantifiers(&self, index: usize) -> &[CaptureQuantifier] { + &self.capture_quantifiers[index] } /// Get the index for a given capture name. diff --git a/lib/include/tree_sitter/api.h b/lib/include/tree_sitter/api.h index 275350c9..0f68530f 100644 --- a/lib/include/tree_sitter/api.h +++ b/lib/include/tree_sitter/api.h @@ -756,8 +756,10 @@ const char *ts_query_capture_name_for_id( */ TSQuantifier ts_query_capture_quantifier_for_id( const TSQuery *, - uint32_t id + uint32_t pattern_id, + uint32_t capture_id ); + const char *ts_query_string_value_for_id( const TSQuery *, uint32_t id, diff --git a/lib/src/query.c b/lib/src/query.c index a8d2436c..14ab91e4 100644 --- a/lib/src/query.c +++ b/lib/src/query.c @@ -267,7 +267,7 @@ typedef struct { */ struct TSQuery { SymbolTable captures; - CaptureQuantifiers capture_quantifiers; + Array(CaptureQuantifiers) capture_quantifiers; SymbolTable predicate_values; Array(QueryStep) steps; Array(PatternEntry) pattern_map; @@ -2500,7 +2500,7 @@ TSQuery *ts_query_new( .steps = array_new(), .pattern_map = array_new(), .captures = symbol_table_new(), - .capture_quantifiers = capture_quantifiers_new(), + .capture_quantifiers = array_new(), .predicate_values = symbol_table_new(), .predicate_steps = array_new(), .patterns = array_new(), @@ -2525,7 +2525,8 @@ TSQuery *ts_query_new( .predicate_steps = (Slice) {.offset = start_predicate_step_index}, .start_byte = stream_offset(&stream), })); - *error_type = ts_query__parse_pattern(self, &stream, 0, false, &self->capture_quantifiers); + CaptureQuantifiers capture_quantifiers = capture_quantifiers_new(); + *error_type = ts_query__parse_pattern(self, &stream, 0, false, &capture_quantifiers); array_push(&self->steps, query_step__new(0, PATTERN_DONE_MARKER, false)); QueryPattern *pattern = array_back(&self->patterns); @@ -2537,10 +2538,14 @@ TSQuery *ts_query_new( if (*error_type) { if (*error_type == PARENT_DONE) *error_type = TSQueryErrorSyntax; *error_offset = stream_offset(&stream); + capture_quantifiers_delete(&capture_quantifiers); ts_query_delete(self); return NULL; } + // Maintain a list of capture quantifiers for each pattern + array_push(&self->capture_quantifiers, capture_quantifiers); + // Maintain a map that can look up patterns for a given root symbol. uint16_t wildcard_root_alternative_index = NONE; for (;;) { @@ -2616,7 +2621,11 @@ void ts_query_delete(TSQuery *self) { array_delete(&self->negated_fields); symbol_table_delete(&self->captures); symbol_table_delete(&self->predicate_values); - capture_quantifiers_delete(&self->capture_quantifiers); + for (uint32_t index = 0; index < self->capture_quantifiers.size; index++) { + CaptureQuantifiers *capture_quantifiers = array_get(&self->capture_quantifiers, index); + capture_quantifiers_delete(capture_quantifiers); + } + array_delete(&self->capture_quantifiers); ts_free(self); } } @@ -2643,9 +2652,11 @@ const char *ts_query_capture_name_for_id( TSQuantifier ts_query_capture_quantifier_for_id( const TSQuery *self, - uint32_t index + uint32_t pattern_index, + uint32_t capture_index ) { - return capture_quantifier_for_id(&self->capture_quantifiers, index); + CaptureQuantifiers *capture_quantifiers = array_get(&self->capture_quantifiers, pattern_index); + return capture_quantifier_for_id(capture_quantifiers, capture_index); } const char *ts_query_string_value_for_id(