From f18c36ca62954fcdb038195f6fafcb78092bf347 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Sat, 6 Mar 2021 14:35:41 -0800 Subject: [PATCH 1/3] Fix handling of extra nodes in query analysis Fixes #967 --- cli/src/tests/query_test.rs | 43 +++++++++++++++++++++++++++++++++++++ lib/src/query.c | 12 ++++++++--- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/cli/src/tests/query_test.rs b/cli/src/tests/query_test.rs index f41cdf3f..f1139c6c 100644 --- a/cli/src/tests/query_test.rs +++ b/cli/src/tests/query_test.rs @@ -642,6 +642,49 @@ fn test_query_matches_capturing_error_nodes() { }); } +#[test] +fn test_query_matches_with_extra_children() { + allocations::record(|| { + let language = get_language("ruby"); + let query = Query::new( + language, + " + (program(comment) @top_level_comment) + (argument_list (heredoc_body) @heredoc_in_args) + ", + ) + .unwrap(); + + assert_query_matches( + language, + &query, + " + # top-level + puts( + # not-top-level + <<-IN_ARGS, bar.baz + HELLO + IN_ARGS + ) + + puts <<-NOT_IN_ARGS + NO + NOT_IN_ARGS + ", + &[ + (0, vec![("top_level_comment", "# top-level")]), + ( + 1, + vec![( + "heredoc_in_args", + "\n HELLO\n IN_ARGS", + )], + ), + ], + ); + }); +} + #[test] fn test_query_matches_with_named_wildcard() { allocations::record(|| { diff --git a/lib/src/query.c b/lib/src/query.c index 5a20603d..c3497d8d 100644 --- a/lib/src/query.c +++ b/lib/src/query.c @@ -959,6 +959,10 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { } #endif + // If no further progress can be made within the current recursion depth limit, then + // bump the depth limit by one, and continue to process the states the exceeded the + // limit. But only allow this if progress has been made since the last time the depth + // limit was increased. if (states.size == 0) { if (deeper_states.size > 0 && final_step_indices.size > prev_final_step_count) { #ifdef DEBUG_ANALYZE_QUERY @@ -1019,12 +1023,12 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { TSStateId next_parse_state; if (lookahead_iterator.action_count) { const TSParseAction *action = &lookahead_iterator.actions[lookahead_iterator.action_count - 1]; - if (action->type == TSParseActionTypeShift && !action->shift.extra) { - next_parse_state = action->shift.state; + if (action->type == TSParseActionTypeShift) { + next_parse_state = action->shift.extra ? parse_state : action->shift.state; } else { continue; } - } else if (lookahead_iterator.next_state != 0 && lookahead_iterator.next_state != parse_state) { + } else if (lookahead_iterator.next_state != 0) { next_parse_state = lookahead_iterator.next_state; } else { continue; @@ -1127,6 +1131,8 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { next_step->depth <= parent_depth + 1 ) break; } + } else if (next_parse_state == parse_state) { + continue; } for (;;) { From 5a2534ac4a1c8dd85d353d7ce8cd6d2d194a07ab Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Sun, 7 Mar 2021 08:48:03 -0800 Subject: [PATCH 2/3] Fix crash on unterminated alternation in query Fixes #968 --- cli/src/tests/query_test.rs | 12 ++++++++++++ lib/src/query.c | 1 + 2 files changed, 13 insertions(+) diff --git a/cli/src/tests/query_test.rs b/cli/src/tests/query_test.rs index f1139c6c..3bfa4b68 100644 --- a/cli/src/tests/query_test.rs +++ b/cli/src/tests/query_test.rs @@ -118,6 +118,18 @@ fn test_query_errors_on_invalid_syntax() { ] .join("\n") ); + + // tree-sitter/tree-sitter/issues/968 + assert_eq!( + Query::new(get_language("c"), r#"(parameter_list [ ")" @foo)"#) + .unwrap_err() + .message, + [ + r#"(parameter_list [ ")" @foo)"#, + r#" ^"# + ] + .join("\n") + ); }); } diff --git a/lib/src/query.c b/lib/src/query.c index c3497d8d..cc127079 100644 --- a/lib/src/query.c +++ b/lib/src/query.c @@ -1541,6 +1541,7 @@ static TSQueryError ts_query__parse_pattern( stream_advance(stream); break; } else if (e) { + if (e == PARENT_DONE) e = TSQueryErrorSyntax; array_delete(&branch_step_indices); return e; } From 6ba11d7778a7a7205a4b0a3c99677d831c261dd3 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 8 Mar 2021 09:33:26 -0800 Subject: [PATCH 3/3] Fix query analysis with top-level non-terminal extras Fixes https://github.com/tree-sitter/tree-sitter/issues/968#issuecomment-792314281 --- cli/src/tests/query_test.rs | 14 ++++++++++++++ lib/src/query.c | 6 ++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cli/src/tests/query_test.rs b/cli/src/tests/query_test.rs index 3bfa4b68..30d8e9c9 100644 --- a/cli/src/tests/query_test.rs +++ b/cli/src/tests/query_test.rs @@ -3132,6 +3132,20 @@ fn test_query_step_is_definite() { "#, results_by_substring: &[("name:", true)], }, + Row { + description: "top-level non-terminal extra nodes", + language: get_language("ruby"), + pattern: r#" + (heredoc_body + (interpolation) + (heredoc_end) @end) + "#, + results_by_substring: &[ + ("(heredoc_body", false), + ("(interpolation)", false), + ("(heredoc_end)", true), + ], + }, ]; allocations::record(|| { diff --git a/lib/src/query.c b/lib/src/query.c index cc127079..eebae855 100644 --- a/lib/src/query.c +++ b/lib/src/query.c @@ -784,8 +784,10 @@ static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { state_predecessor_map_add(&predecessor_map, next_state, state); } } - } else if (lookahead_iterator.next_state != 0 && lookahead_iterator.next_state != state) { - state_predecessor_map_add(&predecessor_map, lookahead_iterator.next_state, state); + } else if (lookahead_iterator.next_state != 0) { + if (lookahead_iterator.next_state != state) { + state_predecessor_map_add(&predecessor_map, lookahead_iterator.next_state, state); + } const TSSymbol *aliases, *aliases_end; ts_language_aliases_for_symbol( self->language,