From 83ef0aea12c68372265ce388e1098e7dae36bb69 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Sun, 16 Jan 2022 20:05:58 -0600 Subject: [PATCH 1/5] prevent future matches for '#is-not? local' patterns --- highlight/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/highlight/src/lib.rs b/highlight/src/lib.rs index 58d7e88c..e0016951 100644 --- a/highlight/src/lib.rs +++ b/highlight/src/lib.rs @@ -111,6 +111,7 @@ struct HighlightIterLayer<'a> { scope_stack: Vec>, ranges: Vec, depth: usize, + ignore_match_ids: Vec, } impl Highlighter { @@ -413,6 +414,7 @@ impl<'a> HighlightIterLayer<'a> { captures, config, ranges, + ignore_match_ids: Vec::new(), }); } @@ -683,6 +685,10 @@ where let (mut match_, capture_index) = layer.captures.next().unwrap(); let mut capture = match_.captures[capture_index]; + if layer.ignore_match_ids.contains(&match_.id()) { + continue 'main; + } + // If this capture represents an injection, then process the injection. if match_.pattern_index < layer.config.locals_pattern_index { let (language_name, content_node, include_children) = @@ -835,6 +841,7 @@ where // highlighting patterns that are disabled for local variables. if definition_highlight.is_some() || reference_highlight.is_some() { while layer.config.non_local_variable_patterns[match_.pattern_index] { + layer.ignore_match_ids.push(match_.id()); if let Some((next_match, next_capture_index)) = layer.captures.peek() { let next_capture = next_match.captures[*next_capture_index]; if next_capture.node == capture.node { From 51354ef776064ab34e9d469f4a1d954e0b65a02a Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Mon, 17 Jan 2022 22:20:05 -0600 Subject: [PATCH 2/5] use just an i32 to ignore match IDs --- highlight/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/highlight/src/lib.rs b/highlight/src/lib.rs index e0016951..d37b13e9 100644 --- a/highlight/src/lib.rs +++ b/highlight/src/lib.rs @@ -111,7 +111,7 @@ struct HighlightIterLayer<'a> { scope_stack: Vec>, ranges: Vec, depth: usize, - ignore_match_ids: Vec, + ignore_match_id: i32, } impl Highlighter { @@ -414,7 +414,7 @@ impl<'a> HighlightIterLayer<'a> { captures, config, ranges, - ignore_match_ids: Vec::new(), + ignore_match_id: -1, }); } @@ -685,7 +685,7 @@ where let (mut match_, capture_index) = layer.captures.next().unwrap(); let mut capture = match_.captures[capture_index]; - if layer.ignore_match_ids.contains(&match_.id()) { + if layer.ignore_match_id >= match_.id() as i32 { continue 'main; } @@ -841,7 +841,7 @@ where // highlighting patterns that are disabled for local variables. if definition_highlight.is_some() || reference_highlight.is_some() { while layer.config.non_local_variable_patterns[match_.pattern_index] { - layer.ignore_match_ids.push(match_.id()); + layer.ignore_match_id = match_.id() as i32; if let Some((next_match, next_capture_index)) = layer.captures.peek() { let next_capture = next_match.captures[*next_capture_index]; if next_capture.node == capture.node { From 716ef245780a0ca671bdd91a9d7c1032f5dae042 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 18 Jan 2022 17:01:07 -0600 Subject: [PATCH 3/5] remove unfinished queries from 'ts_query_cursor_remove_match' --- lib/src/query.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/src/query.c b/lib/src/query.c index 470bcfd2..23fddceb 100644 --- a/lib/src/query.c +++ b/lib/src/query.c @@ -3725,6 +3725,20 @@ void ts_query_cursor_remove_match( return; } } + + // Remove unfinished query states as well to prevent future + // captures for a match being removed. + for (unsigned i = 0; i < self->states.size; i++) { + const QueryState *state = &self->states.contents[i]; + if (state->id == match_id) { + capture_list_pool_release( + &self->capture_list_pool, + state->capture_list_id + ); + array_erase(&self->states, i); + return; + } + } } bool ts_query_cursor_next_capture( From a3609aa07e2c728e8b2ed61528270266202eed6b Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 18 Jan 2022 17:04:00 -0600 Subject: [PATCH 4/5] remove non-local query matches for locals --- highlight/src/lib.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/highlight/src/lib.rs b/highlight/src/lib.rs index d37b13e9..0d097201 100644 --- a/highlight/src/lib.rs +++ b/highlight/src/lib.rs @@ -111,7 +111,6 @@ struct HighlightIterLayer<'a> { scope_stack: Vec>, ranges: Vec, depth: usize, - ignore_match_id: i32, } impl Highlighter { @@ -414,7 +413,6 @@ impl<'a> HighlightIterLayer<'a> { captures, config, ranges, - ignore_match_id: -1, }); } @@ -685,10 +683,6 @@ where let (mut match_, capture_index) = layer.captures.next().unwrap(); let mut capture = match_.captures[capture_index]; - if layer.ignore_match_id >= match_.id() as i32 { - continue 'main; - } - // If this capture represents an injection, then process the injection. if match_.pattern_index < layer.config.locals_pattern_index { let (language_name, content_node, include_children) = @@ -841,7 +835,7 @@ where // highlighting patterns that are disabled for local variables. if definition_highlight.is_some() || reference_highlight.is_some() { while layer.config.non_local_variable_patterns[match_.pattern_index] { - layer.ignore_match_id = match_.id() as i32; + match_.remove(); if let Some((next_match, next_capture_index)) = layer.captures.peek() { let next_capture = next_match.captures[*next_capture_index]; if next_capture.node == capture.node { From 02abc2a0634b75cee43f6e89e143338be86f716d Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 18 Jan 2022 20:54:55 -0600 Subject: [PATCH 5/5] add test for removals in eager query matches --- cli/src/tests/query_test.rs | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/cli/src/tests/query_test.rs b/cli/src/tests/query_test.rs index 6634c62f..215a8ee1 100644 --- a/cli/src/tests/query_test.rs +++ b/cli/src/tests/query_test.rs @@ -3091,6 +3091,53 @@ fn test_query_captures_with_matches_removed() { }); } +#[test] +fn test_query_captures_with_matches_removed_before_they_finish() { + allocations::record(|| { + let language = get_language("javascript"); + // When Tree-sitter detects that a pattern is guaranteed to match, + // it will start to eagerly return the captures that it has found, + // even though it hasn't matched the entire pattern yet. A + // namespace_import node always has "*", "as" and then an identifier + // for children, so captures will be emitted eagerly for this pattern. + let query = Query::new( + language, + r#" + (namespace_import + "*" @star + "as" @as + (identifier) @identifier) + "#, + ) + .unwrap(); + + let source = " + import * as name from 'module-name'; + "; + + let mut parser = Parser::new(); + parser.set_language(language).unwrap(); + let tree = parser.parse(&source, None).unwrap(); + let mut cursor = QueryCursor::new(); + + let mut captured_strings = Vec::new(); + for (m, i) in cursor.captures(&query, tree.root_node(), source.as_bytes()) { + let capture = m.captures[i]; + let text = capture.node.utf8_text(source.as_bytes()).unwrap(); + if text == "as" { + m.remove(); + continue; + } + captured_strings.push(text); + } + + // .remove() removes the match before it is finished. The identifier + // "name" is part of this match, so we expect that removing the "as" + // capture from the match should prevent "name" from matching: + assert_eq!(captured_strings, &["*",]); + }); +} + #[test] fn test_query_captures_and_matches_iterators_are_fused() { allocations::record(|| {