feat: add an --apply-all-captures argument to highlight & test

cli/loader/src/lib.rs

@@ -484,6 +484,7 @@ impl Loader {
     pub fn highlight_config_for_injection_string<'a>(
         &'a self,
         string: &str,
+        apply_all_captures: bool,
     ) -> Option<&'a HighlightConfiguration> {
         match self.language_configuration_for_injection_string(string) {
             Err(e) => {
@@ -494,17 +495,19 @@ impl Loader {
                 None
             }
             Ok(None) => None,
-            Ok(Some((language, configuration))) => match configuration.highlight_config(language) {
-                Err(e) => {
-                    eprintln!(
-                        "Failed to load property sheet for injection string '{}': {}",
-                        string, e
-                    );
-                    None
+            Ok(Some((language, configuration))) => {
+                match configuration.highlight_config(language, apply_all_captures) {
+                    Err(e) => {
+                        eprintln!(
+                            "Failed to load property sheet for injection string '{}': {}",
+                            string, e
+                        );
+                        None
+                    }
+                    Ok(None) => None,
+                    Ok(Some(config)) => Some(config),
                 }
-                Ok(None) => None,
-                Ok(Some(config)) => Some(config),
-            },
+            }
         }
     }
 
@@ -701,7 +704,11 @@ impl Loader {
 }
 
 impl<'a> LanguageConfiguration<'a> {
-    pub fn highlight_config(&self, language: Language) -> Result<Option<&HighlightConfiguration>> {
+    pub fn highlight_config(
+        &self,
+        language: Language,
+        apply_all_captures: bool,
+    ) -> Result<Option<&HighlightConfiguration>> {
         return self
             .highlight_config
             .get_or_try_init(|| {
@@ -720,6 +727,7 @@ impl<'a> LanguageConfiguration<'a> {
                         &highlights_query,
                         &injections_query,
                         &locals_query,
+                        apply_all_captures,
                     )
                     .map_err(|error| match error.kind {
                         QueryErrorKind::Language => Error::from(error),

cli/src/highlight.rs

@@ -348,7 +348,7 @@ pub fn ansi(
     let mut highlighter = Highlighter::new();
 
     let events = highlighter.highlight(config, source, cancellation_flag, |string| {
-        loader.highlight_config_for_injection_string(string)
+        loader.highlight_config_for_injection_string(string, config.apply_all_captures)
     })?;
 
     let mut style_stack = vec![theme.default_style().ansi];
@@ -394,7 +394,7 @@ pub fn html(
     let mut highlighter = Highlighter::new();
 
     let events = highlighter.highlight(config, source, cancellation_flag, |string| {
-        loader.highlight_config_for_injection_string(string)
+        loader.highlight_config_for_injection_string(string, config.apply_all_captures)
     })?;
 
     let mut renderer = HtmlRenderer::new();

cli/src/main.rs

@@ -78,6 +78,10 @@ fn run() -> Result<()> {
         .long("quiet")
         .short("q");
 
+    let apply_all_captures_arg = Arg::with_name("apply-all-captures")
+        .help("Apply all captures to highlights")
+        .long("apply-all-captures");
+
     let matches = App::new("tree-sitter")
         .author("Max Brunsfeld <maxbrunsfeld@gmail.com>")
         .about("Generates and tests parsers")
@@ -236,7 +240,8 @@ fn run() -> Result<()> {
                 )
                 .arg(&debug_arg)
                 .arg(&debug_build_arg)
-                .arg(&debug_graph_arg),
+                .arg(&debug_graph_arg)
+                .arg(&apply_all_captures_arg),
         )
         .subcommand(
             SubCommand::with_name("highlight")
@@ -256,7 +261,8 @@ fn run() -> Result<()> {
                 .arg(&time_arg)
                 .arg(&quiet_arg)
                 .arg(&paths_file_arg)
-                .arg(&paths_arg),
+                .arg(&paths_arg)
+                .arg(&apply_all_captures_arg),
         )
         .subcommand(
             SubCommand::with_name("build-wasm")
@@ -362,6 +368,7 @@ fn run() -> Result<()> {
             let debug_build = matches.is_present("debug-build");
             let update = matches.is_present("update");
             let filter = matches.value_of("filter");
+            let apply_all_captures = matches.is_present("apply-all-captures");
 
             if debug {
                 // For augmenting debug logging in external scanners
@@ -398,7 +405,7 @@ fn run() -> Result<()> {
             // Run the syntax highlighting tests.
             let test_highlight_dir = test_dir.join("highlight");
             if test_highlight_dir.is_dir() {
-                test_highlight::test_highlights(&loader, &test_highlight_dir)?;
+                test_highlight::test_highlights(&loader, &test_highlight_dir, apply_all_captures)?;
             }
 
             let test_tag_dir = test_dir.join("tags");
@@ -562,6 +569,7 @@ fn run() -> Result<()> {
             let html_mode = quiet || matches.is_present("html");
             let should_check = matches.is_present("check");
             let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?;
+            let apply_all_captures = matches.is_present("apply-all-captures");
 
             if html_mode && !quiet {
                 println!("{}", highlight::HTML_HEADER);
@@ -590,7 +598,9 @@ fn run() -> Result<()> {
                     },
                 };
 
-                if let Some(highlight_config) = language_config.highlight_config(language)? {
+                if let Some(highlight_config) =
+                    language_config.highlight_config(language, apply_all_captures)?
+                {
                     if should_check {
                         let names = highlight_config.nonconformant_capture_names();
                         if names.is_empty() {

cli/src/test_highlight.rs

@@ -38,12 +38,17 @@ impl std::fmt::Display for Failure {
     }
 }
 
-pub fn test_highlights(loader: &Loader, directory: &Path) -> Result<()> {
+pub fn test_highlights(loader: &Loader, directory: &Path, apply_all_captures: bool) -> Result<()> {
     println!("syntax highlighting:");
-    test_highlights_indented(loader, directory, 2)
+    test_highlights_indented(loader, directory, apply_all_captures, 2)
 }
 
-fn test_highlights_indented(loader: &Loader, directory: &Path, indent_level: usize) -> Result<()> {
+fn test_highlights_indented(
+    loader: &Loader,
+    directory: &Path,
+    apply_all_captures: bool,
+    indent_level: usize,
+) -> Result<()> {
     let mut failed = false;
     let mut highlighter = Highlighter::new();
 
@@ -58,7 +63,12 @@ fn test_highlights_indented(loader: &Loader, directory: &Path, indent_level: usi
         );
         if test_file_path.is_dir() && !test_file_path.read_dir()?.next().is_none() {
             println!("{}:", test_file_name.into_string().unwrap());
-            if let Err(_) = test_highlights_indented(loader, &test_file_path, indent_level + 1) {
+            if let Err(_) = test_highlights_indented(
+                loader,
+                &test_file_path,
+                apply_all_captures,
+                indent_level + 1,
+            ) {
                 failed = true;
             }
         } else {
@@ -66,7 +76,7 @@ fn test_highlights_indented(loader: &Loader, directory: &Path, indent_level: usi
                 .language_configuration_for_file_name(&test_file_path)?
                 .ok_or_else(|| anyhow!("No language found for path {:?}", test_file_path))?;
             let highlight_config = language_config
-                .highlight_config(language)?
+                .highlight_config(language, apply_all_captures)?
                 .ok_or_else(|| anyhow!("No highlighting config found for {:?}", test_file_path))?;
             match test_highlight(
                 &loader,
@@ -111,7 +121,7 @@ pub fn iterate_assertions(
     // Iterate through all of the highlighting assertions, checking each one against the
     // actual highlights.
     let mut i = 0;
-    let mut actual_highlights = Vec::<&String>::new();
+    let mut actual_highlights = Vec::new();
     for Assertion {
         position,
         negative,
@@ -202,7 +212,7 @@ pub fn get_highlight_positions(
     let source = String::from_utf8_lossy(source);
     let mut char_indices = source.char_indices();
     for event in highlighter.highlight(highlight_config, source.as_bytes(), None, |string| {
-        loader.highlight_config_for_injection_string(string)
+        loader.highlight_config_for_injection_string(string, highlight_config.apply_all_captures)
     })? {
         match event? {
             HighlightEvent::HighlightStart(h) => highlight_stack.push(h),

cli/src/tests/helpers/fixtures.rs

@@ -55,6 +55,7 @@ pub fn get_highlight_config(
         &highlights_query,
         &injections_query,
         &locals_query,
+        false,
     )
     .unwrap();
     result.configure(&highlight_names);

cli/src/tests/highlight_test.rs

@@ -522,6 +522,7 @@ fn test_highlighting_via_c_api() {
         highlights_query.len() as u32,
         injections_query.len() as u32,
         locals_query.len() as u32,
+        false,
     );
 
     let html_scope = c_string("text.html.basic");
@@ -541,6 +542,7 @@ fn test_highlighting_via_c_api() {
         highlights_query.len() as u32,
         injections_query.len() as u32,
         0,
+        false,
     );
 
     let buffer = c::ts_highlight_buffer_new();
@@ -587,6 +589,65 @@ fn test_highlighting_via_c_api() {
     c::ts_highlight_buffer_delete(buffer);
 }
 
+#[test]
+fn test_highlighting_with_all_captures_applied() {
+    let source = "fn main(a: u32, b: u32) -> { let c = a + b; }";
+    let language = get_language("rust");
+    let highlights_query = indoc::indoc! {"
+        [
+          \"fn\"
+          \"let\"
+        ] @keyword
+        (identifier) @variable
+        (function_item name: (identifier) @function)
+        (parameter pattern: (identifier) @variable.parameter)
+        (primitive_type) @type.builtin
+        \"=\" @operator
+        [ \"->\" \":\" \";\" ] @punctuation.delimiter
+        [ \"{\" \"}\" \"(\" \")\" ] @punctuation.bracket
+    "};
+    let mut rust_highlight_reverse =
+        HighlightConfiguration::new(language, &highlights_query, "", "", true).unwrap();
+    rust_highlight_reverse.configure(&HIGHLIGHT_NAMES);
+
+    assert_eq!(
+        &to_token_vector(&source, &rust_highlight_reverse).unwrap(),
+        &[[
+            ("fn", vec!["keyword"]),
+            (" ", vec![]),
+            ("main", vec!["function"]),
+            ("(", vec!["punctuation.bracket"]),
+            ("a", vec!["variable.parameter"]),
+            (":", vec!["punctuation.delimiter"]),
+            (" ", vec![]),
+            ("u32", vec!["type.builtin"]),
+            (", ", vec![]),
+            ("b", vec!["variable.parameter"]),
+            (":", vec!["punctuation.delimiter"]),
+            (" ", vec![]),
+            ("u32", vec!["type.builtin"]),
+            (")", vec!["punctuation.bracket"]),
+            (" ", vec![]),
+            ("->", vec!["punctuation.delimiter"]),
+            (" ", vec![]),
+            ("{", vec!["punctuation.bracket"]),
+            (" ", vec![]),
+            ("let", vec!["keyword"]),
+            (" ", vec![]),
+            ("c", vec!["variable"]),
+            (" ", vec![]),
+            ("=", vec!["operator"]),
+            (" ", vec![]),
+            ("a", vec!["variable"]),
+            (" + ", vec![]),
+            ("b", vec!["variable"]),
+            (";", vec!["punctuation.delimiter"]),
+            (" ", vec![]),
+            ("}", vec!["punctuation.bracket"])
+        ]],
+    );
+}
+
 #[test]
 fn test_decode_utf8_lossy() {
     use tree_sitter::LossyUtf8;

highlight/include/tree_sitter/highlight.h

@@ -48,7 +48,8 @@ TSHighlightError ts_highlighter_add_language(
   const char *locals_query,
   uint32_t highlight_query_len,
   uint32_t injection_query_len,
-  uint32_t locals_query_len
+  uint32_t locals_query_len,
+  bool apply_all_captures
 );
 
 // Compute syntax highlighting for a given document. You must first

highlight/src/c_lib.rs

@@ -70,6 +70,7 @@ pub extern "C" fn ts_highlighter_add_language(
     highlight_query_len: u32,
     injection_query_len: u32,
     locals_query_len: u32,
+    apply_all_captures: bool,
 ) -> ErrorCode {
     let f = move || {
         let this = unwrap_mut_ptr(this);
@@ -109,9 +110,14 @@ pub extern "C" fn ts_highlighter_add_language(
             ""
         };
 
-        let mut config =
-            HighlightConfiguration::new(language, highlight_query, injection_query, locals_query)
-                .or(Err(ErrorCode::InvalidQuery))?;
+        let mut config = HighlightConfiguration::new(
+            language,
+            highlight_query,
+            injection_query,
+            locals_query,
+            apply_all_captures,
+        )
+        .or(Err(ErrorCode::InvalidQuery))?;
         config.configure(&this.highlight_names.as_slice());
         this.languages.insert(scope_name, (injection_regex, config));
 

highlight/src/lib.rs

@@ -103,6 +103,7 @@ pub enum HighlightEvent {
 pub struct HighlightConfiguration {
     pub language: Language,
     pub query: Query,
+    pub apply_all_captures: bool,
     combined_injections_query: Option<Query>,
     locals_pattern_index: usize,
     highlights_pattern_index: usize,
@@ -160,6 +161,7 @@ where
     iter_count: usize,
     next_event: Option<HighlightEvent>,
     last_highlight_range: Option<(usize, usize, usize)>,
+    apply_all_captures: bool,
 }
 
 struct HighlightIterLayer<'a> {
@@ -215,9 +217,10 @@ impl Highlighter {
             cancellation_flag,
             highlighter: self,
             iter_count: 0,
-            layers: layers,
+            layers,
             next_event: None,
             last_highlight_range: None,
+            apply_all_captures: config.apply_all_captures,
         };
         result.sort_layers();
         Ok(result)
@@ -244,6 +247,7 @@ impl HighlightConfiguration {
         highlights_query: &str,
         injection_query: &str,
         locals_query: &str,
+        apply_all_captures: bool,
     ) -> Result<Self, QueryError> {
         // Concatenate the query strings, keeping track of the start offset of each section.
         let mut query_source = String::new();
@@ -324,6 +328,7 @@ impl HighlightConfiguration {
         Ok(HighlightConfiguration {
             language,
             query,
+            apply_all_captures,
             combined_injections_query,
             locals_pattern_index,
             highlights_pattern_index,
@@ -929,7 +934,13 @@ where
             while let Some((next_match, next_capture_index)) = layer.captures.peek() {
                 let next_capture = next_match.captures[*next_capture_index];
                 if next_capture.node == capture.node {
-                    layer.captures.next();
+                    if self.apply_all_captures {
+                        match_.remove();
+                        capture = next_capture;
+                        match_ = layer.captures.next().unwrap().0;
+                    } else {
+                        layer.captures.next();
+                    }
                 } else {
                     break;
                 }