feat: add an --apply-all-captures argument to highlight & test

cli/src/highlight.rs

@@ -348,7 +348,7 @@ pub fn ansi(
     let mut highlighter = Highlighter::new();
 
     let events = highlighter.highlight(config, source, cancellation_flag, |string| {
-        loader.highlight_config_for_injection_string(string)
+        loader.highlight_config_for_injection_string(string, config.apply_all_captures)
     })?;
 
     let mut style_stack = vec![theme.default_style().ansi];
@@ -394,7 +394,7 @@ pub fn html(
     let mut highlighter = Highlighter::new();
 
     let events = highlighter.highlight(config, source, cancellation_flag, |string| {
-        loader.highlight_config_for_injection_string(string)
+        loader.highlight_config_for_injection_string(string, config.apply_all_captures)
     })?;
 
     let mut renderer = HtmlRenderer::new();

cli/src/main.rs

@@ -78,6 +78,10 @@ fn run() -> Result<()> {
         .long("quiet")
         .short("q");
 
+    let apply_all_captures_arg = Arg::with_name("apply-all-captures")
+        .help("Apply all captures to highlights")
+        .long("apply-all-captures");
+
     let matches = App::new("tree-sitter")
         .author("Max Brunsfeld <maxbrunsfeld@gmail.com>")
         .about("Generates and tests parsers")
@@ -236,7 +240,8 @@ fn run() -> Result<()> {
                 )
                 .arg(&debug_arg)
                 .arg(&debug_build_arg)
-                .arg(&debug_graph_arg),
+                .arg(&debug_graph_arg)
+                .arg(&apply_all_captures_arg),
         )
         .subcommand(
             SubCommand::with_name("highlight")
@@ -256,7 +261,8 @@ fn run() -> Result<()> {
                 .arg(&time_arg)
                 .arg(&quiet_arg)
                 .arg(&paths_file_arg)
-                .arg(&paths_arg),
+                .arg(&paths_arg)
+                .arg(&apply_all_captures_arg),
         )
         .subcommand(
             SubCommand::with_name("build-wasm")
@@ -362,6 +368,7 @@ fn run() -> Result<()> {
             let debug_build = matches.is_present("debug-build");
             let update = matches.is_present("update");
             let filter = matches.value_of("filter");
+            let apply_all_captures = matches.is_present("apply-all-captures");
 
             if debug {
                 // For augmenting debug logging in external scanners
@@ -398,7 +405,7 @@ fn run() -> Result<()> {
             // Run the syntax highlighting tests.
             let test_highlight_dir = test_dir.join("highlight");
             if test_highlight_dir.is_dir() {
-                test_highlight::test_highlights(&loader, &test_highlight_dir)?;
+                test_highlight::test_highlights(&loader, &test_highlight_dir, apply_all_captures)?;
             }
 
             let test_tag_dir = test_dir.join("tags");
@@ -562,6 +569,7 @@ fn run() -> Result<()> {
             let html_mode = quiet || matches.is_present("html");
             let should_check = matches.is_present("check");
             let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?;
+            let apply_all_captures = matches.is_present("apply-all-captures");
 
             if html_mode && !quiet {
                 println!("{}", highlight::HTML_HEADER);
@@ -590,7 +598,9 @@ fn run() -> Result<()> {
                     },
                 };
 
-                if let Some(highlight_config) = language_config.highlight_config(language)? {
+                if let Some(highlight_config) =
+                    language_config.highlight_config(language, apply_all_captures)?
+                {
                     if should_check {
                         let names = highlight_config.nonconformant_capture_names();
                         if names.is_empty() {

cli/src/test_highlight.rs

@@ -38,12 +38,17 @@ impl std::fmt::Display for Failure {
     }
 }
 
-pub fn test_highlights(loader: &Loader, directory: &Path) -> Result<()> {
+pub fn test_highlights(loader: &Loader, directory: &Path, apply_all_captures: bool) -> Result<()> {
     println!("syntax highlighting:");
-    test_highlights_indented(loader, directory, 2)
+    test_highlights_indented(loader, directory, apply_all_captures, 2)
 }
 
-fn test_highlights_indented(loader: &Loader, directory: &Path, indent_level: usize) -> Result<()> {
+fn test_highlights_indented(
+    loader: &Loader,
+    directory: &Path,
+    apply_all_captures: bool,
+    indent_level: usize,
+) -> Result<()> {
     let mut failed = false;
     let mut highlighter = Highlighter::new();
 
@@ -58,7 +63,12 @@ fn test_highlights_indented(loader: &Loader, directory: &Path, indent_level: usi
         );
         if test_file_path.is_dir() && !test_file_path.read_dir()?.next().is_none() {
             println!("{}:", test_file_name.into_string().unwrap());
-            if let Err(_) = test_highlights_indented(loader, &test_file_path, indent_level + 1) {
+            if let Err(_) = test_highlights_indented(
+                loader,
+                &test_file_path,
+                apply_all_captures,
+                indent_level + 1,
+            ) {
                 failed = true;
             }
         } else {
@@ -66,7 +76,7 @@ fn test_highlights_indented(loader: &Loader, directory: &Path, indent_level: usi
                 .language_configuration_for_file_name(&test_file_path)?
                 .ok_or_else(|| anyhow!("No language found for path {:?}", test_file_path))?;
             let highlight_config = language_config
-                .highlight_config(language)?
+                .highlight_config(language, apply_all_captures)?
                 .ok_or_else(|| anyhow!("No highlighting config found for {:?}", test_file_path))?;
             match test_highlight(
                 &loader,
@@ -111,7 +121,7 @@ pub fn iterate_assertions(
     // Iterate through all of the highlighting assertions, checking each one against the
     // actual highlights.
     let mut i = 0;
-    let mut actual_highlights = Vec::<&String>::new();
+    let mut actual_highlights = Vec::new();
     for Assertion {
         position,
         negative,
@@ -202,7 +212,7 @@ pub fn get_highlight_positions(
     let source = String::from_utf8_lossy(source);
     let mut char_indices = source.char_indices();
     for event in highlighter.highlight(highlight_config, source.as_bytes(), None, |string| {
-        loader.highlight_config_for_injection_string(string)
+        loader.highlight_config_for_injection_string(string, highlight_config.apply_all_captures)
     })? {
         match event? {
             HighlightEvent::HighlightStart(h) => highlight_stack.push(h),

cli/src/tests/helpers/fixtures.rs

@@ -55,6 +55,7 @@ pub fn get_highlight_config(
         &highlights_query,
         &injections_query,
         &locals_query,
+        false,
     )
     .unwrap();
     result.configure(&highlight_names);

cli/src/tests/highlight_test.rs

@@ -522,6 +522,7 @@ fn test_highlighting_via_c_api() {
         highlights_query.len() as u32,
         injections_query.len() as u32,
         locals_query.len() as u32,
+        false,
     );
 
     let html_scope = c_string("text.html.basic");
@@ -541,6 +542,7 @@ fn test_highlighting_via_c_api() {
         highlights_query.len() as u32,
         injections_query.len() as u32,
         0,
+        false,
     );
 
     let buffer = c::ts_highlight_buffer_new();
@@ -587,6 +589,65 @@ fn test_highlighting_via_c_api() {
     c::ts_highlight_buffer_delete(buffer);
 }
 
+#[test]
+fn test_highlighting_with_all_captures_applied() {
+    let source = "fn main(a: u32, b: u32) -> { let c = a + b; }";
+    let language = get_language("rust");
+    let highlights_query = indoc::indoc! {"
+        [
+          \"fn\"
+          \"let\"
+        ] @keyword
+        (identifier) @variable
+        (function_item name: (identifier) @function)
+        (parameter pattern: (identifier) @variable.parameter)
+        (primitive_type) @type.builtin
+        \"=\" @operator
+        [ \"->\" \":\" \";\" ] @punctuation.delimiter
+        [ \"{\" \"}\" \"(\" \")\" ] @punctuation.bracket
+    "};
+    let mut rust_highlight_reverse =
+        HighlightConfiguration::new(language, &highlights_query, "", "", true).unwrap();
+    rust_highlight_reverse.configure(&HIGHLIGHT_NAMES);
+
+    assert_eq!(
+        &to_token_vector(&source, &rust_highlight_reverse).unwrap(),
+        &[[
+            ("fn", vec!["keyword"]),
+            (" ", vec![]),
+            ("main", vec!["function"]),
+            ("(", vec!["punctuation.bracket"]),
+            ("a", vec!["variable.parameter"]),
+            (":", vec!["punctuation.delimiter"]),
+            (" ", vec![]),
+            ("u32", vec!["type.builtin"]),
+            (", ", vec![]),
+            ("b", vec!["variable.parameter"]),
+            (":", vec!["punctuation.delimiter"]),
+            (" ", vec![]),
+            ("u32", vec!["type.builtin"]),
+            (")", vec!["punctuation.bracket"]),
+            (" ", vec![]),
+            ("->", vec!["punctuation.delimiter"]),
+            (" ", vec![]),
+            ("{", vec!["punctuation.bracket"]),
+            (" ", vec![]),
+            ("let", vec!["keyword"]),
+            (" ", vec![]),
+            ("c", vec!["variable"]),
+            (" ", vec![]),
+            ("=", vec!["operator"]),
+            (" ", vec![]),
+            ("a", vec!["variable"]),
+            (" + ", vec![]),
+            ("b", vec!["variable"]),
+            (";", vec!["punctuation.delimiter"]),
+            (" ", vec![]),
+            ("}", vec!["punctuation.bracket"])
+        ]],
+    );
+}
+
 #[test]
 fn test_decode_utf8_lossy() {
     use tree_sitter::LossyUtf8;