diff --git a/cli/loader/src/lib.rs b/cli/loader/src/lib.rs index c0f050e0..e2f4b703 100644 --- a/cli/loader/src/lib.rs +++ b/cli/loader/src/lib.rs @@ -484,6 +484,7 @@ impl Loader { pub fn highlight_config_for_injection_string<'a>( &'a self, string: &str, + apply_all_captures: bool, ) -> Option<&'a HighlightConfiguration> { match self.language_configuration_for_injection_string(string) { Err(e) => { @@ -494,17 +495,19 @@ impl Loader { None } Ok(None) => None, - Ok(Some((language, configuration))) => match configuration.highlight_config(language) { - Err(e) => { - eprintln!( - "Failed to load property sheet for injection string '{}': {}", - string, e - ); - None + Ok(Some((language, configuration))) => { + match configuration.highlight_config(language, apply_all_captures) { + Err(e) => { + eprintln!( + "Failed to load property sheet for injection string '{}': {}", + string, e + ); + None + } + Ok(None) => None, + Ok(Some(config)) => Some(config), } - Ok(None) => None, - Ok(Some(config)) => Some(config), - }, + } } } @@ -701,7 +704,11 @@ impl Loader { } impl<'a> LanguageConfiguration<'a> { - pub fn highlight_config(&self, language: Language) -> Result> { + pub fn highlight_config( + &self, + language: Language, + apply_all_captures: bool, + ) -> Result> { return self .highlight_config .get_or_try_init(|| { @@ -720,6 +727,7 @@ impl<'a> LanguageConfiguration<'a> { &highlights_query, &injections_query, &locals_query, + apply_all_captures, ) .map_err(|error| match error.kind { QueryErrorKind::Language => Error::from(error), diff --git a/cli/src/highlight.rs b/cli/src/highlight.rs index ba3fcd70..a7a98936 100644 --- a/cli/src/highlight.rs +++ b/cli/src/highlight.rs @@ -348,7 +348,7 @@ pub fn ansi( let mut highlighter = Highlighter::new(); let events = highlighter.highlight(config, source, cancellation_flag, |string| { - loader.highlight_config_for_injection_string(string) + loader.highlight_config_for_injection_string(string, config.apply_all_captures) })?; let mut style_stack = vec![theme.default_style().ansi]; @@ -394,7 +394,7 @@ pub fn html( let mut highlighter = Highlighter::new(); let events = highlighter.highlight(config, source, cancellation_flag, |string| { - loader.highlight_config_for_injection_string(string) + loader.highlight_config_for_injection_string(string, config.apply_all_captures) })?; let mut renderer = HtmlRenderer::new(); diff --git a/cli/src/main.rs b/cli/src/main.rs index f66864ff..cbcbb181 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -78,6 +78,10 @@ fn run() -> Result<()> { .long("quiet") .short("q"); + let apply_all_captures_arg = Arg::with_name("apply-all-captures") + .help("Apply all captures to highlights") + .long("apply-all-captures"); + let matches = App::new("tree-sitter") .author("Max Brunsfeld ") .about("Generates and tests parsers") @@ -236,7 +240,8 @@ fn run() -> Result<()> { ) .arg(&debug_arg) .arg(&debug_build_arg) - .arg(&debug_graph_arg), + .arg(&debug_graph_arg) + .arg(&apply_all_captures_arg), ) .subcommand( SubCommand::with_name("highlight") @@ -256,7 +261,8 @@ fn run() -> Result<()> { .arg(&time_arg) .arg(&quiet_arg) .arg(&paths_file_arg) - .arg(&paths_arg), + .arg(&paths_arg) + .arg(&apply_all_captures_arg), ) .subcommand( SubCommand::with_name("build-wasm") @@ -362,6 +368,7 @@ fn run() -> Result<()> { let debug_build = matches.is_present("debug-build"); let update = matches.is_present("update"); let filter = matches.value_of("filter"); + let apply_all_captures = matches.is_present("apply-all-captures"); if debug { // For augmenting debug logging in external scanners @@ -398,7 +405,7 @@ fn run() -> Result<()> { // Run the syntax highlighting tests. let test_highlight_dir = test_dir.join("highlight"); if test_highlight_dir.is_dir() { - test_highlight::test_highlights(&loader, &test_highlight_dir)?; + test_highlight::test_highlights(&loader, &test_highlight_dir, apply_all_captures)?; } let test_tag_dir = test_dir.join("tags"); @@ -562,6 +569,7 @@ fn run() -> Result<()> { let html_mode = quiet || matches.is_present("html"); let should_check = matches.is_present("check"); let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; + let apply_all_captures = matches.is_present("apply-all-captures"); if html_mode && !quiet { println!("{}", highlight::HTML_HEADER); @@ -590,7 +598,9 @@ fn run() -> Result<()> { }, }; - if let Some(highlight_config) = language_config.highlight_config(language)? { + if let Some(highlight_config) = + language_config.highlight_config(language, apply_all_captures)? + { if should_check { let names = highlight_config.nonconformant_capture_names(); if names.is_empty() { diff --git a/cli/src/test_highlight.rs b/cli/src/test_highlight.rs index a47f4560..78a93be6 100644 --- a/cli/src/test_highlight.rs +++ b/cli/src/test_highlight.rs @@ -38,12 +38,17 @@ impl std::fmt::Display for Failure { } } -pub fn test_highlights(loader: &Loader, directory: &Path) -> Result<()> { +pub fn test_highlights(loader: &Loader, directory: &Path, apply_all_captures: bool) -> Result<()> { println!("syntax highlighting:"); - test_highlights_indented(loader, directory, 2) + test_highlights_indented(loader, directory, apply_all_captures, 2) } -fn test_highlights_indented(loader: &Loader, directory: &Path, indent_level: usize) -> Result<()> { +fn test_highlights_indented( + loader: &Loader, + directory: &Path, + apply_all_captures: bool, + indent_level: usize, +) -> Result<()> { let mut failed = false; let mut highlighter = Highlighter::new(); @@ -58,7 +63,12 @@ fn test_highlights_indented(loader: &Loader, directory: &Path, indent_level: usi ); if test_file_path.is_dir() && !test_file_path.read_dir()?.next().is_none() { println!("{}:", test_file_name.into_string().unwrap()); - if let Err(_) = test_highlights_indented(loader, &test_file_path, indent_level + 1) { + if let Err(_) = test_highlights_indented( + loader, + &test_file_path, + apply_all_captures, + indent_level + 1, + ) { failed = true; } } else { @@ -66,7 +76,7 @@ fn test_highlights_indented(loader: &Loader, directory: &Path, indent_level: usi .language_configuration_for_file_name(&test_file_path)? .ok_or_else(|| anyhow!("No language found for path {:?}", test_file_path))?; let highlight_config = language_config - .highlight_config(language)? + .highlight_config(language, apply_all_captures)? .ok_or_else(|| anyhow!("No highlighting config found for {:?}", test_file_path))?; match test_highlight( &loader, @@ -111,7 +121,7 @@ pub fn iterate_assertions( // Iterate through all of the highlighting assertions, checking each one against the // actual highlights. let mut i = 0; - let mut actual_highlights = Vec::<&String>::new(); + let mut actual_highlights = Vec::new(); for Assertion { position, negative, @@ -202,7 +212,7 @@ pub fn get_highlight_positions( let source = String::from_utf8_lossy(source); let mut char_indices = source.char_indices(); for event in highlighter.highlight(highlight_config, source.as_bytes(), None, |string| { - loader.highlight_config_for_injection_string(string) + loader.highlight_config_for_injection_string(string, highlight_config.apply_all_captures) })? { match event? { HighlightEvent::HighlightStart(h) => highlight_stack.push(h), diff --git a/cli/src/tests/helpers/fixtures.rs b/cli/src/tests/helpers/fixtures.rs index f790ed75..7232e84a 100644 --- a/cli/src/tests/helpers/fixtures.rs +++ b/cli/src/tests/helpers/fixtures.rs @@ -55,6 +55,7 @@ pub fn get_highlight_config( &highlights_query, &injections_query, &locals_query, + false, ) .unwrap(); result.configure(&highlight_names); diff --git a/cli/src/tests/highlight_test.rs b/cli/src/tests/highlight_test.rs index e0b356d2..8a5f22d3 100644 --- a/cli/src/tests/highlight_test.rs +++ b/cli/src/tests/highlight_test.rs @@ -522,6 +522,7 @@ fn test_highlighting_via_c_api() { highlights_query.len() as u32, injections_query.len() as u32, locals_query.len() as u32, + false, ); let html_scope = c_string("text.html.basic"); @@ -541,6 +542,7 @@ fn test_highlighting_via_c_api() { highlights_query.len() as u32, injections_query.len() as u32, 0, + false, ); let buffer = c::ts_highlight_buffer_new(); @@ -587,6 +589,65 @@ fn test_highlighting_via_c_api() { c::ts_highlight_buffer_delete(buffer); } +#[test] +fn test_highlighting_with_all_captures_applied() { + let source = "fn main(a: u32, b: u32) -> { let c = a + b; }"; + let language = get_language("rust"); + let highlights_query = indoc::indoc! {" + [ + \"fn\" + \"let\" + ] @keyword + (identifier) @variable + (function_item name: (identifier) @function) + (parameter pattern: (identifier) @variable.parameter) + (primitive_type) @type.builtin + \"=\" @operator + [ \"->\" \":\" \";\" ] @punctuation.delimiter + [ \"{\" \"}\" \"(\" \")\" ] @punctuation.bracket + "}; + let mut rust_highlight_reverse = + HighlightConfiguration::new(language, &highlights_query, "", "", true).unwrap(); + rust_highlight_reverse.configure(&HIGHLIGHT_NAMES); + + assert_eq!( + &to_token_vector(&source, &rust_highlight_reverse).unwrap(), + &[[ + ("fn", vec!["keyword"]), + (" ", vec![]), + ("main", vec!["function"]), + ("(", vec!["punctuation.bracket"]), + ("a", vec!["variable.parameter"]), + (":", vec!["punctuation.delimiter"]), + (" ", vec![]), + ("u32", vec!["type.builtin"]), + (", ", vec![]), + ("b", vec!["variable.parameter"]), + (":", vec!["punctuation.delimiter"]), + (" ", vec![]), + ("u32", vec!["type.builtin"]), + (")", vec!["punctuation.bracket"]), + (" ", vec![]), + ("->", vec!["punctuation.delimiter"]), + (" ", vec![]), + ("{", vec!["punctuation.bracket"]), + (" ", vec![]), + ("let", vec!["keyword"]), + (" ", vec![]), + ("c", vec!["variable"]), + (" ", vec![]), + ("=", vec!["operator"]), + (" ", vec![]), + ("a", vec!["variable"]), + (" + ", vec![]), + ("b", vec!["variable"]), + (";", vec!["punctuation.delimiter"]), + (" ", vec![]), + ("}", vec!["punctuation.bracket"]) + ]], + ); +} + #[test] fn test_decode_utf8_lossy() { use tree_sitter::LossyUtf8; diff --git a/highlight/include/tree_sitter/highlight.h b/highlight/include/tree_sitter/highlight.h index 496faea4..325cf413 100644 --- a/highlight/include/tree_sitter/highlight.h +++ b/highlight/include/tree_sitter/highlight.h @@ -48,7 +48,8 @@ TSHighlightError ts_highlighter_add_language( const char *locals_query, uint32_t highlight_query_len, uint32_t injection_query_len, - uint32_t locals_query_len + uint32_t locals_query_len, + bool apply_all_captures ); // Compute syntax highlighting for a given document. You must first diff --git a/highlight/src/c_lib.rs b/highlight/src/c_lib.rs index d48a180c..7f1c9490 100644 --- a/highlight/src/c_lib.rs +++ b/highlight/src/c_lib.rs @@ -70,6 +70,7 @@ pub extern "C" fn ts_highlighter_add_language( highlight_query_len: u32, injection_query_len: u32, locals_query_len: u32, + apply_all_captures: bool, ) -> ErrorCode { let f = move || { let this = unwrap_mut_ptr(this); @@ -109,9 +110,14 @@ pub extern "C" fn ts_highlighter_add_language( "" }; - let mut config = - HighlightConfiguration::new(language, highlight_query, injection_query, locals_query) - .or(Err(ErrorCode::InvalidQuery))?; + let mut config = HighlightConfiguration::new( + language, + highlight_query, + injection_query, + locals_query, + apply_all_captures, + ) + .or(Err(ErrorCode::InvalidQuery))?; config.configure(&this.highlight_names.as_slice()); this.languages.insert(scope_name, (injection_regex, config)); diff --git a/highlight/src/lib.rs b/highlight/src/lib.rs index 1042a1b2..4e72e729 100644 --- a/highlight/src/lib.rs +++ b/highlight/src/lib.rs @@ -103,6 +103,7 @@ pub enum HighlightEvent { pub struct HighlightConfiguration { pub language: Language, pub query: Query, + pub apply_all_captures: bool, combined_injections_query: Option, locals_pattern_index: usize, highlights_pattern_index: usize, @@ -160,6 +161,7 @@ where iter_count: usize, next_event: Option, last_highlight_range: Option<(usize, usize, usize)>, + apply_all_captures: bool, } struct HighlightIterLayer<'a> { @@ -215,9 +217,10 @@ impl Highlighter { cancellation_flag, highlighter: self, iter_count: 0, - layers: layers, + layers, next_event: None, last_highlight_range: None, + apply_all_captures: config.apply_all_captures, }; result.sort_layers(); Ok(result) @@ -244,6 +247,7 @@ impl HighlightConfiguration { highlights_query: &str, injection_query: &str, locals_query: &str, + apply_all_captures: bool, ) -> Result { // Concatenate the query strings, keeping track of the start offset of each section. let mut query_source = String::new(); @@ -324,6 +328,7 @@ impl HighlightConfiguration { Ok(HighlightConfiguration { language, query, + apply_all_captures, combined_injections_query, locals_pattern_index, highlights_pattern_index, @@ -929,7 +934,13 @@ where while let Some((next_match, next_capture_index)) = layer.captures.peek() { let next_capture = next_match.captures[*next_capture_index]; if next_capture.node == capture.node { - layer.captures.next(); + if self.apply_all_captures { + match_.remove(); + capture = next_capture; + match_ = layer.captures.next().unwrap().0; + } else { + layer.captures.next(); + } } else { break; }