From 26899f7246e6150e2139ad4e3aa95106a3b9d57f Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Sat, 18 Dec 2021 09:44:05 -0600 Subject: [PATCH 1/6] add getter for TagsContext parser --- tags/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tags/src/lib.rs b/tags/src/lib.rs index 2b148baa..df147fef 100644 --- a/tags/src/lib.rs +++ b/tags/src/lib.rs @@ -252,6 +252,10 @@ impl TagsContext { } } + pub fn parser(&mut self) -> &mut Parser { + &mut self.parser + } + pub fn generate_tags<'a>( &'a mut self, config: &'a TagsConfiguration, From 7df82c825f2b4cc81a445e1b21d0a6a8f8dbf569 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Sat, 18 Dec 2021 09:46:40 -0600 Subject: [PATCH 2/6] add module for testing tags --- cli/src/test_tags.rs | 153 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 cli/src/test_tags.rs diff --git a/cli/src/test_tags.rs b/cli/src/test_tags.rs new file mode 100644 index 00000000..a072330e --- /dev/null +++ b/cli/src/test_tags.rs @@ -0,0 +1,153 @@ +use crate::query_testing::{parse_position_comments, Assertion}; +use ansi_term::Colour; +use anyhow::{anyhow, Result}; +use std::fs; +use std::path::Path; +use tree_sitter::Point; +use tree_sitter_loader::Loader; +use tree_sitter_tags::{TagsConfiguration, TagsContext}; + +#[derive(Debug)] +pub struct Failure { + row: usize, + column: usize, + expected_tag: String, + actual_tags: Vec, +} + +impl std::error::Error for Failure {} + +impl std::fmt::Display for Failure { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "Failure - row: {}, column: {}, expected tag: '{}', actual tag: ", + self.row, self.column, self.expected_tag + )?; + if self.actual_tags.is_empty() { + write!(f, "none.")?; + } else { + for (i, actual_tag) in self.actual_tags.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "'{}'", actual_tag)?; + } + } + Ok(()) + } +} + +pub fn test_tags(loader: &Loader, directory: &Path) -> Result<()> { + let mut failed = false; + let mut tags_context = TagsContext::new(); + + println!("tags:"); + for tag_test_file in fs::read_dir(directory)? { + let tag_test_file = tag_test_file?; + let test_file_path = tag_test_file.path(); + let test_file_name = tag_test_file.file_name(); + let (language, language_config) = loader + .language_configuration_for_file_name(&test_file_path)? + .ok_or_else(|| anyhow!("No language found for path {:?}", test_file_path))?; + let tags_config = language_config + .tags_config(language)? + .ok_or_else(|| anyhow!("No tags config found for {:?}", test_file_path))?; + match test_tag( + &mut tags_context, + tags_config, + fs::read(&test_file_path)?.as_slice(), + ) { + Ok(assertion_count) => { + println!( + " ✓ {} ({} assertions)", + Colour::Green.paint(test_file_name.to_string_lossy().as_ref()), + assertion_count + ); + } + Err(e) => { + println!( + " ✗ {}", + Colour::Red.paint(test_file_name.to_string_lossy().as_ref()) + ); + println!(" {}", e); + failed = true; + } + } + } + + if failed { + Err(anyhow!("")) + } else { + Ok(()) + } +} + +pub fn test_tag( + tags_context: &mut TagsContext, + tags_config: &TagsConfiguration, + source: &[u8], +) -> Result { + let (tags_iter, _has_error) = tags_context.generate_tags(&tags_config, &source, None)?; + let tags: Vec = tags_iter.filter_map(|t| t.ok()).collect(); + let assertions = parse_position_comments(tags_context.parser(), tags_config.language, source)?; + + // Iterate through all of the assertions, checking against the actual tags. + let mut i = 0; + let mut actual_tags = Vec::::new(); + for Assertion { + position, + expected_capture_name: expected_tag, + } in &assertions + { + let mut passed = false; + + 'tag_loop: loop { + if let Some(tag) = tags.get(i) { + if tag.span.end <= *position { + i += 1; + continue; + } + + // Iterate through all of the tags that start at or before this assertion's + // position, looking for one that matches the assertion + let mut j = i; + while let (false, Some(tag)) = (passed, tags.get(j)) { + if tag.span.start > *position { + break 'tag_loop; + } + + let tag_postfix = tags_config.syntax_type_name(tag.syntax_type_id).to_string(); + let tag_name = if tag.is_definition { + format!("definition.{}", tag_postfix) + } else { + format!("reference.{}", tag_postfix) + }; + + if tag_name == *expected_tag { + passed = true; + break 'tag_loop; + } else { + actual_tags.push(tag_name); + } + + j += 1; + } + } else { + break; + } + } + + if !passed { + return Err(Failure { + row: position.row, + column: position.column, + expected_tag: expected_tag.clone(), + actual_tags: actual_tags.into_iter().collect(), + } + .into()); + } + } + + Ok(assertions.len()) +} From 067f742ad365c6b694eb42bf3679ef4f9a706b24 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Sat, 18 Dec 2021 09:44:59 -0600 Subject: [PATCH 3/6] run test_tag in 'tree-sitter test' when tags test dir exists --- cli/src/lib.rs | 1 + cli/src/main.rs | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 7de4afc5..d36417c2 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -8,6 +8,7 @@ pub mod query_testing; pub mod tags; pub mod test; pub mod test_highlight; +pub mod test_tags; pub mod util; pub mod wasm; diff --git a/cli/src/main.rs b/cli/src/main.rs index 554eb2c5..7d6cb703 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -4,7 +4,8 @@ use glob::glob; use std::path::Path; use std::{env, fs, u64}; use tree_sitter_cli::{ - generate, highlight, logger, parse, playground, query, tags, test, test_highlight, util, wasm, + generate, highlight, logger, parse, playground, query, tags, test, test_highlight, test_tags, + util, wasm, }; use tree_sitter_config::Config; use tree_sitter_loader as loader; @@ -316,6 +317,11 @@ fn run() -> Result<()> { if test_highlight_dir.is_dir() { test_highlight::test_highlights(&loader, &test_highlight_dir)?; } + + let test_tag_dir = test_dir.join("tags"); + if test_tag_dir.is_dir() { + test_tags::test_tags(&loader, &test_tag_dir)?; + } } ("parse", Some(matches)) => { From 47dc88b6471f8cb7bf74773f9bbabdcccb66b569 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Sat, 18 Dec 2021 10:48:59 -0600 Subject: [PATCH 4/6] add tags config fixture test helper --- cli/src/tests/helpers/fixtures.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cli/src/tests/helpers/fixtures.rs b/cli/src/tests/helpers/fixtures.rs index 91c4956a..7d04b24a 100644 --- a/cli/src/tests/helpers/fixtures.rs +++ b/cli/src/tests/helpers/fixtures.rs @@ -4,6 +4,7 @@ use std::path::{Path, PathBuf}; use tree_sitter::Language; use tree_sitter_highlight::HighlightConfiguration; use tree_sitter_loader::Loader; +use tree_sitter_tags::TagsConfiguration; include!("./dirs.rs"); @@ -54,6 +55,14 @@ pub fn get_highlight_config( result } +pub fn get_tags_config(language_name: &str) -> TagsConfiguration { + let language = get_language(language_name); + let queries_path = get_language_queries_path(language_name); + let tags_query = fs::read_to_string(queries_path.join("tags.scm")).unwrap(); + let locals_query = fs::read_to_string(queries_path.join("locals.scm")).unwrap_or(String::new()); + TagsConfiguration::new(language, &tags_query, &locals_query).unwrap() +} + pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> Language { let parser_c_path = SCRATCH_DIR.join(&format!("{}-parser.c", name)); if !fs::read_to_string(&parser_c_path) From dba7a138088f21527d8d6d20f93f847504081e55 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Sat, 18 Dec 2021 10:49:42 -0600 Subject: [PATCH 5/6] refactor out get_tag_positions helper function --- cli/src/test_tags.rs | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/cli/src/test_tags.rs b/cli/src/test_tags.rs index a072330e..024d094c 100644 --- a/cli/src/test_tags.rs +++ b/cli/src/test_tags.rs @@ -88,13 +88,12 @@ pub fn test_tag( tags_config: &TagsConfiguration, source: &[u8], ) -> Result { - let (tags_iter, _has_error) = tags_context.generate_tags(&tags_config, &source, None)?; - let tags: Vec = tags_iter.filter_map(|t| t.ok()).collect(); + let tags = get_tag_positions(tags_context, tags_config, source)?; let assertions = parse_position_comments(tags_context.parser(), tags_config.language, source)?; // Iterate through all of the assertions, checking against the actual tags. let mut i = 0; - let mut actual_tags = Vec::::new(); + let mut actual_tags = Vec::<&String>::new(); for Assertion { position, expected_capture_name: expected_tag, @@ -104,7 +103,7 @@ pub fn test_tag( 'tag_loop: loop { if let Some(tag) = tags.get(i) { - if tag.span.end <= *position { + if tag.1 <= *position { i += 1; continue; } @@ -113,18 +112,12 @@ pub fn test_tag( // position, looking for one that matches the assertion let mut j = i; while let (false, Some(tag)) = (passed, tags.get(j)) { - if tag.span.start > *position { + if tag.0 > *position { break 'tag_loop; } - let tag_postfix = tags_config.syntax_type_name(tag.syntax_type_id).to_string(); - let tag_name = if tag.is_definition { - format!("definition.{}", tag_postfix) - } else { - format!("reference.{}", tag_postfix) - }; - - if tag_name == *expected_tag { + let tag_name = &tag.2; + if *tag_name == *expected_tag { passed = true; break 'tag_loop; } else { @@ -143,7 +136,7 @@ pub fn test_tag( row: position.row, column: position.column, expected_tag: expected_tag.clone(), - actual_tags: actual_tags.into_iter().collect(), + actual_tags: actual_tags.into_iter().cloned().collect(), } .into()); } @@ -151,3 +144,24 @@ pub fn test_tag( Ok(assertions.len()) } + +pub fn get_tag_positions( + tags_context: &mut TagsContext, + tags_config: &TagsConfiguration, + source: &[u8], +) -> Result> { + let (tags_iter, _has_error) = tags_context.generate_tags(&tags_config, &source, None)?; + let tag_positions = tags_iter + .filter_map(|t| t.ok()) + .map(|tag| { + let tag_postfix = tags_config.syntax_type_name(tag.syntax_type_id).to_string(); + let tag_name = if tag.is_definition { + format!("definition.{}", tag_postfix) + } else { + format!("reference.{}", tag_postfix) + }; + (tag.span.start, tag.span.end, tag_name) + }) + .collect(); + Ok(tag_positions) +} From e7f0cc4e2449fd691c3cfbf898a2a80e58054bf8 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Sat, 18 Dec 2021 10:50:02 -0600 Subject: [PATCH 6/6] add python test case for tag testing --- cli/src/tests/mod.rs | 1 + cli/src/tests/test_tags_test.rs | 66 +++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 cli/src/tests/test_tags_test.rs diff --git a/cli/src/tests/mod.rs b/cli/src/tests/mod.rs index 24e8160e..1b804450 100644 --- a/cli/src/tests/mod.rs +++ b/cli/src/tests/mod.rs @@ -7,4 +7,5 @@ mod pathological_test; mod query_test; mod tags_test; mod test_highlight_test; +mod test_tags_test; mod tree_test; diff --git a/cli/src/tests/test_tags_test.rs b/cli/src/tests/test_tags_test.rs new file mode 100644 index 00000000..61f98abd --- /dev/null +++ b/cli/src/tests/test_tags_test.rs @@ -0,0 +1,66 @@ +use super::helpers::fixtures::{get_language, get_tags_config}; +use crate::query_testing::{parse_position_comments, Assertion}; +use crate::test_tags::get_tag_positions; +use tree_sitter::{Parser, Point}; +use tree_sitter_tags::TagsContext; + +#[test] +fn test_tags_test_with_basic_test() { + let language = get_language("python"); + let config = get_tags_config("python"); + let source = [ + "# hi", + "def abc(d):", + " # <- definition.function", + " e = fgh(d)", + " # ^ reference.call", + " return d(e)", + " # ^ reference.call", + "", + ] + .join("\n"); + + let assertions = + parse_position_comments(&mut Parser::new(), language, source.as_bytes()).unwrap(); + + assert_eq!( + assertions, + &[ + Assertion { + position: Point::new(1, 4), + expected_capture_name: "definition.function".to_string(), + }, + Assertion { + position: Point::new(3, 9), + expected_capture_name: "reference.call".to_string(), + }, + Assertion { + position: Point::new(5, 11), + expected_capture_name: "reference.call".to_string(), + }, + ] + ); + + let mut tags_context = TagsContext::new(); + let tag_positions = get_tag_positions(&mut tags_context, &config, source.as_bytes()).unwrap(); + assert_eq!( + tag_positions, + &[ + ( + Point::new(1, 4), + Point::new(1, 7), + "definition.function".to_string() + ), + ( + Point::new(3, 8), + Point::new(3, 11), + "reference.call".to_string() + ), + ( + Point::new(5, 11), + Point::new(5, 12), + "reference.call".to_string() + ), + ] + ) +}