From 6f13d6bbba9e9cd13a75919e1bcab686709e6d5f Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 21 Oct 2020 11:22:56 -0400 Subject: [PATCH 01/30] Define Python fixture --- test/fixtures/queries/python.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 test/fixtures/queries/python.py diff --git a/test/fixtures/queries/python.py b/test/fixtures/queries/python.py new file mode 100644 index 00000000..c90830a7 --- /dev/null +++ b/test/fixtures/queries/python.py @@ -0,0 +1,7 @@ +def foo(): pass +# declaration: function: 0, 0 + +def bar(): +# declaration: function, 3, 0 + foo() +# reference: call, 5, 4 From 91d5d59d85bf24a32840ea54404cfe80ef76cd2c Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 21 Oct 2020 12:37:24 -0400 Subject: [PATCH 02/30] Introduce query/assert and call it in query.rs. --- cli/src/main.rs | 13 +++++++++++-- cli/src/query.rs | 26 +++++++++++++++++++++----- cli/src/query/assert.rs | 23 +++++++++++++++++++++++ 3 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 cli/src/query/assert.rs diff --git a/cli/src/main.rs b/cli/src/main.rs index 2e55c2fb..7594ce27 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -95,7 +95,8 @@ fn run() -> error::Result<()> { .takes_value(true), ) .arg(Arg::with_name("scope").long("scope").takes_value(true)) - .arg(Arg::with_name("captures").long("captures").short("c")), + .arg(Arg::with_name("captures").long("captures").short("c")) + .arg(Arg::with_name("test").long("test")), ) .subcommand( SubCommand::with_name("tags") @@ -289,7 +290,15 @@ fn run() -> error::Result<()> { let r: Vec<&str> = br.split(":").collect(); (r[0].parse().unwrap(), r[1].parse().unwrap()) }); - query::query_files_at_paths(language, paths, query_path, ordered_captures, range)?; + let should_test = matches.is_present("test"); + query::query_files_at_paths( + language, + paths, + query_path, + ordered_captures, + range, + should_test, + )?; } else if let Some(matches) = matches.subcommand_matches("tags") { loader.find_all_languages(&config.parser_directories)?; let paths = collect_paths(matches.value_of("paths-file"), matches.values_of("paths"))?; diff --git a/cli/src/query.rs b/cli/src/query.rs index e71e6254..d2aefc7f 100644 --- a/cli/src/query.rs +++ b/cli/src/query.rs @@ -4,12 +4,17 @@ use std::io::{self, Write}; use std::path::Path; use tree_sitter::{Language, Node, Parser, Query, QueryCursor}; +mod assert; + +use assert::CaptureInfo; + pub fn query_files_at_paths( language: Language, paths: Vec, query_path: &Path, ordered_captures: bool, range: Option<(usize, usize)>, + should_test: bool, ) -> Result<()> { let stdout = io::stdout(); let mut stdout = stdout.lock(); @@ -29,6 +34,8 @@ pub fn query_files_at_paths( parser.set_language(language).map_err(|e| e.to_string())?; for path in paths { + let mut results = Vec::new(); + writeln!(&mut stdout, "{}", path)?; let source_code = fs::read(&path).map_err(Error::wrap(|| { @@ -42,14 +49,18 @@ pub fn query_files_at_paths( query_cursor.captures(&query, tree.root_node(), text_callback) { let capture = mat.captures[capture_index]; + let capture_name = &query.capture_names()[capture.index as usize]; writeln!( &mut stdout, " pattern: {}, capture: {}, row: {}, text: {:?}", mat.pattern_index, - &query.capture_names()[capture.index as usize], + capture_name, capture.node.start_position().row, capture.node.utf8_text(&source_code).unwrap_or("") )?; + results.push(CaptureInfo { + name: capture_name.to_string(), + }); } } else { for m in query_cursor.matches(&query, tree.root_node(), text_callback) { @@ -57,11 +68,12 @@ pub fn query_files_at_paths( for capture in m.captures { let start = capture.node.start_position(); let end = capture.node.end_position(); + let capture_name = &query.capture_names()[capture.index as usize]; if end.row == start.row { writeln!( &mut stdout, " capture: {}, start: {}, text: {:?}", - &query.capture_names()[capture.index as usize], + capture_name, start, capture.node.utf8_text(&source_code).unwrap_or("") )?; @@ -69,14 +81,18 @@ pub fn query_files_at_paths( writeln!( &mut stdout, " capture: {}, start: {}, end: {}", - &query.capture_names()[capture.index as usize], - start, - end, + capture_name, start, end, )?; } + results.push(CaptureInfo { + name: capture_name.to_string(), + }); } } } + if should_test { + assert::assert_expected_captures(results, path); + } } Ok(()) diff --git a/cli/src/query/assert.rs b/cli/src/query/assert.rs new file mode 100644 index 00000000..5fb7e1d6 --- /dev/null +++ b/cli/src/query/assert.rs @@ -0,0 +1,23 @@ +use lazy_static::lazy_static; +use regex::Regex; +use tree_sitter::Point; + +// TODO: It would be cooler to do this with a comments query rather than with a regex +// directly. +lazy_static! { + static ref METADATA_PAIR_REGEX: Regex = Regex::new(r#"(\w+): ([^\s,]+)"#).unwrap(); + static ref NUMBER_REGEX: Regex = Regex::new(r#"\d+"#).unwrap(); +} + +pub struct CaptureInfo { + pub name: String, +} + +#[derive(Debug, Eq, PartialEq)] +struct Assertion { + position: Point, + line_numbers: Vec, + capture_type: String, +} + +pub fn assert_expected_captures(_captures: Vec, _path: String) {} From 947528f01903930dd8e7201fff6e32d00ea79541 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 21 Oct 2020 12:49:41 -0400 Subject: [PATCH 03/30] use our Result type here --- cli/src/query.rs | 2 +- cli/src/query/assert.rs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cli/src/query.rs b/cli/src/query.rs index d2aefc7f..56b86740 100644 --- a/cli/src/query.rs +++ b/cli/src/query.rs @@ -91,7 +91,7 @@ pub fn query_files_at_paths( } } if should_test { - assert::assert_expected_captures(results, path); + assert::assert_expected_captures(results, path)? } } diff --git a/cli/src/query/assert.rs b/cli/src/query/assert.rs index 5fb7e1d6..d4140f23 100644 --- a/cli/src/query/assert.rs +++ b/cli/src/query/assert.rs @@ -1,3 +1,4 @@ +use super::super::error::Result; use lazy_static::lazy_static; use regex::Regex; use tree_sitter::Point; @@ -20,4 +21,6 @@ struct Assertion { capture_type: String, } -pub fn assert_expected_captures(_captures: Vec, _path: String) {} +pub fn assert_expected_captures(_captures: Vec, _path: String) -> Result<()> { + Ok(()) +} From c691df5ae22ff1ed3e20685186fcafc5adb163bf Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 21 Oct 2020 12:56:11 -0400 Subject: [PATCH 04/30] reading in the source correctly --- cli/src/query/assert.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/src/query/assert.rs b/cli/src/query/assert.rs index d4140f23..35a7f4b7 100644 --- a/cli/src/query/assert.rs +++ b/cli/src/query/assert.rs @@ -1,6 +1,7 @@ use super::super::error::Result; use lazy_static::lazy_static; use regex::Regex; +use std::fs; use tree_sitter::Point; // TODO: It would be cooler to do this with a comments query rather than with a regex @@ -21,6 +22,10 @@ struct Assertion { capture_type: String, } -pub fn assert_expected_captures(_captures: Vec, _path: String) -> Result<()> { +pub fn assert_expected_captures(_captures: Vec, path: String) -> Result<()> { + let contents = fs::read_to_string(path)?; + for m in METADATA_PAIR_REGEX.captures_iter(&contents) { + println!("pair: {:?}", m); + } Ok(()) } From 0dfe89f3538d7ed8f08d8b9b7b05e515d82989c0 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 21 Oct 2020 13:32:04 -0400 Subject: [PATCH 05/30] parse assertions from regex capture --- cli/src/query/assert.rs | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/cli/src/query/assert.rs b/cli/src/query/assert.rs index 35a7f4b7..d5998eaf 100644 --- a/cli/src/query/assert.rs +++ b/cli/src/query/assert.rs @@ -7,8 +7,7 @@ use tree_sitter::Point; // TODO: It would be cooler to do this with a comments query rather than with a regex // directly. lazy_static! { - static ref METADATA_PAIR_REGEX: Regex = Regex::new(r#"(\w+): ([^\s,]+)"#).unwrap(); - static ref NUMBER_REGEX: Regex = Regex::new(r#"\d+"#).unwrap(); + static ref METADATA_REGEX: Regex = Regex::new(r#"(\w+): ([^\s,]+), (\d+), (\d+)"#).unwrap(); } pub struct CaptureInfo { @@ -18,14 +17,44 @@ pub struct CaptureInfo { #[derive(Debug, Eq, PartialEq)] struct Assertion { position: Point, - line_numbers: Vec, + capture_class: String, capture_type: String, } +impl From> for Assertion { + fn from(re: regex::Captures) -> Assertion { + Assertion { + capture_class: re.get(1).unwrap().as_str().to_string(), + capture_type: re.get(2).unwrap().as_str().to_string(), + position: Point { + row: re + .get(3) + .iter() + .flat_map(|m| m.as_str().parse::()) + .next() + .unwrap(), + column: re + .get(4) + .iter() + .flat_map(|m| m.as_str().parse::()) + .next() + .unwrap(), + }, + } + } +} + pub fn assert_expected_captures(_captures: Vec, path: String) -> Result<()> { let contents = fs::read_to_string(path)?; - for m in METADATA_PAIR_REGEX.captures_iter(&contents) { - println!("pair: {:?}", m); + + let assertions: Vec = METADATA_REGEX + .captures_iter(&contents) + .map(|c| Assertion::from(c)) + .collect(); + + for a in assertions { + println!("a: {:?}", a); } + Ok(()) } From 363a0ce4fccd59230df9063cfded5dbede15907c Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Wed, 21 Oct 2020 14:54:47 -0400 Subject: [PATCH 06/30] things are working: time to piggyback off the highlighter's parser --- cli/src/query.rs | 2 ++ cli/src/query/assert.rs | 25 +++++++++++++++++++++---- test/fixtures/queries/python.py | 4 ++-- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/cli/src/query.rs b/cli/src/query.rs index 56b86740..704a2c56 100644 --- a/cli/src/query.rs +++ b/cli/src/query.rs @@ -60,6 +60,7 @@ pub fn query_files_at_paths( )?; results.push(CaptureInfo { name: capture_name.to_string(), + position: capture.node.start_position(), }); } } else { @@ -86,6 +87,7 @@ pub fn query_files_at_paths( } results.push(CaptureInfo { name: capture_name.to_string(), + position: capture.node.start_position(), }); } } diff --git a/cli/src/query/assert.rs b/cli/src/query/assert.rs index d5998eaf..1b31c1c0 100644 --- a/cli/src/query/assert.rs +++ b/cli/src/query/assert.rs @@ -1,6 +1,8 @@ +use super::super::error; use super::super::error::Result; use lazy_static::lazy_static; use regex::Regex; +use std::collections::hash_map::HashMap; use std::fs; use tree_sitter::Point; @@ -10,8 +12,10 @@ lazy_static! { static ref METADATA_REGEX: Regex = Regex::new(r#"(\w+): ([^\s,]+), (\d+), (\d+)"#).unwrap(); } +#[derive(Debug, Eq, PartialEq)] pub struct CaptureInfo { pub name: String, + pub position: Point, } #[derive(Debug, Eq, PartialEq)] @@ -44,7 +48,7 @@ impl From> for Assertion { } } -pub fn assert_expected_captures(_captures: Vec, path: String) -> Result<()> { +pub fn assert_expected_captures(captures: Vec, path: String) -> Result<()> { let contents = fs::read_to_string(path)?; let assertions: Vec = METADATA_REGEX @@ -52,9 +56,22 @@ pub fn assert_expected_captures(_captures: Vec, path: String) -> Re .map(|c| Assertion::from(c)) .collect(); - for a in assertions { - println!("a: {:?}", a); - } + let per_position_index: HashMap = + assertions.iter().map(|a| (a.position, a)).collect(); + for capture in &captures { + let oFound = per_position_index.get(&capture.position); + if oFound.is_none() { + continue; + } + let found = oFound.unwrap(); + let joined = format!("{}.{}", found.capture_class, found.capture_type); + if joined != capture.name && capture.name != "name" { + Err(error::Error::new(format!( + "Assertion failed: at {}, found {}, expected {}", + capture.position, capture.name, joined + )))? + } + } Ok(()) } diff --git a/test/fixtures/queries/python.py b/test/fixtures/queries/python.py index c90830a7..a48ed2de 100644 --- a/test/fixtures/queries/python.py +++ b/test/fixtures/queries/python.py @@ -1,7 +1,7 @@ def foo(): pass -# declaration: function: 0, 0 +# definition: function: 0, 0 def bar(): -# declaration: function, 3, 0 +# definition: function, 3, 0 foo() # reference: call, 5, 4 From e370c5053e2134a44b9f35a5347be408b0c88135 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Fri, 23 Oct 2020 14:11:46 -0400 Subject: [PATCH 07/30] this is nicer, though --- cli/src/query/assert.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/cli/src/query/assert.rs b/cli/src/query/assert.rs index 1b31c1c0..96162c5b 100644 --- a/cli/src/query/assert.rs +++ b/cli/src/query/assert.rs @@ -1,5 +1,5 @@ -use super::super::error; -use super::super::error::Result; +use crate::error; +use crate::error::Result; use lazy_static::lazy_static; use regex::Regex; use std::collections::hash_map::HashMap; @@ -48,7 +48,7 @@ impl From> for Assertion { } } -pub fn assert_expected_captures(captures: Vec, path: String) -> Result<()> { +pub fn assert_expected_captures(infos: Vec, path: String) -> Result<()> { let contents = fs::read_to_string(path)?; let assertions: Vec = METADATA_REGEX @@ -59,17 +59,16 @@ pub fn assert_expected_captures(captures: Vec, path: String) -> Res let per_position_index: HashMap = assertions.iter().map(|a| (a.position, a)).collect(); - for capture in &captures { - let oFound = per_position_index.get(&capture.position); - if oFound.is_none() { + for info in &infos { + if !per_position_index.contains_key(&info.position) { continue; } - let found = oFound.unwrap(); + let found = per_position_index.get(&info.position).unwrap(); let joined = format!("{}.{}", found.capture_class, found.capture_type); - if joined != capture.name && capture.name != "name" { + if joined != info.name && info.name != "name" { Err(error::Error::new(format!( "Assertion failed: at {}, found {}, expected {}", - capture.position, capture.name, joined + info.position, info.name, joined )))? } } From 3e18e97f7c1e8806bcf471b69b37de65e9a6572e Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 26 Oct 2020 12:58:32 -0400 Subject: [PATCH 08/30] start pulling in the stuff from test_highlight --- cli/src/query.rs | 2 +- cli/src/query/assert.rs | 11 +++++++++-- test/fixtures/queries/python.py | 6 +++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cli/src/query.rs b/cli/src/query.rs index 704a2c56..de320fba 100644 --- a/cli/src/query.rs +++ b/cli/src/query.rs @@ -93,7 +93,7 @@ pub fn query_files_at_paths( } } if should_test { - assert::assert_expected_captures(results, path)? + assert::assert_expected_captures(results, path, &mut parser, language)? } } diff --git a/cli/src/query/assert.rs b/cli/src/query/assert.rs index 96162c5b..69f0de91 100644 --- a/cli/src/query/assert.rs +++ b/cli/src/query/assert.rs @@ -1,10 +1,11 @@ use crate::error; use crate::error::Result; +use crate::test_highlight::parse_highlight_test; use lazy_static::lazy_static; use regex::Regex; use std::collections::hash_map::HashMap; use std::fs; -use tree_sitter::Point; +use tree_sitter::{Language, Parser, Point}; // TODO: It would be cooler to do this with a comments query rather than with a regex // directly. @@ -48,8 +49,14 @@ impl From> for Assertion { } } -pub fn assert_expected_captures(infos: Vec, path: String) -> Result<()> { +pub fn assert_expected_captures( + infos: Vec, + path: String, + parser: &mut Parser, + language: Language, +) -> Result<()> { let contents = fs::read_to_string(path)?; + let _pairs = parse_highlight_test(parser, language, contents.as_bytes()); let assertions: Vec = METADATA_REGEX .captures_iter(&contents) diff --git a/test/fixtures/queries/python.py b/test/fixtures/queries/python.py index a48ed2de..01ec9ab0 100644 --- a/test/fixtures/queries/python.py +++ b/test/fixtures/queries/python.py @@ -1,7 +1,7 @@ def foo(): pass -# definition: function: 0, 0 +# <- definition.function def bar(): -# definition: function, 3, 0 +# <- definition.function foo() -# reference: call, 5, 4 + # <- reference.call From 9af9d66e194a6ea809d0646b513015829ecf3343 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 26 Oct 2020 13:13:25 -0400 Subject: [PATCH 09/30] it works --- cli/src/query/assert.rs | 39 +++++++++++---------------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/cli/src/query/assert.rs b/cli/src/query/assert.rs index 69f0de91..5f042f3b 100644 --- a/cli/src/query/assert.rs +++ b/cli/src/query/assert.rs @@ -22,29 +22,15 @@ pub struct CaptureInfo { #[derive(Debug, Eq, PartialEq)] struct Assertion { position: Point, - capture_class: String, - capture_type: String, + expected: String, } -impl From> for Assertion { - fn from(re: regex::Captures) -> Assertion { +impl From<&(Point, String)> for Assertion { + fn from(item: &(Point, String)) -> Assertion { + let (pos, info) = item; Assertion { - capture_class: re.get(1).unwrap().as_str().to_string(), - capture_type: re.get(2).unwrap().as_str().to_string(), - position: Point { - row: re - .get(3) - .iter() - .flat_map(|m| m.as_str().parse::()) - .next() - .unwrap(), - column: re - .get(4) - .iter() - .flat_map(|m| m.as_str().parse::()) - .next() - .unwrap(), - }, + position: *pos, + expected: info.to_string(), } } } @@ -56,12 +42,10 @@ pub fn assert_expected_captures( language: Language, ) -> Result<()> { let contents = fs::read_to_string(path)?; - let _pairs = parse_highlight_test(parser, language, contents.as_bytes()); + let pairs = parse_highlight_test(parser, language, contents.as_bytes())?; + println!("{:?}", pairs); - let assertions: Vec = METADATA_REGEX - .captures_iter(&contents) - .map(|c| Assertion::from(c)) - .collect(); + let assertions: Vec = pairs.iter().map(Assertion::from).collect(); let per_position_index: HashMap = assertions.iter().map(|a| (a.position, a)).collect(); @@ -71,11 +55,10 @@ pub fn assert_expected_captures( continue; } let found = per_position_index.get(&info.position).unwrap(); - let joined = format!("{}.{}", found.capture_class, found.capture_type); - if joined != info.name && info.name != "name" { + if found.expected != info.name && info.name != "name" { Err(error::Error::new(format!( "Assertion failed: at {}, found {}, expected {}", - info.position, info.name, joined + info.position, info.name, found.expected )))? } } From f364ce2304371cbe89f40fa14e471919dac23015 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 26 Oct 2020 13:22:12 -0400 Subject: [PATCH 10/30] Remove old assertion stuff --- cli/src/query/assert.rs | 33 +++------------------------------ 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/cli/src/query/assert.rs b/cli/src/query/assert.rs index 5f042f3b..352f8de5 100644 --- a/cli/src/query/assert.rs +++ b/cli/src/query/assert.rs @@ -1,40 +1,16 @@ use crate::error; use crate::error::Result; use crate::test_highlight::parse_highlight_test; -use lazy_static::lazy_static; -use regex::Regex; use std::collections::hash_map::HashMap; use std::fs; use tree_sitter::{Language, Parser, Point}; -// TODO: It would be cooler to do this with a comments query rather than with a regex -// directly. -lazy_static! { - static ref METADATA_REGEX: Regex = Regex::new(r#"(\w+): ([^\s,]+), (\d+), (\d+)"#).unwrap(); -} - #[derive(Debug, Eq, PartialEq)] pub struct CaptureInfo { pub name: String, pub position: Point, } -#[derive(Debug, Eq, PartialEq)] -struct Assertion { - position: Point, - expected: String, -} - -impl From<&(Point, String)> for Assertion { - fn from(item: &(Point, String)) -> Assertion { - let (pos, info) = item; - Assertion { - position: *pos, - expected: info.to_string(), - } - } -} - pub fn assert_expected_captures( infos: Vec, path: String, @@ -45,20 +21,17 @@ pub fn assert_expected_captures( let pairs = parse_highlight_test(parser, language, contents.as_bytes())?; println!("{:?}", pairs); - let assertions: Vec = pairs.iter().map(Assertion::from).collect(); - - let per_position_index: HashMap = - assertions.iter().map(|a| (a.position, a)).collect(); + let per_position_index: HashMap = pairs.iter().map(|(a, b)| (*a, b)).collect(); for info in &infos { if !per_position_index.contains_key(&info.position) { continue; } let found = per_position_index.get(&info.position).unwrap(); - if found.expected != info.name && info.name != "name" { + if **found != info.name && info.name != "name" { Err(error::Error::new(format!( "Assertion failed: at {}, found {}, expected {}", - info.position, info.name, found.expected + info.position, info.name, found )))? } } From 1012bea3f4565231058b7c57602150b9cdd63ad3 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 26 Oct 2020 13:35:10 -0400 Subject: [PATCH 11/30] let's start sharing this code --- cli/src/query/assert.rs | 4 ++-- cli/src/test_highlight.rs | 30 ++++++++++++++++++++++-------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/cli/src/query/assert.rs b/cli/src/query/assert.rs index 352f8de5..10992959 100644 --- a/cli/src/query/assert.rs +++ b/cli/src/query/assert.rs @@ -19,9 +19,9 @@ pub fn assert_expected_captures( ) -> Result<()> { let contents = fs::read_to_string(path)?; let pairs = parse_highlight_test(parser, language, contents.as_bytes())?; - println!("{:?}", pairs); - let per_position_index: HashMap = pairs.iter().map(|(a, b)| (*a, b)).collect(); + let per_position_index: HashMap = + pairs.iter().map(|a| (a.position, &a.expected)).collect(); for info in &infos { if !per_position_index.contains_key(&info.position) { diff --git a/cli/src/test_highlight.rs b/cli/src/test_highlight.rs index cf163c05..45841a7b 100644 --- a/cli/src/test_highlight.rs +++ b/cli/src/test_highlight.rs @@ -12,6 +12,11 @@ lazy_static! { static ref HIGHLIGHT_NAME_REGEX: Regex = Regex::new("[\\w_\\-.]+").unwrap(); } +pub struct Assertion { + pub position: Point, + pub expected: String, +} + pub struct Failure { row: usize, column: usize, @@ -102,7 +107,11 @@ pub fn test_highlight( // actual highlights. let mut i = 0; let mut actual_highlights = Vec::<&String>::new(); - for (position, expected_highlight) in &assertions { + for Assertion { + position, + expected: expected_highlight, + } in &assertions + { let mut passed = false; actual_highlights.clear(); @@ -163,7 +172,7 @@ pub fn parse_highlight_test( parser: &mut Parser, language: Language, source: &[u8], -) -> Result> { +) -> Result> { let mut result = Vec::new(); let mut assertion_ranges = Vec::new(); @@ -213,7 +222,10 @@ pub fn parse_highlight_test( (has_arrow, HIGHLIGHT_NAME_REGEX.find(&text[arrow_end..])) { assertion_ranges.push((node.start_position(), node.end_position())); - result.push((position, mat.as_str().to_string())); + result.push(Assertion { + position: position, + expected: mat.as_str().to_string(), + }); } } } @@ -233,15 +245,17 @@ pub fn parse_highlight_test( // code *above* the assertion. There can be multiple lines of assertion comments, // so the positions may have to be decremented by more than one row. let mut i = 0; - for (position, _) in result.iter_mut() { + for assertion in result.iter_mut() { loop { let on_assertion_line = assertion_ranges[i..] .iter() - .any(|(start, _)| start.row == position.row); + .any(|(start, _)| start.row == assertion.position.row); if on_assertion_line { - position.row -= 1; + assertion.position.row -= 1; } else { - while i < assertion_ranges.len() && assertion_ranges[i].0.row < position.row { + while i < assertion_ranges.len() + && assertion_ranges[i].0.row < assertion.position.row + { i += 1; } break; @@ -250,7 +264,7 @@ pub fn parse_highlight_test( } // The assertions can end up out of order due to the line adjustments. - result.sort_unstable_by_key(|a| a.0); + result.sort_unstable_by_key(|a| a.position); Ok(result) } From 6adeb7b40d01fc85f5f14d8fda2339c9e17cc7a4 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 26 Oct 2020 14:27:33 -0400 Subject: [PATCH 12/30] move shared code to query_testing --- cli/src/lib.rs | 1 + cli/src/query.rs | 11 +-- cli/src/query/assert.rs | 39 ---------- cli/src/query_testing.rs | 153 ++++++++++++++++++++++++++++++++++++++ cli/src/test_highlight.rs | 118 +---------------------------- 5 files changed, 160 insertions(+), 162 deletions(-) delete mode 100644 cli/src/query/assert.rs create mode 100644 cli/src/query_testing.rs diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 97c288a1..e00323b7 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -6,6 +6,7 @@ pub mod loader; pub mod logger; pub mod parse; pub mod query; +pub mod query_testing; pub mod tags; pub mod test; pub mod test_highlight; diff --git a/cli/src/query.rs b/cli/src/query.rs index de320fba..9c524877 100644 --- a/cli/src/query.rs +++ b/cli/src/query.rs @@ -1,13 +1,10 @@ use super::error::{Error, Result}; +use crate::query_testing; use std::fs; use std::io::{self, Write}; use std::path::Path; use tree_sitter::{Language, Node, Parser, Query, QueryCursor}; -mod assert; - -use assert::CaptureInfo; - pub fn query_files_at_paths( language: Language, paths: Vec, @@ -58,7 +55,7 @@ pub fn query_files_at_paths( capture.node.start_position().row, capture.node.utf8_text(&source_code).unwrap_or("") )?; - results.push(CaptureInfo { + results.push(query_testing::CaptureInfo { name: capture_name.to_string(), position: capture.node.start_position(), }); @@ -85,7 +82,7 @@ pub fn query_files_at_paths( capture_name, start, end, )?; } - results.push(CaptureInfo { + results.push(query_testing::CaptureInfo { name: capture_name.to_string(), position: capture.node.start_position(), }); @@ -93,7 +90,7 @@ pub fn query_files_at_paths( } } if should_test { - assert::assert_expected_captures(results, path, &mut parser, language)? + query_testing::assert_expected_captures(results, path, &mut parser, language)? } } diff --git a/cli/src/query/assert.rs b/cli/src/query/assert.rs deleted file mode 100644 index 10992959..00000000 --- a/cli/src/query/assert.rs +++ /dev/null @@ -1,39 +0,0 @@ -use crate::error; -use crate::error::Result; -use crate::test_highlight::parse_highlight_test; -use std::collections::hash_map::HashMap; -use std::fs; -use tree_sitter::{Language, Parser, Point}; - -#[derive(Debug, Eq, PartialEq)] -pub struct CaptureInfo { - pub name: String, - pub position: Point, -} - -pub fn assert_expected_captures( - infos: Vec, - path: String, - parser: &mut Parser, - language: Language, -) -> Result<()> { - let contents = fs::read_to_string(path)?; - let pairs = parse_highlight_test(parser, language, contents.as_bytes())?; - - let per_position_index: HashMap = - pairs.iter().map(|a| (a.position, &a.expected)).collect(); - - for info in &infos { - if !per_position_index.contains_key(&info.position) { - continue; - } - let found = per_position_index.get(&info.position).unwrap(); - if **found != info.name && info.name != "name" { - Err(error::Error::new(format!( - "Assertion failed: at {}, found {}, expected {}", - info.position, info.name, found - )))? - } - } - Ok(()) -} diff --git a/cli/src/query_testing.rs b/cli/src/query_testing.rs new file mode 100644 index 00000000..b6e2c169 --- /dev/null +++ b/cli/src/query_testing.rs @@ -0,0 +1,153 @@ +use crate::error; +use crate::error::Result; +use lazy_static::lazy_static; +use regex::Regex; +use std::collections::hash_map::HashMap; +use std::fs; +use tree_sitter::{Language, Parser, Point}; + +lazy_static! { + static ref HIGHLIGHT_NAME_REGEX: Regex = Regex::new("[\\w_\\-.]+").unwrap(); +} + +#[derive(Debug, Eq, PartialEq)] +pub struct CaptureInfo { + pub name: String, + pub position: Point, +} + +pub struct Assertion { + pub position: Point, + pub expected: String, +} + +/// Parse the given source code, finding all of the comments that contain +/// highlighting assertions. Return a vector of (position, expected highlight name) +/// pairs. +pub fn parse_highlight_test( + parser: &mut Parser, + language: Language, + source: &[u8], +) -> Result> { + let mut result = Vec::new(); + let mut assertion_ranges = Vec::new(); + + // Parse the code. + parser.set_included_ranges(&[]).unwrap(); + parser.set_language(language).unwrap(); + let tree = parser.parse(source, None).unwrap(); + + // Walk the tree, finding comment nodes that contain assertions. + let mut ascending = false; + let mut cursor = tree.root_node().walk(); + loop { + if ascending { + let node = cursor.node(); + + // Find every comment node. + if node.kind().contains("comment") { + if let Ok(text) = node.utf8_text(source) { + let mut position = node.start_position(); + if position.row == 0 { + continue; + } + + // Find the arrow character ("^" or '<-") in the comment. A left arrow + // refers to the column where the comment node starts. An up arrow refers + // to its own column. + let mut has_left_caret = false; + let mut has_arrow = false; + let mut arrow_end = 0; + for (i, c) in text.char_indices() { + arrow_end = i + 1; + if c == '-' && has_left_caret { + has_arrow = true; + break; + } + if c == '^' { + has_arrow = true; + position.column += i; + break; + } + has_left_caret = c == '<'; + } + + // If the comment node contains an arrow and a highlight name, record the + // highlight name and the position. + if let (true, Some(mat)) = + (has_arrow, HIGHLIGHT_NAME_REGEX.find(&text[arrow_end..])) + { + assertion_ranges.push((node.start_position(), node.end_position())); + result.push(Assertion { + position: position, + expected: mat.as_str().to_string(), + }); + } + } + } + + // Continue walking the tree. + if cursor.goto_next_sibling() { + ascending = false; + } else if !cursor.goto_parent() { + break; + } + } else if !cursor.goto_first_child() { + ascending = true; + } + } + + // Adjust the row number in each assertion's position to refer to the line of + // code *above* the assertion. There can be multiple lines of assertion comments, + // so the positions may have to be decremented by more than one row. + let mut i = 0; + for assertion in result.iter_mut() { + loop { + let on_assertion_line = assertion_ranges[i..] + .iter() + .any(|(start, _)| start.row == assertion.position.row); + if on_assertion_line { + assertion.position.row -= 1; + } else { + while i < assertion_ranges.len() + && assertion_ranges[i].0.row < assertion.position.row + { + i += 1; + } + break; + } + } + } + + // The assertions can end up out of order due to the line adjustments. + result.sort_unstable_by_key(|a| a.position); + + Ok(result) +} + +pub fn assert_expected_captures( + infos: Vec, + path: String, + parser: &mut Parser, + language: Language, +) -> Result<()> { + let contents = fs::read_to_string(path)?; + let pairs = parse_highlight_test(parser, language, contents.as_bytes())?; + + let per_position_index: HashMap = + pairs.iter().map(|a| (a.position, &a.expected)).collect(); + + for info in &infos { + if !per_position_index.contains_key(&info.position) { + continue; + } + let found = per_position_index.get(&info.position).unwrap(); + if **found != info.name && info.name != "name" { + Err(error::Error::new(format!( + "Assertion failed: at {}, found {}, expected {}", + info.position, found, info.name + )))? + } + } + Ok(()) +} diff --git a/cli/src/test_highlight.rs b/cli/src/test_highlight.rs index 45841a7b..da67f753 100644 --- a/cli/src/test_highlight.rs +++ b/cli/src/test_highlight.rs @@ -1,22 +1,12 @@ use super::error::Result; use crate::loader::Loader; +use crate::query_testing::{parse_highlight_test, Assertion}; use ansi_term::Colour; -use lazy_static::lazy_static; -use regex::Regex; use std::fs; use std::path::Path; -use tree_sitter::{Language, Parser, Point}; +use tree_sitter::Point; use tree_sitter_highlight::{Highlight, HighlightConfiguration, HighlightEvent, Highlighter}; -lazy_static! { - static ref HIGHLIGHT_NAME_REGEX: Regex = Regex::new("[\\w_\\-.]+").unwrap(); -} - -pub struct Assertion { - pub position: Point, - pub expected: String, -} - pub struct Failure { row: usize, column: usize, @@ -165,110 +155,6 @@ pub fn test_highlight( Ok(assertions.len()) } -/// Parse the given source code, finding all of the comments that contain -/// highlighting assertions. Return a vector of (position, expected highlight name) -/// pairs. -pub fn parse_highlight_test( - parser: &mut Parser, - language: Language, - source: &[u8], -) -> Result> { - let mut result = Vec::new(); - let mut assertion_ranges = Vec::new(); - - // Parse the code. - parser.set_included_ranges(&[]).unwrap(); - parser.set_language(language).unwrap(); - let tree = parser.parse(source, None).unwrap(); - - // Walk the tree, finding comment nodes that contain assertions. - let mut ascending = false; - let mut cursor = tree.root_node().walk(); - loop { - if ascending { - let node = cursor.node(); - - // Find every comment node. - if node.kind().contains("comment") { - if let Ok(text) = node.utf8_text(source) { - let mut position = node.start_position(); - if position.row == 0 { - continue; - } - - // Find the arrow character ("^" or '<-") in the comment. A left arrow - // refers to the column where the comment node starts. An up arrow refers - // to its own column. - let mut has_left_caret = false; - let mut has_arrow = false; - let mut arrow_end = 0; - for (i, c) in text.char_indices() { - arrow_end = i + 1; - if c == '-' && has_left_caret { - has_arrow = true; - break; - } - if c == '^' { - has_arrow = true; - position.column += i; - break; - } - has_left_caret = c == '<'; - } - - // If the comment node contains an arrow and a highlight name, record the - // highlight name and the position. - if let (true, Some(mat)) = - (has_arrow, HIGHLIGHT_NAME_REGEX.find(&text[arrow_end..])) - { - assertion_ranges.push((node.start_position(), node.end_position())); - result.push(Assertion { - position: position, - expected: mat.as_str().to_string(), - }); - } - } - } - - // Continue walking the tree. - if cursor.goto_next_sibling() { - ascending = false; - } else if !cursor.goto_parent() { - break; - } - } else if !cursor.goto_first_child() { - ascending = true; - } - } - - // Adjust the row number in each assertion's position to refer to the line of - // code *above* the assertion. There can be multiple lines of assertion comments, - // so the positions may have to be decremented by more than one row. - let mut i = 0; - for assertion in result.iter_mut() { - loop { - let on_assertion_line = assertion_ranges[i..] - .iter() - .any(|(start, _)| start.row == assertion.position.row); - if on_assertion_line { - assertion.position.row -= 1; - } else { - while i < assertion_ranges.len() - && assertion_ranges[i].0.row < assertion.position.row - { - i += 1; - } - break; - } - } - } - - // The assertions can end up out of order due to the line adjustments. - result.sort_unstable_by_key(|a| a.position); - - Ok(result) -} - pub fn get_highlight_positions( loader: &Loader, highlighter: &mut Highlighter, From 1aee60a7c074f6482695a2babf4519bc1064d640 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 26 Oct 2020 14:35:18 -0400 Subject: [PATCH 13/30] propitiate the tests --- cli/src/query_testing.rs | 5 +++-- cli/src/test_highlight.rs | 5 +++-- cli/src/tests/test_highlight_test.rs | 21 ++++++++++++++++----- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/cli/src/query_testing.rs b/cli/src/query_testing.rs index b6e2c169..9618d1f6 100644 --- a/cli/src/query_testing.rs +++ b/cli/src/query_testing.rs @@ -16,6 +16,7 @@ pub struct CaptureInfo { pub position: Point, } +#[derive(Debug, PartialEq, Eq)] pub struct Assertion { pub position: Point, pub expected: String, @@ -24,7 +25,7 @@ pub struct Assertion { /// Parse the given source code, finding all of the comments that contain /// highlighting assertions. Return a vector of (position, expected highlight name) /// pairs. -pub fn parse_highlight_test( +pub fn parse_position_comments( parser: &mut Parser, language: Language, source: &[u8], @@ -132,7 +133,7 @@ pub fn assert_expected_captures( language: Language, ) -> Result<()> { let contents = fs::read_to_string(path)?; - let pairs = parse_highlight_test(parser, language, contents.as_bytes())?; + let pairs = parse_position_comments(parser, language, contents.as_bytes())?; let per_position_index: HashMap = pairs.iter().map(|a| (a.position, &a.expected)).collect(); diff --git a/cli/src/test_highlight.rs b/cli/src/test_highlight.rs index da67f753..55a709aa 100644 --- a/cli/src/test_highlight.rs +++ b/cli/src/test_highlight.rs @@ -1,6 +1,6 @@ use super::error::Result; use crate::loader::Loader; -use crate::query_testing::{parse_highlight_test, Assertion}; +use crate::query_testing::{parse_position_comments, Assertion}; use ansi_term::Colour; use std::fs; use std::path::Path; @@ -91,7 +91,8 @@ pub fn test_highlight( // Highlight the file, and parse out all of the highlighting assertions. let highlight_names = loader.highlight_names(); let highlights = get_highlight_positions(loader, highlighter, highlight_config, source)?; - let assertions = parse_highlight_test(highlighter.parser(), highlight_config.language, source)?; + let assertions = + parse_position_comments(highlighter.parser(), highlight_config.language, source)?; // Iterate through all of the highlighting assertions, checking each one against the // actual highlights. diff --git a/cli/src/tests/test_highlight_test.rs b/cli/src/tests/test_highlight_test.rs index 6a857dd9..66920823 100644 --- a/cli/src/tests/test_highlight_test.rs +++ b/cli/src/tests/test_highlight_test.rs @@ -1,5 +1,6 @@ use super::helpers::fixtures::{get_highlight_config, get_language, test_loader}; -use crate::test_highlight::{get_highlight_positions, parse_highlight_test}; +use crate::query_testing::{parse_position_comments, Assertion}; +use crate::test_highlight::get_highlight_positions; use tree_sitter::{Parser, Point}; use tree_sitter_highlight::{Highlight, Highlighter}; @@ -25,13 +26,23 @@ fn test_highlight_test_with_basic_test() { ] .join("\n"); - let assertions = parse_highlight_test(&mut Parser::new(), language, source.as_bytes()).unwrap(); + let assertions = + parse_position_comments(&mut Parser::new(), language, source.as_bytes()).unwrap(); assert_eq!( assertions, &[ - (Point::new(0, 5), "function".to_string()), - (Point::new(0, 11), "keyword".to_string()), - (Point::new(3, 9), "variable.parameter".to_string()), + Assertion { + position: Point::new(0, 5), + expected: "function".to_string() + }, + Assertion { + position: Point::new(0, 11), + expected: "keyword".to_string() + }, + Assertion { + position: Point::new(3, 9), + expected: "variable.parameter".to_string() + }, ] ); From 0bd223f032e9bfc92dde4fc7bb83164b16bb039c Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 27 Oct 2020 13:11:57 -0400 Subject: [PATCH 14/30] Better naming for this regex. --- cli/src/query_testing.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/query_testing.rs b/cli/src/query_testing.rs index 9618d1f6..8c6af706 100644 --- a/cli/src/query_testing.rs +++ b/cli/src/query_testing.rs @@ -7,7 +7,7 @@ use std::fs; use tree_sitter::{Language, Parser, Point}; lazy_static! { - static ref HIGHLIGHT_NAME_REGEX: Regex = Regex::new("[\\w_\\-.]+").unwrap(); + static ref PROPERTY_NAME_REGEX: Regex = Regex::new("[\\w_\\-.]+").unwrap(); } #[derive(Debug, Eq, PartialEq)] @@ -76,7 +76,7 @@ pub fn parse_position_comments( // If the comment node contains an arrow and a highlight name, record the // highlight name and the position. if let (true, Some(mat)) = - (has_arrow, HIGHLIGHT_NAME_REGEX.find(&text[arrow_end..])) + (has_arrow, PROPERTY_NAME_REGEX.find(&text[arrow_end..])) { assertion_ranges.push((node.start_position(), node.end_position())); result.push(Assertion { From 521297fdfe2e466bd0c7e81ac687b15431ffb496 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 10 Nov 2020 16:19:17 -0500 Subject: [PATCH 15/30] remove testing file --- test/fixtures/queries/python.py | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 test/fixtures/queries/python.py diff --git a/test/fixtures/queries/python.py b/test/fixtures/queries/python.py deleted file mode 100644 index 01ec9ab0..00000000 --- a/test/fixtures/queries/python.py +++ /dev/null @@ -1,7 +0,0 @@ -def foo(): pass -# <- definition.function - -def bar(): -# <- definition.function - foo() - # <- reference.call From 50bccdf5dad00681cdcbd3d1275c40f6689ac7f1 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 10 Nov 2020 16:20:51 -0500 Subject: [PATCH 16/30] rename Assertion.expected to expected_capture_name --- cli/src/query_testing.rs | 10 ++++++---- cli/src/test_highlight.rs | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cli/src/query_testing.rs b/cli/src/query_testing.rs index 8c6af706..fe4ec8a1 100644 --- a/cli/src/query_testing.rs +++ b/cli/src/query_testing.rs @@ -19,7 +19,7 @@ pub struct CaptureInfo { #[derive(Debug, PartialEq, Eq)] pub struct Assertion { pub position: Point, - pub expected: String, + pub expected_capture_name: String, } /// Parse the given source code, finding all of the comments that contain @@ -81,7 +81,7 @@ pub fn parse_position_comments( assertion_ranges.push((node.start_position(), node.end_position())); result.push(Assertion { position: position, - expected: mat.as_str().to_string(), + expected_capture_name: mat.as_str().to_string(), }); } } @@ -135,8 +135,10 @@ pub fn assert_expected_captures( let contents = fs::read_to_string(path)?; let pairs = parse_position_comments(parser, language, contents.as_bytes())?; - let per_position_index: HashMap = - pairs.iter().map(|a| (a.position, &a.expected)).collect(); + let per_position_index: HashMap = pairs + .iter() + .map(|a| (a.position, &a.expected_capture_name)) + .collect(); for info in &infos { if !per_position_index.contains_key(&info.position) { diff --git a/cli/src/test_highlight.rs b/cli/src/test_highlight.rs index 55a709aa..2517ea3c 100644 --- a/cli/src/test_highlight.rs +++ b/cli/src/test_highlight.rs @@ -100,7 +100,7 @@ pub fn test_highlight( let mut actual_highlights = Vec::<&String>::new(); for Assertion { position, - expected: expected_highlight, + expected_capture_name: expected_highlight, } in &assertions { let mut passed = false; From 4604b40b72db3c7b52f3c6034ffd8527a5c5cbe4 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Tue, 10 Nov 2020 16:23:39 -0500 Subject: [PATCH 17/30] better name for capture regex --- cli/src/query_testing.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/query_testing.rs b/cli/src/query_testing.rs index fe4ec8a1..58feec42 100644 --- a/cli/src/query_testing.rs +++ b/cli/src/query_testing.rs @@ -7,7 +7,7 @@ use std::fs; use tree_sitter::{Language, Parser, Point}; lazy_static! { - static ref PROPERTY_NAME_REGEX: Regex = Regex::new("[\\w_\\-.]+").unwrap(); + static ref CAPTURE_NAME_REGEX: Regex = Regex::new("[\\w_\\-.]+").unwrap(); } #[derive(Debug, Eq, PartialEq)] @@ -76,7 +76,7 @@ pub fn parse_position_comments( // If the comment node contains an arrow and a highlight name, record the // highlight name and the position. if let (true, Some(mat)) = - (has_arrow, PROPERTY_NAME_REGEX.find(&text[arrow_end..])) + (has_arrow, CAPTURE_NAME_REGEX.find(&text[arrow_end..])) { assertion_ranges.push((node.start_position(), node.end_position())); result.push(Assertion { From f3d16f4770336c32b57d9547bd52f92fb7d6a257 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 23 Nov 2020 11:34:56 -0500 Subject: [PATCH 18/30] Fix tests. --- cli/src/test_highlight.rs | 68 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/cli/src/test_highlight.rs b/cli/src/test_highlight.rs index 2517ea3c..df870bf6 100644 --- a/cli/src/test_highlight.rs +++ b/cli/src/test_highlight.rs @@ -81,6 +81,72 @@ pub fn test_highlights(loader: &Loader, directory: &Path) -> Result<()> { Ok(()) } } +pub fn iterate_assertions( + assertions: &Vec, + highlights: &Vec<(Point, Point, Highlight)>, + highlight_names: &Vec, +) -> Result { + // Iterate through all of the highlighting assertions, checking each one against the + // actual highlights. + let mut i = 0; + let mut actual_highlights = Vec::<&String>::new(); + for Assertion { + position, + expected_capture_name: expected_highlight, + } in assertions + { + let mut passed = false; + actual_highlights.clear(); + + 'highlight_loop: loop { + // The assertions are ordered by position, so skip past all of the highlights that + // end at or before this assertion's position. + if let Some(highlight) = highlights.get(i) { + if highlight.1 <= *position { + i += 1; + continue; + } + + // Iterate through all of the highlights that start at or before this assertion's, + // position, looking for one that matches the assertion. + let mut j = i; + while let (false, Some(highlight)) = (passed, highlights.get(j)) { + if highlight.0 > *position { + break 'highlight_loop; + } + + // If the highlight matches the assertion, this test passes. Otherwise, + // add this highlight to the list of actual highlights that span the + // assertion's position, in order to generate an error message in the event + // of a failure. + let highlight_name = &highlight_names[(highlight.2).0]; + if *highlight_name == *expected_highlight { + passed = true; + break 'highlight_loop; + } else { + actual_highlights.push(highlight_name); + } + + j += 1; + } + } else { + break; + } + } + + if !passed { + return Err(Failure { + row: position.row, + column: position.column, + expected_highlight: expected_highlight.clone(), + actual_highlights: actual_highlights.into_iter().cloned().collect(), + } + .into()); + } + } + + Ok(assertions.len()) +} pub fn test_highlight( loader: &Loader, @@ -94,6 +160,8 @@ pub fn test_highlight( let assertions = parse_position_comments(highlighter.parser(), highlight_config.language, source)?; + iterate_assertions(&assertions, &highlights, &highlight_names)?; + // Iterate through all of the highlighting assertions, checking each one against the // actual highlights. let mut i = 0; From 0b4661e401f6430f56b1ad84467aa596fe7afad9 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 23 Nov 2020 11:41:16 -0500 Subject: [PATCH 19/30] Really fix the tests. --- cli/src/tests/test_highlight_test.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/src/tests/test_highlight_test.rs b/cli/src/tests/test_highlight_test.rs index 66920823..1a658281 100644 --- a/cli/src/tests/test_highlight_test.rs +++ b/cli/src/tests/test_highlight_test.rs @@ -33,15 +33,15 @@ fn test_highlight_test_with_basic_test() { &[ Assertion { position: Point::new(0, 5), - expected: "function".to_string() + expected_capture_name: "function".to_string() }, Assertion { position: Point::new(0, 11), - expected: "keyword".to_string() + expected_capture_name: "keyword".to_string() }, Assertion { position: Point::new(3, 9), - expected: "variable.parameter".to_string() + expected_capture_name: "variable.parameter".to_string() }, ] ); From 6764b803a0b93425586615decf6e343cdf1b31b0 Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 23 Nov 2020 11:58:07 -0500 Subject: [PATCH 20/30] Allow overlap in specs. --- cli/src/query.rs | 2 ++ cli/src/query_testing.rs | 30 ++++++++++++++---------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cli/src/query.rs b/cli/src/query.rs index 9c524877..bf67edf6 100644 --- a/cli/src/query.rs +++ b/cli/src/query.rs @@ -58,6 +58,7 @@ pub fn query_files_at_paths( results.push(query_testing::CaptureInfo { name: capture_name.to_string(), position: capture.node.start_position(), + terminus: capture.node.end_position(), }); } } else { @@ -85,6 +86,7 @@ pub fn query_files_at_paths( results.push(query_testing::CaptureInfo { name: capture_name.to_string(), position: capture.node.start_position(), + terminus: capture.node.end_position(), }); } } diff --git a/cli/src/query_testing.rs b/cli/src/query_testing.rs index 58feec42..96ccf6b2 100644 --- a/cli/src/query_testing.rs +++ b/cli/src/query_testing.rs @@ -2,7 +2,6 @@ use crate::error; use crate::error::Result; use lazy_static::lazy_static; use regex::Regex; -use std::collections::hash_map::HashMap; use std::fs; use tree_sitter::{Language, Parser, Point}; @@ -14,6 +13,7 @@ lazy_static! { pub struct CaptureInfo { pub name: String, pub position: Point, + pub terminus: Point, } #[derive(Debug, PartialEq, Eq)] @@ -134,22 +134,20 @@ pub fn assert_expected_captures( ) -> Result<()> { let contents = fs::read_to_string(path)?; let pairs = parse_position_comments(parser, language, contents.as_bytes())?; - - let per_position_index: HashMap = pairs - .iter() - .map(|a| (a.position, &a.expected_capture_name)) - .collect(); - for info in &infos { - if !per_position_index.contains_key(&info.position) { - continue; - } - let found = per_position_index.get(&info.position).unwrap(); - if **found != info.name && info.name != "name" { - Err(error::Error::new(format!( - "Assertion failed: at {}, found {}, expected {}", - info.position, found, info.name - )))? + let found = pairs.iter().find(|p| { + p.position.row == info.position.row + && p.position >= info.position + && p.position < info.terminus + }); + + if let Some(found) = found { + if found.expected_capture_name != info.name && info.name != "name" { + Err(error::Error::new(format!( + "Assertion failed: at {}, found {}, expected {}", + info.position, found.expected_capture_name, info.name + )))? + } } } Ok(()) From e1da6e554bf9235f613f44baeb5496663a6c12df Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 23 Nov 2020 12:01:08 -0500 Subject: [PATCH 21/30] Remove fanciful nomenclature. --- cli/src/query.rs | 8 ++++---- cli/src/query_testing.rs | 10 ++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cli/src/query.rs b/cli/src/query.rs index bf67edf6..485fdb82 100644 --- a/cli/src/query.rs +++ b/cli/src/query.rs @@ -57,8 +57,8 @@ pub fn query_files_at_paths( )?; results.push(query_testing::CaptureInfo { name: capture_name.to_string(), - position: capture.node.start_position(), - terminus: capture.node.end_position(), + start: capture.node.start_position(), + end: capture.node.end_position(), }); } } else { @@ -85,8 +85,8 @@ pub fn query_files_at_paths( } results.push(query_testing::CaptureInfo { name: capture_name.to_string(), - position: capture.node.start_position(), - terminus: capture.node.end_position(), + start: capture.node.start_position(), + end: capture.node.end_position(), }); } } diff --git a/cli/src/query_testing.rs b/cli/src/query_testing.rs index 96ccf6b2..2a9a8c2d 100644 --- a/cli/src/query_testing.rs +++ b/cli/src/query_testing.rs @@ -12,8 +12,8 @@ lazy_static! { #[derive(Debug, Eq, PartialEq)] pub struct CaptureInfo { pub name: String, - pub position: Point, - pub terminus: Point, + pub start: Point, + pub end: Point, } #[derive(Debug, PartialEq, Eq)] @@ -136,16 +136,14 @@ pub fn assert_expected_captures( let pairs = parse_position_comments(parser, language, contents.as_bytes())?; for info in &infos { let found = pairs.iter().find(|p| { - p.position.row == info.position.row - && p.position >= info.position - && p.position < info.terminus + p.position.row == info.start.row && p.position >= info.start && p.position < info.end }); if let Some(found) = found { if found.expected_capture_name != info.name && info.name != "name" { Err(error::Error::new(format!( "Assertion failed: at {}, found {}, expected {}", - info.position, found.expected_capture_name, info.name + info.start, found.expected_capture_name, info.name )))? } } From cc8f978b3b4007975f7a6f2a9a43dd5bc5f8ec4b Mon Sep 17 00:00:00 2001 From: Patrick Thomson Date: Mon, 23 Nov 2020 12:05:32 -0500 Subject: [PATCH 22/30] inline this lambda --- cli/src/query_testing.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cli/src/query_testing.rs b/cli/src/query_testing.rs index 2a9a8c2d..ef02ec69 100644 --- a/cli/src/query_testing.rs +++ b/cli/src/query_testing.rs @@ -135,11 +135,9 @@ pub fn assert_expected_captures( let contents = fs::read_to_string(path)?; let pairs = parse_position_comments(parser, language, contents.as_bytes())?; for info in &infos { - let found = pairs.iter().find(|p| { + if let Some(found) = pairs.iter().find(|p| { p.position.row == info.start.row && p.position >= info.start && p.position < info.end - }); - - if let Some(found) = found { + }) { if found.expected_capture_name != info.name && info.name != "name" { Err(error::Error::new(format!( "Assertion failed: at {}, found {}, expected {}", From 7ef73b2e085acd8f45bbf998fea84c756a05674f Mon Sep 17 00:00:00 2001 From: Joel Spadin Date: Thu, 26 Nov 2020 14:43:27 -0600 Subject: [PATCH 23/30] web binding: fix equals() Node.equals() used to always return true. Now it works. Also added unit tests for it. --- lib/binding_web/binding.js | 6 +----- lib/binding_web/test/node-test.js | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/binding_web/binding.js b/lib/binding_web/binding.js index f731e8f8..15b07116 100644 --- a/lib/binding_web/binding.js +++ b/lib/binding_web/binding.js @@ -258,11 +258,7 @@ class Node { } equals(other) { - if (this === other) return true; - for (let i = 0; i < 5; i++) { - if (this[i] !== other[i]) return false; - } - return true; + return this.id === other.id; } child(index) { diff --git a/lib/binding_web/test/node-test.js b/lib/binding_web/test/node-test.js index 933ff38f..6bbcafb0 100644 --- a/lib/binding_web/test/node-test.js +++ b/lib/binding_web/test/node-test.js @@ -388,4 +388,24 @@ describe("Node", () => { assert.throws(() => number.closest({a: 1}), /Argument must be a string or array of strings/) }); }); + + describe('.equals(other)', () => { + it('returns true if the nodes are the same', () => { + tree = parser.parse('1 + 2'); + + const sumNode = tree.rootNode.firstChild.firstChild; + const node1 = sumNode.firstChild; + const node2 = sumNode.firstChild; + assert(node1.equals(node2)); + }); + + it('returns false if the nodes are not the same', () => { + tree = parser.parse('1 + 2'); + + const sumNode = tree.rootNode.firstChild.firstChild; + const node1 = sumNode.firstChild; + const node2 = node1.nextSibling; + assert(!node1.equals(node2)); + }); + }); }); From a2d6048226ceb1b09a7e4cf330d75ced0d3f27a3 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 14:28:27 -0800 Subject: [PATCH 24/30] Get the wasm build working w/ latest emscripten --- lib/binding_web/binding.c | 27 +++++++++++++++++---------- lib/binding_web/binding.js | 2 +- lib/binding_web/imports.js | 2 +- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/lib/binding_web/binding.c b/lib/binding_web/binding.c index eb463b26..9180f405 100644 --- a/lib/binding_web/binding.c +++ b/lib/binding_web/binding.c @@ -115,18 +115,10 @@ extern void tree_sitter_parse_callback( ); extern void tree_sitter_log_callback( - void *payload, - TSLogType log_type, + bool is_lex_message, const char *message ); -void ts_parser_new_wasm() { - TSParser *parser = ts_parser_new(); - char *input_buffer = calloc(INPUT_BUFFER_SIZE, sizeof(char)); - TRANSFER_BUFFER[0] = parser; - TRANSFER_BUFFER[1] = input_buffer; -} - static const char *call_parse_callback( void *payload, uint32_t byte, @@ -148,8 +140,23 @@ static const char *call_parse_callback( return buffer; } +static void call_log_callback( + void *payload, + TSLogType log_type, + const char *message +) { + tree_sitter_log_callback(log_type == TSLogTypeLex, message); +} + +void ts_parser_new_wasm() { + TSParser *parser = ts_parser_new(); + char *input_buffer = calloc(INPUT_BUFFER_SIZE, sizeof(char)); + TRANSFER_BUFFER[0] = parser; + TRANSFER_BUFFER[1] = input_buffer; +} + void ts_parser_enable_logger_wasm(TSParser *self, bool should_log) { - TSLogger logger = {self, should_log ? tree_sitter_log_callback : NULL}; + TSLogger logger = {self, should_log ? call_log_callback : NULL}; ts_parser_set_logger(self, logger); } diff --git a/lib/binding_web/binding.js b/lib/binding_web/binding.js index 15b07116..95bfa828 100644 --- a/lib/binding_web/binding.js +++ b/lib/binding_web/binding.js @@ -880,7 +880,7 @@ class Language { } return bytes - .then(bytes => loadWebAssemblyModule(bytes, {loadAsync: true})) + .then(bytes => loadSideModule(bytes, {loadAsync: true})) .then(mod => { const symbolNames = Object.keys(mod) const functionName = symbolNames.find(key => diff --git a/lib/binding_web/imports.js b/lib/binding_web/imports.js index ea34926f..a76c42ac 100644 --- a/lib/binding_web/imports.js +++ b/lib/binding_web/imports.js @@ -16,7 +16,7 @@ mergeInto(LibraryManager.library, { } }, - tree_sitter_log_callback: function(_payload, isLexMessage, messageAddress) { + tree_sitter_log_callback: function(isLexMessage, messageAddress) { if (currentLogCallback) { const message = UTF8ToString(messageAddress); currentLogCallback(message, isLexMessage !== 0); From b118e7d7505d5f5621b7cf269a1b155e6f20588e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 15:28:21 -0800 Subject: [PATCH 25/30] Make binding.js syntactically valid Put the end of the surrounding closure into a separate file, suffix.js. --- lib/binding_web/binding.js | 5 ----- lib/binding_web/suffix.js | 2 ++ script/build-wasm | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) create mode 100644 lib/binding_web/suffix.js diff --git a/lib/binding_web/binding.js b/lib/binding_web/binding.js index 95bfa828..b060715f 100644 --- a/lib/binding_web/binding.js +++ b/lib/binding_web/binding.js @@ -7,7 +7,6 @@ const SIZE_OF_RANGE = 2 * SIZE_OF_INT + 2 * SIZE_OF_POINT; const ZERO_POINT = {row: 0, column: 0}; const QUERY_WORD_REGEX = /[\w-.]*/g; -const PREDICATE_STEP_TYPE_DONE = 0; const PREDICATE_STEP_TYPE_CAPTURE = 1; const PREDICATE_STEP_TYPE_STRING = 2; @@ -1140,7 +1139,3 @@ function marshalEdit(edit) { } Parser.Language = Language; - -return Parser; - -})); diff --git a/lib/binding_web/suffix.js b/lib/binding_web/suffix.js new file mode 100644 index 00000000..0e9fe021 --- /dev/null +++ b/lib/binding_web/suffix.js @@ -0,0 +1,2 @@ +return Parser; +})); diff --git a/script/build-wasm b/script/build-wasm index 63ec4fe0..b139f6c2 100755 --- a/script/build-wasm +++ b/script/build-wasm @@ -95,6 +95,7 @@ $emcc \ --js-library ${web_dir}/imports.js \ --pre-js ${web_dir}/prefix.js \ --post-js ${web_dir}/binding.js \ + --post-js ${web_dir}/suffix.js \ lib/src/lib.c \ ${web_dir}/binding.c \ -o target/scratch/tree-sitter.js From 751ffd2ee13ef7b29de60585fec3a52dab1f5b4e Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 16:25:01 -0800 Subject: [PATCH 26/30] Use new emscripten when building with docker --- cli/src/wasm.rs | 2 +- script/build-wasm | 10 +++++----- script/generate-fixtures-wasm | 8 +++++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/cli/src/wasm.rs b/cli/src/wasm.rs index 47cea90a..8bbcfbdf 100644 --- a/cli/src/wasm.rs +++ b/cli/src/wasm.rs @@ -57,7 +57,7 @@ pub fn compile_language_to_wasm(language_dir: &Path, force_docker: bool) -> Resu } // Run `emcc` in a container using the `emscripten-slim` image - command.args(&["trzeci/emscripten-slim", "emcc"]); + command.args(&["emscripten/emsdk", "emcc"]); } else { return Error::err( "You must have either emcc or docker on your PATH to run this command".to_string(), diff --git a/script/build-wasm b/script/build-wasm index b139f6c2..75c6a7d1 100755 --- a/script/build-wasm +++ b/script/build-wasm @@ -64,11 +64,11 @@ emcc= if which emcc > /dev/null && [[ "$force_docker" == "0" ]]; then emcc=emcc elif which docker > /dev/null; then - emcc="docker run \ - --rm \ - -v $(pwd):/src:Z \ - -u $(id -u) \ - trzeci/emscripten-slim \ + emcc="docker run \ + --rm \ + -v $(pwd):/src:Z \ + -u $(id -u) \ + emscripten/emsdk \ emcc" else echo 'You must have either `docker` or `emcc` on your PATH to run this script' diff --git a/script/generate-fixtures-wasm b/script/generate-fixtures-wasm index a987e31a..9d44b58c 100755 --- a/script/generate-fixtures-wasm +++ b/script/generate-fixtures-wasm @@ -4,6 +4,12 @@ set -e cargo build --release +build_wasm_args= +if [[ $1 == "--docker" ]]; then + build_wasm_args="--docker" + shift +fi + filter_grammar_name=$1 root_dir=$PWD @@ -20,7 +26,7 @@ while read -r grammar_file; do fi echo "Compiling ${grammar_name} parser to wasm" - "$tree_sitter" build-wasm $grammar_dir + "$tree_sitter" build-wasm $build_wasm_args $grammar_dir done <<< "$grammar_files" mv tree-sitter-*.wasm target/release/ From 18980b7b99757e4ffa262a49501ae07ad7a8d986 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 16:25:17 -0800 Subject: [PATCH 27/30] wasm: Avoid registering uncaught exception/rejection handlers --- script/build-wasm | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/script/build-wasm b/script/build-wasm index 75c6a7d1..1b5e48ec 100755 --- a/script/build-wasm +++ b/script/build-wasm @@ -30,7 +30,6 @@ EOF set -e web_dir=lib/binding_web -exports=$(cat ${web_dir}/exports.json) emscripten_flags="-O3" minify_js=1 force_docker=0 @@ -79,25 +78,27 @@ mkdir -p target/scratch # Use emscripten to generate `tree-sitter.js` and `tree-sitter.wasm` # in the `target/scratch` directory -$emcc \ - -s WASM=1 \ - -s TOTAL_MEMORY=33554432 \ - -s ALLOW_MEMORY_GROWTH=1 \ - -s MAIN_MODULE=2 \ - -s NO_FILESYSTEM=1 \ - -s "EXPORTED_FUNCTIONS=${exports}" \ - $emscripten_flags \ - -std=c99 \ - -D 'fprintf(...)=' \ - -D NDEBUG= \ - -I lib/src \ - -I lib/include \ - --js-library ${web_dir}/imports.js \ - --pre-js ${web_dir}/prefix.js \ - --post-js ${web_dir}/binding.js \ - --post-js ${web_dir}/suffix.js \ - lib/src/lib.c \ - ${web_dir}/binding.c \ +$emcc \ + -s WASM=1 \ + -s TOTAL_MEMORY=33554432 \ + -s ALLOW_MEMORY_GROWTH=1 \ + -s MAIN_MODULE=2 \ + -s NO_FILESYSTEM=1 \ + -s NODEJS_CATCH_EXIT=0 \ + -s NODEJS_CATCH_REJECTION=0 \ + -s EXPORTED_FUNCTIONS=@${web_dir}/exports.json \ + $emscripten_flags \ + -std=c99 \ + -D 'fprintf(...)=' \ + -D NDEBUG= \ + -I lib/src \ + -I lib/include \ + --js-library ${web_dir}/imports.js \ + --pre-js ${web_dir}/prefix.js \ + --post-js ${web_dir}/binding.js \ + --post-js ${web_dir}/suffix.js \ + lib/src/lib.c \ + ${web_dir}/binding.c \ -o target/scratch/tree-sitter.js # Use terser to write a minified version of `tree-sitter.js` into From 2699c01ab1c588da81f9d86c97488876c0a0b6c4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 16:45:45 -0800 Subject: [PATCH 28/30] Use latest emscripten on CI --- script/fetch-emscripten | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/fetch-emscripten b/script/fetch-emscripten index d16c857e..c1b072ad 100755 --- a/script/fetch-emscripten +++ b/script/fetch-emscripten @@ -2,7 +2,7 @@ set -e -EMSCRIPTEN_VERSION=1.39.15 +EMSCRIPTEN_VERSION=2.0.9 mkdir -p target EMSDK_DIR="./target/emsdk" From 591a2c62495ca81109e061846a7a6eb8e66ecfac Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 16:46:02 -0800 Subject: [PATCH 29/30] Remove web binding paths from travis config --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7205ae03..79d84d13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,8 +58,6 @@ deploy: file_glob: true file: - "tree-sitter-*.gz" - - "target/release/tree-sitter.js" - - "target/release/tree-sitter.wasm" draft: true overwrite: true skip_cleanup: true From d3f30e298b9caed7058c95e9535f5a33a4be6648 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 30 Nov 2020 20:36:36 -0800 Subject: [PATCH 30/30] Use node 12 on travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 79d84d13..282ba02d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,8 @@ matrix: before_install: # Install node - - nvm install 10 - - nvm use 10 + - nvm install 12 + - nvm use 12 # Download emscripten and create a shorthand for adding it to the PATH. # Don't add it to the path globally because it overrides the default