traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lib: Include fields in ts_node_string output

Browse source 
This allows you to assert about fields in tests. But if your test 
s-expression does *not* include fields, the fields will be stripped from 
the regexp before comparison.
This commit is contained in:
Max Brunsfeld 2019-02-13 09:47:21 -08:00
parent 9f608435ee
commit 65d1ce8593
6 changed files with 113 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

16
cli/src/test.rs
View file

@ -22,6 +22,7 @@ lazy_static! {
.build()
.unwrap();
static ref WHITESPACE_REGEX: Regex = Regex::new(r"\s+").unwrap();
static ref SEXP_FIELD_REGEX: Regex = Regex::new(r" \w+: \(").unwrap();
}
#[derive(Debug, PartialEq, Eq)]
@ -34,6 +35,7 @@ pub enum TestEntry {
name: String,
input: Vec<u8>,
output: String,
has_fields: bool,
},
}
@ -135,6 +137,7 @@ fn run_tests(
name,
input,
output,
has_fields,
} => {
if let Some(filter) = filter {
if !name.contains(filter) {
@ -142,7 +145,10 @@ fn run_tests(
}
}
let tree = parser.parse(&input, None).unwrap();
let actual = tree.root_node().to_sexp();
let mut actual = tree.root_node().to_sexp();
if !has_fields {
actual = strip_sexp_fields(actual);
}
for _ in 0..indent_level {
print!(" ");
}
@ -186,6 +192,10 @@ pub fn parse_tests(path: &Path) -> io::Result<TestEntry> {
}
}
pub fn strip_sexp_fields(sexp: String) -> String {
SEXP_FIELD_REGEX.replace_all(&sexp, " (").to_string()
}
fn parse_test_content(name: String, content: String) -> TestEntry {
let mut children = Vec::new();
let bytes = content.as_bytes();
@ -209,10 +219,12 @@ fn parse_test_content(name: String, content: String) -> TestEntry {
let input = bytes[previous_header_end..divider_start].to_vec();
let output = WHITESPACE_REGEX.replace_all(output.trim(), " ").to_string();
let output = output.replace(" )", ")");
let has_fields = SEXP_FIELD_REGEX.is_match(&output);
children.push(TestEntry::Example {
name: previous_name,
input,
output,
has_fields,
});
}
}
@ -265,11 +277,13 @@ d
name: "The first test".to_string(),
input: "\na b c\n".as_bytes().to_vec(),
output: "(a (b c))".to_string(),
has_fields: false,
},
TestEntry::Example {
name: "The second test".to_string(),
input: "d".as_bytes().to_vec(),
output: "(d)".to_string(),
has_fields: false,
},
]
}

30
cli/src/tests/corpus_test.rs
View file

@ -4,7 +4,7 @@ use super::helpers::fixtures::{fixtures_dir, get_language, get_test_language};
use super::helpers::random::Rand;
use super::helpers::scope_sequence::ScopeSequence;
use crate::generate;
use crate::test::{parse_tests, print_diff, print_diff_key, TestEntry};
use crate::test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields, TestEntry};
use crate::util;
use lazy_static::lazy_static;
use std::{env, fs, time, usize};
@ -67,7 +67,7 @@ fn test_real_language_corpus_files() {
eprintln!("language: {:?}", language_name);
}
for (example_name, input, expected_output) in tests {
for (example_name, input, expected_output, has_fields) in tests {
eprintln!(" example: {:?}", example_name);
if TRIAL_FILTER.map_or(true, |t| t == 0) {
@ -76,7 +76,10 @@ fn test_real_language_corpus_files() {
let mut parser = get_parser(&mut log_session, "log.html");
parser.set_language(language).unwrap();
let tree = parser.parse(&input, None).unwrap();
let actual_output = tree.root_node().to_sexp();
let mut actual_output = tree.root_node().to_sexp();
if !has_fields {
actual_output = strip_sexp_fields(actual_output);
}
drop(tree);
drop(parser);
if actual_output != expected_output {
@ -144,7 +147,11 @@ fn test_real_language_corpus_files() {
let tree3 = parser.parse(&input, Some(&tree2)).unwrap();
// Verify that the final tree matches the expectation from the corpus.
let actual_output = tree3.root_node().to_sexp();
let mut actual_output = tree3.root_node().to_sexp();
if !has_fields {
actual_output = strip_sexp_fields(actual_output);
}
if actual_output != expected_output {
println!(
"Incorrect parse for {} - {} - trial {}",
@ -241,7 +248,7 @@ fn test_feature_corpus_files() {
eprintln!("test language: {:?}", language_name);
}
for (name, input, expected_output) in tests {
for (name, input, expected_output, has_fields) in tests {
eprintln!(" example: {:?}", name);
allocations::start_recording();
@ -249,7 +256,11 @@ fn test_feature_corpus_files() {
let mut parser = get_parser(&mut log_session, "log.html");
parser.set_language(language).unwrap();
let tree = parser.parse(&input, None).unwrap();
let actual_output = tree.root_node().to_sexp();
let mut actual_output = tree.root_node().to_sexp();
if !has_fields {
actual_output = strip_sexp_fields(actual_output);
}
drop(tree);
drop(parser);
if actual_output != expected_output {
@ -348,13 +359,14 @@ fn get_parser(session: &mut Option<util::LogSession>, log_filename: &str) -> Par
parser
}
fn flatten_tests(test: TestEntry) -> Vec<(String, Vec<u8>, String)> {
fn helper(test: TestEntry, prefix: &str, result: &mut Vec<(String, Vec<u8>, String)>) {
fn flatten_tests(test: TestEntry) -> Vec<(String, Vec<u8>, String, bool)> {
fn helper(test: TestEntry, prefix: &str, result: &mut Vec<(String, Vec<u8>, String, bool)>) {
match test {
TestEntry::Example {
mut name,
input,
output,
has_fields,
} => {
if !prefix.is_empty() {
name.insert_str(0, " - ");
@ -365,7 +377,7 @@ fn flatten_tests(test: TestEntry) -> Vec<(String, Vec<u8>, String)> {
return;
}
}
result.push((name, input, output));
result.push((name, input, output, has_fields));
}
TestEntry::Group { mut name, children } => {
if !prefix.is_empty() {

2
cli/src/tests/parser_test.rs
View file

@ -721,7 +721,7 @@ fn test_parsing_with_included_ranges_and_missing_tokens() {
let root = tree.root_node();
assert_eq!(
root.to_sexp(),
"(program (A (MISSING)) (b) (c) (A (MISSING)) (b) (c))"
"(program (A (MISSING a)) (b) (c) (A (MISSING a)) (b) (c))"
);
assert_eq!(root.start_byte(), 2);
assert_eq!(root.child(3).unwrap().start_byte(), 4);