traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(cli): support snapshot testing with --update flag

Browse source 
This PR adds an `--update` flag to the `tree-sitter test` command, which adds the ability to replace the _expected_ output in the corpus.txt with the _actual_ output produced by the parser, that is, we can now simply use this `--update` flag to write all the corresponding parser output back to the corpus.txt, and we just need to check the output without typing its actual sexp.

- use the same output format as `tree-sitter parse`, except there won't be any position information printed.
- the corpus.txt won't be touched if there's no difference between the _expected_ output and the _actual_ output in that file.
- if there're differences between _expected_ and _actual_, only the test case that is different will be replaced, the rest test cases will stay as-is. (All the delimiters `===`/`---` will be normalized as 80-column long, though.)
- this flag also works with `--filter` flag.
This commit is contained in:
Ika 2019-09-01 23:52:39 +08:00
parent 4b0489e2f3
commit 1b033fdfa4
6 changed files with 274 additions and 105 deletions
Download patch file Download diff file Expand all files Collapse all files

1
cli/src/lib.rs
View file

@ -9,6 +9,7 @@ pub mod test;
pub mod util;
pub mod wasm;
pub mod web_ui;
pub mod print;
#[cfg(test)]
mod tests;

6
cli/src/main.rs
View file

@ -83,7 +83,8 @@ fn run() -> error::Result<()> {
.takes_value(true),
)
.arg(Arg::with_name("debug").long("debug").short("d"))
.arg(Arg::with_name("debug-graph").long("debug-graph").short("D")),
.arg(Arg::with_name("debug-graph").long("debug-graph").short("D"))
.arg(Arg::with_name("update").long("update").short("u")),
)
.subcommand(
SubCommand::with_name("highlight")
@ -150,9 +151,10 @@ fn run() -> error::Result<()> {
let debug = matches.is_present("debug");
let debug_graph = matches.is_present("debug-graph");
let filter = matches.value_of("filter");
let update = matches.is_present("update");
let corpus_path = current_dir.join("corpus");
if let Some(language) = loader.languages_at_path(&current_dir)?.first() {
test::run_tests_at_path(*language, &corpus_path, debug, debug_graph, filter)?;
test::run_tests_at_path(*language, &corpus_path, debug, debug_graph, filter, update)?;
} else {
eprintln!("No language found");
}

53
cli/src/parse.rs
View file

@ -1,4 +1,5 @@
use super::error::{Error, Result};
use super::print::print_tree;
use super::util;
use std::io::{self, Write};
use std::path::Path;
@ -81,57 +82,7 @@ pub fn parse_file_at_path(
let mut cursor = tree.walk();
if !quiet {
let mut needs_newline = false;
let mut indent_level = 0;
let mut did_visit_children = false;
loop {
let node = cursor.node();
let is_named = node.is_named();
if did_visit_children {
if is_named {
stdout.write(b")")?;
needs_newline = true;
}
if cursor.goto_next_sibling() {
did_visit_children = false;
} else if cursor.goto_parent() {
did_visit_children = true;
indent_level -= 1;
} else {
break;
}
} else {
if is_named {
if needs_newline {
stdout.write(b"\n")?;
}
for _ in 0..indent_level {
stdout.write(b" ")?;
}
let start = node.start_position();
let end = node.end_position();
if let Some(field_name) = cursor.field_name() {
write!(&mut stdout, "{}: ", field_name)?;
}
write!(
&mut stdout,
"({} [{}, {}] - [{}, {}]",
node.kind(),
start.row,
start.column,
end.row,
end.column
)?;
needs_newline = true;
}
if cursor.goto_first_child() {
did_visit_children = false;
indent_level += 1;
} else {
did_visit_children = true;
}
}
}
print_tree(&mut stdout, &mut cursor, true)?;
cursor.reset(tree.root_node());
println!("");
}

60
cli/src/print.rs Normal file
View file

@ -0,0 +1,60 @@
use super::error::{Result};
use std::io::{Write};
use tree_sitter::{TreeCursor};
pub fn print_tree(output: &mut Write, cursor: &mut TreeCursor, prints_position: bool) -> Result<()> {
let mut needs_newline = false;
let mut indent_level = 0;
let mut did_visit_children = false;
loop {
let node = cursor.node();
let is_named = node.is_named();
if did_visit_children {
if is_named {
output.write(b")")?;
needs_newline = true;
}
if cursor.goto_next_sibling() {
did_visit_children = false;
} else if cursor.goto_parent() {
did_visit_children = true;
indent_level -= 1;
} else {
break;
}
} else {
if is_named {
if needs_newline {
output.write(b"\n")?;
}
for _ in 0..indent_level {
output.write(b" ")?;
}
if let Some(field_name) = cursor.field_name() {
write!(output, "{}: ", field_name)?;
}
write!(output, "({}", node.kind())?;
if prints_position {
let start = node.start_position();
let end = node.end_position();
write!(
output,
" [{}, {}] - [{}, {}]",
start.row,
start.column,
end.row,
end.column
)?;
}
needs_newline = true;
}
if cursor.goto_first_child() {
did_visit_children = false;
indent_level += 1;
} else {
did_visit_children = true;
}
}
}
return Ok(());
}

251
cli/src/test.rs
View file

@ -1,4 +1,5 @@
use super::error::{Error, Result};
use super::print::print_tree;
use super::util;
use ansi_term::Colour;
use difference::{Changeset, Difference};
@ -8,7 +9,7 @@ use regex::Regex;
use std::char;
use std::fs;
use std::io::{self, Write};
use std::path::Path;
use std::path::{Path, PathBuf};
use std::str;
use tree_sitter::{Language, LogType, Parser};
@ -30,6 +31,7 @@ pub enum TestEntry {
Group {
name: String,
children: Vec<TestEntry>,
file_path: Option<PathBuf>,
},
Example {
name: String,
@ -44,6 +46,7 @@ impl Default for TestEntry {
TestEntry::Group {
name: String::new(),
children: Vec::new(),
file_path: None,
}
}
}
@ -54,43 +57,52 @@ pub fn run_tests_at_path(
debug: bool,
debug_graph: bool,
filter: Option<&str>,
update: bool,
) -> Result<()> {
let test_entry = parse_tests(path)?;
let test_entry = parse_tests(path, false)?;
let mut _log_session = None;
let mut parser = Parser::new();
parser.set_language(language).map_err(|e| e.to_string())?;
if debug_graph {
_log_session = Some(util::log_graphs(&mut parser, "log.html")?);
} else if debug {
parser.set_logger(Some(Box::new(|log_type, message| {
if log_type == LogType::Lex {
io::stderr().write(b" ").unwrap();
}
write!(&mut io::stderr(), "{}\n", message).unwrap();
})));
}
let mut failures = Vec::new();
if let TestEntry::Group { children, .. } = test_entry {
for child in children {
run_tests(&mut parser, child, filter, 0, &mut failures)?;
if !update {
if debug_graph {
_log_session = Some(util::log_graphs(&mut parser, "log.html")?);
} else if debug {
parser.set_logger(Some(Box::new(|log_type, message| {
if log_type == LogType::Lex {
io::stderr().write(b" ").unwrap();
}
write!(&mut io::stderr(), "{}\n", message).unwrap();
})));
}
}
if failures.len() > 0 {
let mut diffs = Vec::new();
let mut update_entries = Vec::new();
run_tests(&mut parser, test_entry, filter, update, &mut update_entries, -1, &mut diffs)?;
if diffs.len() > 0 {
println!("");
if failures.len() == 1 {
println!("1 failure:")
let diff_name = if update { "update" } else { "failure" };
if diffs.len() == 1 {
println!("1 {}:", diff_name)
} else {
println!("{} failures:", failures.len())
println!("{} {}s:", diffs.len(), diff_name)
}
print_diff_key();
for (i, (name, actual, expected)) in failures.iter().enumerate() {
if update {
print_update_diff_key();
} else {
print_diff_key();
}
for (i, (name, parsed, provided)) in diffs.iter().enumerate() {
println!("\n {}. {}:", i + 1, name);
print_diff(actual, expected);
if update {
print_update_diff(provided, parsed);
} else {
print_diff(parsed, provided);
}
}
Error::err(String::new())
} else {
@ -99,14 +111,40 @@ pub fn run_tests_at_path(
}
pub fn print_diff_key() {
print_diff_key_with_colors("actual", "expected", Colour::Red, Colour::Green);
}
fn print_update_diff_key() {
print_diff_key_with_colors("original", "updated", Colour::Yellow, Colour::Green);
}
fn print_diff_key_with_colors(
actual_name: &str,
expected_name: &str,
actual_color: Colour,
expected_color: Colour,
) {
println!(
"\n{} / {}",
Colour::Green.paint("expected"),
Colour::Red.paint("actual")
expected_color.paint(expected_name),
actual_color.paint(actual_name)
);
}
pub fn print_diff(actual: &String, expected: &String) {
print_diff_with_colors(actual, expected, Colour::Red, Colour::Green);
}
fn print_update_diff(actual: &String, expected: &String) {
print_diff_with_colors(actual, expected, Colour::Yellow, Colour::Green);
}
fn print_diff_with_colors(
actual: &String,
expected: &String,
actual_color: Colour,
expected_color: Colour,
) {
let changeset = Changeset::new(actual, expected, " ");
print!(" ");
for diff in &changeset.diffs {
@ -115,10 +153,10 @@ pub fn print_diff(actual: &String, expected: &String) {
print!("{}{}", part, changeset.split);
}
Difference::Add(part) => {
print!("{}{}", Colour::Green.paint(part), changeset.split);
print!("{}{}", expected_color.paint(part), changeset.split);
}
Difference::Rem(part) => {
print!("{}{}", Colour::Red.paint(part), changeset.split);
print!("{}{}", actual_color.paint(part), changeset.split);
}
}
}
@ -129,8 +167,10 @@ fn run_tests(
parser: &mut Parser,
test_entry: TestEntry,
filter: Option<&str>,
update: bool,
update_entries: &mut Vec<(String, String, String)>,
mut indent_level: i32,
failures: &mut Vec<(String, String, String)>,
diffs: &mut Vec<(String, String, String)>,
) -> Result<()> {
match test_entry {
TestEntry::Example {
@ -141,39 +181,97 @@ fn run_tests(
} => {
if let Some(filter) = filter {
if !name.contains(filter) {
if update {
let input = String::from_utf8(input).unwrap();
update_entries.push((name, input, output));
}
return Ok(());
}
}
let tree = parser.parse(&input, None).unwrap();
let mut actual = tree.root_node().to_sexp();
let mut parsed = tree.root_node().to_sexp();
if !has_fields {
actual = strip_sexp_fields(actual);
parsed = strip_sexp_fields(parsed);
}
for _ in 0..indent_level {
print!(" ");
}
if actual == output {
let provided = normalize_sexp(&output);
if parsed == provided {
println!("✓ {}", Colour::Green.paint(&name));
if update {
let input = String::from_utf8(input).unwrap();
update_entries.push((name, input, output));
}
} else {
println!("✗ {}", Colour::Red.paint(&name));
failures.push((name, actual, output));
if update {
let input = String::from_utf8(input).unwrap();
let mut fixed_output = Vec::new();
let mut cursor = tree.walk();
print_tree(&mut fixed_output, &mut cursor, false)?;
let fixed_output = String::from_utf8(fixed_output).unwrap();
update_entries.push((name.clone(), input, fixed_output));
println!("✓ {}", Colour::Yellow.paint(&name));
} else {
println!("✗ {}", Colour::Red.paint(&name));
}
diffs.push((name, parsed, provided));
}
}
TestEntry::Group { name, children } => {
for _ in 0..indent_level {
print!(" ");
TestEntry::Group { name, children, file_path } => {
if indent_level >= 0 {
for _ in 0..indent_level {
print!(" ");
}
println!("{}:", name);
}
println!("{}:", name);
let diff_count = diffs.len();
indent_level += 1;
for child in children {
run_tests(parser, child, filter, indent_level, failures)?;
run_tests(parser, child, filter, update, update_entries, indent_level, diffs)?;
}
if let Some(file_path) = file_path {
if update && diffs.len() - diff_count > 0 {
write_tests(&file_path, &update_entries)?;
}
update_entries.clear();
}
}
}
Ok(())
}
pub fn parse_tests(path: &Path) -> io::Result<TestEntry> {
fn write_tests(file_path: &Path, update_entries: &Vec<(String, String, String)>) -> Result<()> {
let mut buffer = fs::File::create(file_path)?;
write_tests_to_buffer(&mut buffer, update_entries)
}
fn write_tests_to_buffer(
buffer: &mut Write,
update_entries: &Vec<(String, String, String)>,
) -> Result<()> {
for (i, (name, input, output)) in update_entries.iter().enumerate() {
if i > 0 {
write!(buffer, "\n")?;
}
write!(
buffer,
"{}\n{}\n{}\n{}\n{}\n\n{}\n",
"=".repeat(80),
name,
"=".repeat(80),
input,
"-".repeat(80),
output.trim()
)?;
}
Ok(())
}
pub fn parse_tests(path: &Path, norm_sexp: bool) -> io::Result<TestEntry> {
let name = path
.file_stem()
.and_then(|s| s.to_str())
@ -189,13 +287,13 @@ pub fn parse_tests(path: &Path) -> io::Result<TestEntry> {
.unwrap_or("")
.starts_with(".");
if !hidden {
children.push(parse_tests(&entry.path())?);
children.push(parse_tests(&entry.path(), norm_sexp)?);
}
}
Ok(TestEntry::Group { name, children })
Ok(TestEntry::Group { name, children, file_path: None })
} else {
let content = fs::read_to_string(path)?;
Ok(parse_test_content(name, content))
Ok(parse_test_content(name, content, Some(path.to_path_buf()), norm_sexp))
}
}
@ -203,7 +301,12 @@ pub fn strip_sexp_fields(sexp: String) -> String {
SEXP_FIELD_REGEX.replace_all(&sexp, " (").to_string()
}
fn parse_test_content(name: String, content: String) -> TestEntry {
fn parse_test_content(
name: String,
content: String,
file_path: Option<PathBuf>,
norm_sexp: bool,
) -> TestEntry {
let mut children = Vec::new();
let bytes = content.as_bytes();
let mut previous_name = String::new();
@ -224,8 +327,11 @@ fn parse_test_content(name: String, content: String) -> TestEntry {
);
if let Ok(output) = str::from_utf8(&bytes[divider_end..header_start]) {
let input = bytes[previous_header_end..divider_start].to_vec();
let output = WHITESPACE_REGEX.replace_all(output.trim(), " ").to_string();
let output = output.replace(" )", ")");
let output = if norm_sexp {
normalize_sexp(output)
} else {
output.to_owned()
};
let has_fields = SEXP_FIELD_REGEX.is_match(&output);
children.push(TestEntry::Example {
name: previous_name,
@ -241,7 +347,13 @@ fn parse_test_content(name: String, content: String) -> TestEntry {
.to_string();
previous_header_end = header_end;
}
TestEntry::Group { name, children }
TestEntry::Group { name, children, file_path }
}
fn normalize_sexp(sexp: &str) -> String {
let sexp = WHITESPACE_REGEX.replace_all(sexp.trim(), " ").to_string();
let sexp = sexp.replace(" )", ")");
return sexp;
}
#[cfg(test)]
@ -273,6 +385,8 @@ d
"#
.trim()
.to_string(),
None,
true,
);
assert_eq!(
@ -292,8 +406,49 @@ d
output: "(d)".to_string(),
has_fields: false,
},
]
],
file_path: None
}
);
}
#[test]
fn test_write_tests_to_buffer() {
let mut buffer = Vec::new();
let update_entries = vec![
(
"title 1".to_string(),
"input 1".to_string(),
"output 1".to_string(),
),
(
"title 2".to_string(),
"input 2".to_string(),
"output 2".to_string(),
),
];
write_tests_to_buffer(&mut buffer, &update_entries).unwrap();
assert_eq!(
String::from_utf8(buffer).unwrap(),
r#"
================================================================================
title 1
================================================================================
input 1
--------------------------------------------------------------------------------
output 1
================================================================================
title 2
================================================================================
input 2
--------------------------------------------------------------------------------
output 2
"#
.trim_start()
.to_string()
);
}
}

8
cli/src/tests/corpus_test.rs
View file

@ -59,8 +59,8 @@ fn test_real_language_corpus_files() {
let language = get_language(language_name);
let corpus_dir = grammars_dir.join(language_name).join("corpus");
let error_corpus_file = error_corpus_dir.join(&format!("{}_errors.txt", language_name));
let main_tests = parse_tests(&corpus_dir).unwrap();
let error_tests = parse_tests(&error_corpus_file).unwrap_or(TestEntry::default());
let main_tests = parse_tests(&corpus_dir, true).unwrap();
let error_tests = parse_tests(&error_corpus_file, true).unwrap_or(TestEntry::default());
let mut tests = flatten_tests(main_tests);
tests.extend(flatten_tests(error_tests));
@ -243,7 +243,7 @@ fn test_feature_corpus_files() {
let corpus_path = test_path.join("corpus.txt");
let c_code = generate_result.unwrap().1;
let language = get_test_language(language_name, &c_code, Some(&test_path));
let test = parse_tests(&corpus_path).unwrap();
let test = parse_tests(&corpus_path, true).unwrap();
let tests = flatten_tests(test);
if !tests.is_empty() {
@ -381,7 +381,7 @@ fn flatten_tests(test: TestEntry) -> Vec<(String, Vec<u8>, String, bool)> {
}
result.push((name, input, output, has_fields));
}
TestEntry::Group { mut name, children } => {
TestEntry::Group { mut name, children, .. } => {
if !prefix.is_empty() {
name.insert_str(0, " - ");
name.insert_str(0, prefix);