From 4b0489e2f3c1136b206e93915ebedcc207d70969 Mon Sep 17 00:00:00 2001 From: Ika Date: Sun, 1 Sep 2019 14:30:33 +0800 Subject: [PATCH 01/27] fix: allow lowercase unicode escape (#440) --- cli/src/generate/prepare_grammar/expand_tokens.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cli/src/generate/prepare_grammar/expand_tokens.rs b/cli/src/generate/prepare_grammar/expand_tokens.rs index 2b88762b..9b594f3c 100644 --- a/cli/src/generate/prepare_grammar/expand_tokens.rs +++ b/cli/src/generate/prepare_grammar/expand_tokens.rs @@ -12,7 +12,7 @@ use std::i32; lazy_static! { static ref CURLY_BRACE_REGEX: Regex = - Regex::new(r#"(^|[^\\])\{([^}]*[^0-9A-F,}][^}]*)\}"#).unwrap(); + Regex::new(r#"(^|[^\\])\{([^}]*[^0-9A-Fa-f,}][^}]*)\}"#).unwrap(); } const ALLOWED_REDUNDANT_ESCAPED_CHARS: [char; 4] = ['!', '\'', '"', '/']; @@ -653,12 +653,15 @@ mod tests { Rule::pattern(r#"\{[ab]{3}\}"#), // Unicode codepoints Rule::pattern(r#"\u{1000A}"#), + // Unicode codepoints (lowercase) + Rule::pattern(r#"\u{1000b}"#), ], separators: vec![], examples: vec![ ("u{1234} ok", Some((0, "u{1234}"))), ("{aba}}", Some((1, "{aba}"))), ("\u{1000A}", Some((2, "\u{1000A}"))), + ("\u{1000b}", Some((3, "\u{1000b}"))), ], }, ]; From 1b033fdfa45faf14672f5f8895aef85eb136da75 Mon Sep 17 00:00:00 2001 From: Ika Date: Sun, 1 Sep 2019 23:52:39 +0800 Subject: [PATCH 02/27] feat(cli): support snapshot testing with `--update` flag This PR adds an `--update` flag to the `tree-sitter test` command, which adds the ability to replace the _expected_ output in the corpus.txt with the _actual_ output produced by the parser, that is, we can now simply use this `--update` flag to write all the corresponding parser output back to the corpus.txt, and we just need to check the output without typing its actual sexp. - use the same output format as `tree-sitter parse`, except there won't be any position information printed. - the corpus.txt won't be touched if there's no difference between the _expected_ output and the _actual_ output in that file. - if there're differences between _expected_ and _actual_, only the test case that is different will be replaced, the rest test cases will stay as-is. (All the delimiters `===`/`---` will be normalized as 80-column long, though.) - this flag also works with `--filter` flag. --- cli/src/lib.rs | 1 + cli/src/main.rs | 6 +- cli/src/parse.rs | 53 +------- cli/src/print.rs | 60 +++++++++ cli/src/test.rs | 251 ++++++++++++++++++++++++++++------- cli/src/tests/corpus_test.rs | 8 +- 6 files changed, 274 insertions(+), 105 deletions(-) create mode 100644 cli/src/print.rs diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 33a9904f..e996083a 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -9,6 +9,7 @@ pub mod test; pub mod util; pub mod wasm; pub mod web_ui; +pub mod print; #[cfg(test)] mod tests; diff --git a/cli/src/main.rs b/cli/src/main.rs index 59d04a97..7a5adbf1 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -83,7 +83,8 @@ fn run() -> error::Result<()> { .takes_value(true), ) .arg(Arg::with_name("debug").long("debug").short("d")) - .arg(Arg::with_name("debug-graph").long("debug-graph").short("D")), + .arg(Arg::with_name("debug-graph").long("debug-graph").short("D")) + .arg(Arg::with_name("update").long("update").short("u")), ) .subcommand( SubCommand::with_name("highlight") @@ -150,9 +151,10 @@ fn run() -> error::Result<()> { let debug = matches.is_present("debug"); let debug_graph = matches.is_present("debug-graph"); let filter = matches.value_of("filter"); + let update = matches.is_present("update"); let corpus_path = current_dir.join("corpus"); if let Some(language) = loader.languages_at_path(¤t_dir)?.first() { - test::run_tests_at_path(*language, &corpus_path, debug, debug_graph, filter)?; + test::run_tests_at_path(*language, &corpus_path, debug, debug_graph, filter, update)?; } else { eprintln!("No language found"); } diff --git a/cli/src/parse.rs b/cli/src/parse.rs index d1ddb499..065ffb02 100644 --- a/cli/src/parse.rs +++ b/cli/src/parse.rs @@ -1,4 +1,5 @@ use super::error::{Error, Result}; +use super::print::print_tree; use super::util; use std::io::{self, Write}; use std::path::Path; @@ -81,57 +82,7 @@ pub fn parse_file_at_path( let mut cursor = tree.walk(); if !quiet { - let mut needs_newline = false; - let mut indent_level = 0; - let mut did_visit_children = false; - loop { - let node = cursor.node(); - let is_named = node.is_named(); - if did_visit_children { - if is_named { - stdout.write(b")")?; - needs_newline = true; - } - if cursor.goto_next_sibling() { - did_visit_children = false; - } else if cursor.goto_parent() { - did_visit_children = true; - indent_level -= 1; - } else { - break; - } - } else { - if is_named { - if needs_newline { - stdout.write(b"\n")?; - } - for _ in 0..indent_level { - stdout.write(b" ")?; - } - let start = node.start_position(); - let end = node.end_position(); - if let Some(field_name) = cursor.field_name() { - write!(&mut stdout, "{}: ", field_name)?; - } - write!( - &mut stdout, - "({} [{}, {}] - [{}, {}]", - node.kind(), - start.row, - start.column, - end.row, - end.column - )?; - needs_newline = true; - } - if cursor.goto_first_child() { - did_visit_children = false; - indent_level += 1; - } else { - did_visit_children = true; - } - } - } + print_tree(&mut stdout, &mut cursor, true)?; cursor.reset(tree.root_node()); println!(""); } diff --git a/cli/src/print.rs b/cli/src/print.rs new file mode 100644 index 00000000..7a57c625 --- /dev/null +++ b/cli/src/print.rs @@ -0,0 +1,60 @@ +use super::error::{Result}; +use std::io::{Write}; +use tree_sitter::{TreeCursor}; + +pub fn print_tree(output: &mut Write, cursor: &mut TreeCursor, prints_position: bool) -> Result<()> { + let mut needs_newline = false; + let mut indent_level = 0; + let mut did_visit_children = false; + loop { + let node = cursor.node(); + let is_named = node.is_named(); + if did_visit_children { + if is_named { + output.write(b")")?; + needs_newline = true; + } + if cursor.goto_next_sibling() { + did_visit_children = false; + } else if cursor.goto_parent() { + did_visit_children = true; + indent_level -= 1; + } else { + break; + } + } else { + if is_named { + if needs_newline { + output.write(b"\n")?; + } + for _ in 0..indent_level { + output.write(b" ")?; + } + if let Some(field_name) = cursor.field_name() { + write!(output, "{}: ", field_name)?; + } + write!(output, "({}", node.kind())?; + if prints_position { + let start = node.start_position(); + let end = node.end_position(); + write!( + output, + " [{}, {}] - [{}, {}]", + start.row, + start.column, + end.row, + end.column + )?; + } + needs_newline = true; + } + if cursor.goto_first_child() { + did_visit_children = false; + indent_level += 1; + } else { + did_visit_children = true; + } + } + } + return Ok(()); +} diff --git a/cli/src/test.rs b/cli/src/test.rs index f742e887..7a08e805 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -1,4 +1,5 @@ use super::error::{Error, Result}; +use super::print::print_tree; use super::util; use ansi_term::Colour; use difference::{Changeset, Difference}; @@ -8,7 +9,7 @@ use regex::Regex; use std::char; use std::fs; use std::io::{self, Write}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::str; use tree_sitter::{Language, LogType, Parser}; @@ -30,6 +31,7 @@ pub enum TestEntry { Group { name: String, children: Vec, + file_path: Option, }, Example { name: String, @@ -44,6 +46,7 @@ impl Default for TestEntry { TestEntry::Group { name: String::new(), children: Vec::new(), + file_path: None, } } } @@ -54,43 +57,52 @@ pub fn run_tests_at_path( debug: bool, debug_graph: bool, filter: Option<&str>, + update: bool, ) -> Result<()> { - let test_entry = parse_tests(path)?; + let test_entry = parse_tests(path, false)?; let mut _log_session = None; let mut parser = Parser::new(); parser.set_language(language).map_err(|e| e.to_string())?; - if debug_graph { - _log_session = Some(util::log_graphs(&mut parser, "log.html")?); - } else if debug { - parser.set_logger(Some(Box::new(|log_type, message| { - if log_type == LogType::Lex { - io::stderr().write(b" ").unwrap(); - } - write!(&mut io::stderr(), "{}\n", message).unwrap(); - }))); - } - - let mut failures = Vec::new(); - if let TestEntry::Group { children, .. } = test_entry { - for child in children { - run_tests(&mut parser, child, filter, 0, &mut failures)?; + if !update { + if debug_graph { + _log_session = Some(util::log_graphs(&mut parser, "log.html")?); + } else if debug { + parser.set_logger(Some(Box::new(|log_type, message| { + if log_type == LogType::Lex { + io::stderr().write(b" ").unwrap(); + } + write!(&mut io::stderr(), "{}\n", message).unwrap(); + }))); } } - if failures.len() > 0 { + let mut diffs = Vec::new(); + let mut update_entries = Vec::new(); + run_tests(&mut parser, test_entry, filter, update, &mut update_entries, -1, &mut diffs)?; + + if diffs.len() > 0 { println!(""); - if failures.len() == 1 { - println!("1 failure:") + let diff_name = if update { "update" } else { "failure" }; + if diffs.len() == 1 { + println!("1 {}:", diff_name) } else { - println!("{} failures:", failures.len()) + println!("{} {}s:", diffs.len(), diff_name) } - print_diff_key(); - for (i, (name, actual, expected)) in failures.iter().enumerate() { + if update { + print_update_diff_key(); + } else { + print_diff_key(); + } + for (i, (name, parsed, provided)) in diffs.iter().enumerate() { println!("\n {}. {}:", i + 1, name); - print_diff(actual, expected); + if update { + print_update_diff(provided, parsed); + } else { + print_diff(parsed, provided); + } } Error::err(String::new()) } else { @@ -99,14 +111,40 @@ pub fn run_tests_at_path( } pub fn print_diff_key() { + print_diff_key_with_colors("actual", "expected", Colour::Red, Colour::Green); +} + +fn print_update_diff_key() { + print_diff_key_with_colors("original", "updated", Colour::Yellow, Colour::Green); +} + +fn print_diff_key_with_colors( + actual_name: &str, + expected_name: &str, + actual_color: Colour, + expected_color: Colour, +) { println!( "\n{} / {}", - Colour::Green.paint("expected"), - Colour::Red.paint("actual") + expected_color.paint(expected_name), + actual_color.paint(actual_name) ); } pub fn print_diff(actual: &String, expected: &String) { + print_diff_with_colors(actual, expected, Colour::Red, Colour::Green); +} + +fn print_update_diff(actual: &String, expected: &String) { + print_diff_with_colors(actual, expected, Colour::Yellow, Colour::Green); +} + +fn print_diff_with_colors( + actual: &String, + expected: &String, + actual_color: Colour, + expected_color: Colour, +) { let changeset = Changeset::new(actual, expected, " "); print!(" "); for diff in &changeset.diffs { @@ -115,10 +153,10 @@ pub fn print_diff(actual: &String, expected: &String) { print!("{}{}", part, changeset.split); } Difference::Add(part) => { - print!("{}{}", Colour::Green.paint(part), changeset.split); + print!("{}{}", expected_color.paint(part), changeset.split); } Difference::Rem(part) => { - print!("{}{}", Colour::Red.paint(part), changeset.split); + print!("{}{}", actual_color.paint(part), changeset.split); } } } @@ -129,8 +167,10 @@ fn run_tests( parser: &mut Parser, test_entry: TestEntry, filter: Option<&str>, + update: bool, + update_entries: &mut Vec<(String, String, String)>, mut indent_level: i32, - failures: &mut Vec<(String, String, String)>, + diffs: &mut Vec<(String, String, String)>, ) -> Result<()> { match test_entry { TestEntry::Example { @@ -141,39 +181,97 @@ fn run_tests( } => { if let Some(filter) = filter { if !name.contains(filter) { + if update { + let input = String::from_utf8(input).unwrap(); + update_entries.push((name, input, output)); + } return Ok(()); } } let tree = parser.parse(&input, None).unwrap(); - let mut actual = tree.root_node().to_sexp(); + let mut parsed = tree.root_node().to_sexp(); if !has_fields { - actual = strip_sexp_fields(actual); + parsed = strip_sexp_fields(parsed); } for _ in 0..indent_level { print!(" "); } - if actual == output { + let provided = normalize_sexp(&output); + if parsed == provided { println!("✓ {}", Colour::Green.paint(&name)); + if update { + let input = String::from_utf8(input).unwrap(); + update_entries.push((name, input, output)); + } } else { - println!("✗ {}", Colour::Red.paint(&name)); - failures.push((name, actual, output)); + if update { + let input = String::from_utf8(input).unwrap(); + let mut fixed_output = Vec::new(); + let mut cursor = tree.walk(); + print_tree(&mut fixed_output, &mut cursor, false)?; + let fixed_output = String::from_utf8(fixed_output).unwrap(); + update_entries.push((name.clone(), input, fixed_output)); + println!("✓ {}", Colour::Yellow.paint(&name)); + } else { + println!("✗ {}", Colour::Red.paint(&name)); + } + diffs.push((name, parsed, provided)); } } - TestEntry::Group { name, children } => { - for _ in 0..indent_level { - print!(" "); + TestEntry::Group { name, children, file_path } => { + if indent_level >= 0 { + for _ in 0..indent_level { + print!(" "); + } + println!("{}:", name); } - println!("{}:", name); + + let diff_count = diffs.len(); + indent_level += 1; for child in children { - run_tests(parser, child, filter, indent_level, failures)?; + run_tests(parser, child, filter, update, update_entries, indent_level, diffs)?; + } + + if let Some(file_path) = file_path { + if update && diffs.len() - diff_count > 0 { + write_tests(&file_path, &update_entries)?; + } + update_entries.clear(); } } } Ok(()) } -pub fn parse_tests(path: &Path) -> io::Result { +fn write_tests(file_path: &Path, update_entries: &Vec<(String, String, String)>) -> Result<()> { + let mut buffer = fs::File::create(file_path)?; + write_tests_to_buffer(&mut buffer, update_entries) +} + +fn write_tests_to_buffer( + buffer: &mut Write, + update_entries: &Vec<(String, String, String)>, +) -> Result<()> { + for (i, (name, input, output)) in update_entries.iter().enumerate() { + if i > 0 { + write!(buffer, "\n")?; + } + write!( + buffer, + "{}\n{}\n{}\n{}\n{}\n\n{}\n", + "=".repeat(80), + name, + "=".repeat(80), + input, + "-".repeat(80), + output.trim() + )?; + } + Ok(()) +} + +pub fn parse_tests(path: &Path, norm_sexp: bool) -> io::Result { let name = path .file_stem() .and_then(|s| s.to_str()) @@ -189,13 +287,13 @@ pub fn parse_tests(path: &Path) -> io::Result { .unwrap_or("") .starts_with("."); if !hidden { - children.push(parse_tests(&entry.path())?); + children.push(parse_tests(&entry.path(), norm_sexp)?); } } - Ok(TestEntry::Group { name, children }) + Ok(TestEntry::Group { name, children, file_path: None }) } else { let content = fs::read_to_string(path)?; - Ok(parse_test_content(name, content)) + Ok(parse_test_content(name, content, Some(path.to_path_buf()), norm_sexp)) } } @@ -203,7 +301,12 @@ pub fn strip_sexp_fields(sexp: String) -> String { SEXP_FIELD_REGEX.replace_all(&sexp, " (").to_string() } -fn parse_test_content(name: String, content: String) -> TestEntry { +fn parse_test_content( + name: String, + content: String, + file_path: Option, + norm_sexp: bool, +) -> TestEntry { let mut children = Vec::new(); let bytes = content.as_bytes(); let mut previous_name = String::new(); @@ -224,8 +327,11 @@ fn parse_test_content(name: String, content: String) -> TestEntry { ); if let Ok(output) = str::from_utf8(&bytes[divider_end..header_start]) { let input = bytes[previous_header_end..divider_start].to_vec(); - let output = WHITESPACE_REGEX.replace_all(output.trim(), " ").to_string(); - let output = output.replace(" )", ")"); + let output = if norm_sexp { + normalize_sexp(output) + } else { + output.to_owned() + }; let has_fields = SEXP_FIELD_REGEX.is_match(&output); children.push(TestEntry::Example { name: previous_name, @@ -241,7 +347,13 @@ fn parse_test_content(name: String, content: String) -> TestEntry { .to_string(); previous_header_end = header_end; } - TestEntry::Group { name, children } + TestEntry::Group { name, children, file_path } +} + +fn normalize_sexp(sexp: &str) -> String { + let sexp = WHITESPACE_REGEX.replace_all(sexp.trim(), " ").to_string(); + let sexp = sexp.replace(" )", ")"); + return sexp; } #[cfg(test)] @@ -273,6 +385,8 @@ d "# .trim() .to_string(), + None, + true, ); assert_eq!( @@ -292,8 +406,49 @@ d output: "(d)".to_string(), has_fields: false, }, - ] + ], + file_path: None } ); } + + #[test] + fn test_write_tests_to_buffer() { + let mut buffer = Vec::new(); + let update_entries = vec![ + ( + "title 1".to_string(), + "input 1".to_string(), + "output 1".to_string(), + ), + ( + "title 2".to_string(), + "input 2".to_string(), + "output 2".to_string(), + ), + ]; + write_tests_to_buffer(&mut buffer, &update_entries).unwrap(); + assert_eq!( + String::from_utf8(buffer).unwrap(), + r#" +================================================================================ +title 1 +================================================================================ +input 1 +-------------------------------------------------------------------------------- + +output 1 + +================================================================================ +title 2 +================================================================================ +input 2 +-------------------------------------------------------------------------------- + +output 2 +"# + .trim_start() + .to_string() + ); + } } diff --git a/cli/src/tests/corpus_test.rs b/cli/src/tests/corpus_test.rs index a8adce5f..dae87542 100644 --- a/cli/src/tests/corpus_test.rs +++ b/cli/src/tests/corpus_test.rs @@ -59,8 +59,8 @@ fn test_real_language_corpus_files() { let language = get_language(language_name); let corpus_dir = grammars_dir.join(language_name).join("corpus"); let error_corpus_file = error_corpus_dir.join(&format!("{}_errors.txt", language_name)); - let main_tests = parse_tests(&corpus_dir).unwrap(); - let error_tests = parse_tests(&error_corpus_file).unwrap_or(TestEntry::default()); + let main_tests = parse_tests(&corpus_dir, true).unwrap(); + let error_tests = parse_tests(&error_corpus_file, true).unwrap_or(TestEntry::default()); let mut tests = flatten_tests(main_tests); tests.extend(flatten_tests(error_tests)); @@ -243,7 +243,7 @@ fn test_feature_corpus_files() { let corpus_path = test_path.join("corpus.txt"); let c_code = generate_result.unwrap().1; let language = get_test_language(language_name, &c_code, Some(&test_path)); - let test = parse_tests(&corpus_path).unwrap(); + let test = parse_tests(&corpus_path, true).unwrap(); let tests = flatten_tests(test); if !tests.is_empty() { @@ -381,7 +381,7 @@ fn flatten_tests(test: TestEntry) -> Vec<(String, Vec, String, bool)> { } result.push((name, input, output, has_fields)); } - TestEntry::Group { mut name, children } => { + TestEntry::Group { mut name, children, .. } => { if !prefix.is_empty() { name.insert_str(0, " - "); name.insert_str(0, prefix); From 807fdf3ef0bbefade37607e923ab3893a420d50b Mon Sep 17 00:00:00 2001 From: Ika Date: Fri, 6 Sep 2019 10:42:37 +0800 Subject: [PATCH 03/27] Revert "feat(cli): support snapshot testing with `--update` flag" This reverts commit 1b033fdfa45faf14672f5f8895aef85eb136da75. --- cli/src/lib.rs | 1 - cli/src/main.rs | 6 +- cli/src/parse.rs | 53 +++++++- cli/src/print.rs | 60 --------- cli/src/test.rs | 251 +++++++---------------------------- cli/src/tests/corpus_test.rs | 8 +- 6 files changed, 105 insertions(+), 274 deletions(-) delete mode 100644 cli/src/print.rs diff --git a/cli/src/lib.rs b/cli/src/lib.rs index e996083a..33a9904f 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -9,7 +9,6 @@ pub mod test; pub mod util; pub mod wasm; pub mod web_ui; -pub mod print; #[cfg(test)] mod tests; diff --git a/cli/src/main.rs b/cli/src/main.rs index 7a5adbf1..59d04a97 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -83,8 +83,7 @@ fn run() -> error::Result<()> { .takes_value(true), ) .arg(Arg::with_name("debug").long("debug").short("d")) - .arg(Arg::with_name("debug-graph").long("debug-graph").short("D")) - .arg(Arg::with_name("update").long("update").short("u")), + .arg(Arg::with_name("debug-graph").long("debug-graph").short("D")), ) .subcommand( SubCommand::with_name("highlight") @@ -151,10 +150,9 @@ fn run() -> error::Result<()> { let debug = matches.is_present("debug"); let debug_graph = matches.is_present("debug-graph"); let filter = matches.value_of("filter"); - let update = matches.is_present("update"); let corpus_path = current_dir.join("corpus"); if let Some(language) = loader.languages_at_path(¤t_dir)?.first() { - test::run_tests_at_path(*language, &corpus_path, debug, debug_graph, filter, update)?; + test::run_tests_at_path(*language, &corpus_path, debug, debug_graph, filter)?; } else { eprintln!("No language found"); } diff --git a/cli/src/parse.rs b/cli/src/parse.rs index 065ffb02..d1ddb499 100644 --- a/cli/src/parse.rs +++ b/cli/src/parse.rs @@ -1,5 +1,4 @@ use super::error::{Error, Result}; -use super::print::print_tree; use super::util; use std::io::{self, Write}; use std::path::Path; @@ -82,7 +81,57 @@ pub fn parse_file_at_path( let mut cursor = tree.walk(); if !quiet { - print_tree(&mut stdout, &mut cursor, true)?; + let mut needs_newline = false; + let mut indent_level = 0; + let mut did_visit_children = false; + loop { + let node = cursor.node(); + let is_named = node.is_named(); + if did_visit_children { + if is_named { + stdout.write(b")")?; + needs_newline = true; + } + if cursor.goto_next_sibling() { + did_visit_children = false; + } else if cursor.goto_parent() { + did_visit_children = true; + indent_level -= 1; + } else { + break; + } + } else { + if is_named { + if needs_newline { + stdout.write(b"\n")?; + } + for _ in 0..indent_level { + stdout.write(b" ")?; + } + let start = node.start_position(); + let end = node.end_position(); + if let Some(field_name) = cursor.field_name() { + write!(&mut stdout, "{}: ", field_name)?; + } + write!( + &mut stdout, + "({} [{}, {}] - [{}, {}]", + node.kind(), + start.row, + start.column, + end.row, + end.column + )?; + needs_newline = true; + } + if cursor.goto_first_child() { + did_visit_children = false; + indent_level += 1; + } else { + did_visit_children = true; + } + } + } cursor.reset(tree.root_node()); println!(""); } diff --git a/cli/src/print.rs b/cli/src/print.rs deleted file mode 100644 index 7a57c625..00000000 --- a/cli/src/print.rs +++ /dev/null @@ -1,60 +0,0 @@ -use super::error::{Result}; -use std::io::{Write}; -use tree_sitter::{TreeCursor}; - -pub fn print_tree(output: &mut Write, cursor: &mut TreeCursor, prints_position: bool) -> Result<()> { - let mut needs_newline = false; - let mut indent_level = 0; - let mut did_visit_children = false; - loop { - let node = cursor.node(); - let is_named = node.is_named(); - if did_visit_children { - if is_named { - output.write(b")")?; - needs_newline = true; - } - if cursor.goto_next_sibling() { - did_visit_children = false; - } else if cursor.goto_parent() { - did_visit_children = true; - indent_level -= 1; - } else { - break; - } - } else { - if is_named { - if needs_newline { - output.write(b"\n")?; - } - for _ in 0..indent_level { - output.write(b" ")?; - } - if let Some(field_name) = cursor.field_name() { - write!(output, "{}: ", field_name)?; - } - write!(output, "({}", node.kind())?; - if prints_position { - let start = node.start_position(); - let end = node.end_position(); - write!( - output, - " [{}, {}] - [{}, {}]", - start.row, - start.column, - end.row, - end.column - )?; - } - needs_newline = true; - } - if cursor.goto_first_child() { - did_visit_children = false; - indent_level += 1; - } else { - did_visit_children = true; - } - } - } - return Ok(()); -} diff --git a/cli/src/test.rs b/cli/src/test.rs index 7a08e805..f742e887 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -1,5 +1,4 @@ use super::error::{Error, Result}; -use super::print::print_tree; use super::util; use ansi_term::Colour; use difference::{Changeset, Difference}; @@ -9,7 +8,7 @@ use regex::Regex; use std::char; use std::fs; use std::io::{self, Write}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::str; use tree_sitter::{Language, LogType, Parser}; @@ -31,7 +30,6 @@ pub enum TestEntry { Group { name: String, children: Vec, - file_path: Option, }, Example { name: String, @@ -46,7 +44,6 @@ impl Default for TestEntry { TestEntry::Group { name: String::new(), children: Vec::new(), - file_path: None, } } } @@ -57,52 +54,43 @@ pub fn run_tests_at_path( debug: bool, debug_graph: bool, filter: Option<&str>, - update: bool, ) -> Result<()> { - let test_entry = parse_tests(path, false)?; + let test_entry = parse_tests(path)?; let mut _log_session = None; let mut parser = Parser::new(); parser.set_language(language).map_err(|e| e.to_string())?; - if !update { - if debug_graph { - _log_session = Some(util::log_graphs(&mut parser, "log.html")?); - } else if debug { - parser.set_logger(Some(Box::new(|log_type, message| { - if log_type == LogType::Lex { - io::stderr().write(b" ").unwrap(); - } - write!(&mut io::stderr(), "{}\n", message).unwrap(); - }))); + if debug_graph { + _log_session = Some(util::log_graphs(&mut parser, "log.html")?); + } else if debug { + parser.set_logger(Some(Box::new(|log_type, message| { + if log_type == LogType::Lex { + io::stderr().write(b" ").unwrap(); + } + write!(&mut io::stderr(), "{}\n", message).unwrap(); + }))); + } + + let mut failures = Vec::new(); + if let TestEntry::Group { children, .. } = test_entry { + for child in children { + run_tests(&mut parser, child, filter, 0, &mut failures)?; } } - let mut diffs = Vec::new(); - let mut update_entries = Vec::new(); - run_tests(&mut parser, test_entry, filter, update, &mut update_entries, -1, &mut diffs)?; - - if diffs.len() > 0 { + if failures.len() > 0 { println!(""); - let diff_name = if update { "update" } else { "failure" }; - if diffs.len() == 1 { - println!("1 {}:", diff_name) + if failures.len() == 1 { + println!("1 failure:") } else { - println!("{} {}s:", diffs.len(), diff_name) + println!("{} failures:", failures.len()) } - if update { - print_update_diff_key(); - } else { - print_diff_key(); - } - for (i, (name, parsed, provided)) in diffs.iter().enumerate() { + print_diff_key(); + for (i, (name, actual, expected)) in failures.iter().enumerate() { println!("\n {}. {}:", i + 1, name); - if update { - print_update_diff(provided, parsed); - } else { - print_diff(parsed, provided); - } + print_diff(actual, expected); } Error::err(String::new()) } else { @@ -111,40 +99,14 @@ pub fn run_tests_at_path( } pub fn print_diff_key() { - print_diff_key_with_colors("actual", "expected", Colour::Red, Colour::Green); -} - -fn print_update_diff_key() { - print_diff_key_with_colors("original", "updated", Colour::Yellow, Colour::Green); -} - -fn print_diff_key_with_colors( - actual_name: &str, - expected_name: &str, - actual_color: Colour, - expected_color: Colour, -) { println!( "\n{} / {}", - expected_color.paint(expected_name), - actual_color.paint(actual_name) + Colour::Green.paint("expected"), + Colour::Red.paint("actual") ); } pub fn print_diff(actual: &String, expected: &String) { - print_diff_with_colors(actual, expected, Colour::Red, Colour::Green); -} - -fn print_update_diff(actual: &String, expected: &String) { - print_diff_with_colors(actual, expected, Colour::Yellow, Colour::Green); -} - -fn print_diff_with_colors( - actual: &String, - expected: &String, - actual_color: Colour, - expected_color: Colour, -) { let changeset = Changeset::new(actual, expected, " "); print!(" "); for diff in &changeset.diffs { @@ -153,10 +115,10 @@ fn print_diff_with_colors( print!("{}{}", part, changeset.split); } Difference::Add(part) => { - print!("{}{}", expected_color.paint(part), changeset.split); + print!("{}{}", Colour::Green.paint(part), changeset.split); } Difference::Rem(part) => { - print!("{}{}", actual_color.paint(part), changeset.split); + print!("{}{}", Colour::Red.paint(part), changeset.split); } } } @@ -167,10 +129,8 @@ fn run_tests( parser: &mut Parser, test_entry: TestEntry, filter: Option<&str>, - update: bool, - update_entries: &mut Vec<(String, String, String)>, mut indent_level: i32, - diffs: &mut Vec<(String, String, String)>, + failures: &mut Vec<(String, String, String)>, ) -> Result<()> { match test_entry { TestEntry::Example { @@ -181,97 +141,39 @@ fn run_tests( } => { if let Some(filter) = filter { if !name.contains(filter) { - if update { - let input = String::from_utf8(input).unwrap(); - update_entries.push((name, input, output)); - } return Ok(()); } } let tree = parser.parse(&input, None).unwrap(); - let mut parsed = tree.root_node().to_sexp(); + let mut actual = tree.root_node().to_sexp(); if !has_fields { - parsed = strip_sexp_fields(parsed); + actual = strip_sexp_fields(actual); } for _ in 0..indent_level { print!(" "); } - let provided = normalize_sexp(&output); - if parsed == provided { + if actual == output { println!("✓ {}", Colour::Green.paint(&name)); - if update { - let input = String::from_utf8(input).unwrap(); - update_entries.push((name, input, output)); - } } else { - if update { - let input = String::from_utf8(input).unwrap(); - let mut fixed_output = Vec::new(); - let mut cursor = tree.walk(); - print_tree(&mut fixed_output, &mut cursor, false)?; - let fixed_output = String::from_utf8(fixed_output).unwrap(); - update_entries.push((name.clone(), input, fixed_output)); - println!("✓ {}", Colour::Yellow.paint(&name)); - } else { - println!("✗ {}", Colour::Red.paint(&name)); - } - diffs.push((name, parsed, provided)); + println!("✗ {}", Colour::Red.paint(&name)); + failures.push((name, actual, output)); } } - TestEntry::Group { name, children, file_path } => { - if indent_level >= 0 { - for _ in 0..indent_level { - print!(" "); - } - println!("{}:", name); + TestEntry::Group { name, children } => { + for _ in 0..indent_level { + print!(" "); } - - let diff_count = diffs.len(); - + println!("{}:", name); indent_level += 1; for child in children { - run_tests(parser, child, filter, update, update_entries, indent_level, diffs)?; - } - - if let Some(file_path) = file_path { - if update && diffs.len() - diff_count > 0 { - write_tests(&file_path, &update_entries)?; - } - update_entries.clear(); + run_tests(parser, child, filter, indent_level, failures)?; } } } Ok(()) } -fn write_tests(file_path: &Path, update_entries: &Vec<(String, String, String)>) -> Result<()> { - let mut buffer = fs::File::create(file_path)?; - write_tests_to_buffer(&mut buffer, update_entries) -} - -fn write_tests_to_buffer( - buffer: &mut Write, - update_entries: &Vec<(String, String, String)>, -) -> Result<()> { - for (i, (name, input, output)) in update_entries.iter().enumerate() { - if i > 0 { - write!(buffer, "\n")?; - } - write!( - buffer, - "{}\n{}\n{}\n{}\n{}\n\n{}\n", - "=".repeat(80), - name, - "=".repeat(80), - input, - "-".repeat(80), - output.trim() - )?; - } - Ok(()) -} - -pub fn parse_tests(path: &Path, norm_sexp: bool) -> io::Result { +pub fn parse_tests(path: &Path) -> io::Result { let name = path .file_stem() .and_then(|s| s.to_str()) @@ -287,13 +189,13 @@ pub fn parse_tests(path: &Path, norm_sexp: bool) -> io::Result { .unwrap_or("") .starts_with("."); if !hidden { - children.push(parse_tests(&entry.path(), norm_sexp)?); + children.push(parse_tests(&entry.path())?); } } - Ok(TestEntry::Group { name, children, file_path: None }) + Ok(TestEntry::Group { name, children }) } else { let content = fs::read_to_string(path)?; - Ok(parse_test_content(name, content, Some(path.to_path_buf()), norm_sexp)) + Ok(parse_test_content(name, content)) } } @@ -301,12 +203,7 @@ pub fn strip_sexp_fields(sexp: String) -> String { SEXP_FIELD_REGEX.replace_all(&sexp, " (").to_string() } -fn parse_test_content( - name: String, - content: String, - file_path: Option, - norm_sexp: bool, -) -> TestEntry { +fn parse_test_content(name: String, content: String) -> TestEntry { let mut children = Vec::new(); let bytes = content.as_bytes(); let mut previous_name = String::new(); @@ -327,11 +224,8 @@ fn parse_test_content( ); if let Ok(output) = str::from_utf8(&bytes[divider_end..header_start]) { let input = bytes[previous_header_end..divider_start].to_vec(); - let output = if norm_sexp { - normalize_sexp(output) - } else { - output.to_owned() - }; + let output = WHITESPACE_REGEX.replace_all(output.trim(), " ").to_string(); + let output = output.replace(" )", ")"); let has_fields = SEXP_FIELD_REGEX.is_match(&output); children.push(TestEntry::Example { name: previous_name, @@ -347,13 +241,7 @@ fn parse_test_content( .to_string(); previous_header_end = header_end; } - TestEntry::Group { name, children, file_path } -} - -fn normalize_sexp(sexp: &str) -> String { - let sexp = WHITESPACE_REGEX.replace_all(sexp.trim(), " ").to_string(); - let sexp = sexp.replace(" )", ")"); - return sexp; + TestEntry::Group { name, children } } #[cfg(test)] @@ -385,8 +273,6 @@ d "# .trim() .to_string(), - None, - true, ); assert_eq!( @@ -406,49 +292,8 @@ d output: "(d)".to_string(), has_fields: false, }, - ], - file_path: None + ] } ); } - - #[test] - fn test_write_tests_to_buffer() { - let mut buffer = Vec::new(); - let update_entries = vec![ - ( - "title 1".to_string(), - "input 1".to_string(), - "output 1".to_string(), - ), - ( - "title 2".to_string(), - "input 2".to_string(), - "output 2".to_string(), - ), - ]; - write_tests_to_buffer(&mut buffer, &update_entries).unwrap(); - assert_eq!( - String::from_utf8(buffer).unwrap(), - r#" -================================================================================ -title 1 -================================================================================ -input 1 --------------------------------------------------------------------------------- - -output 1 - -================================================================================ -title 2 -================================================================================ -input 2 --------------------------------------------------------------------------------- - -output 2 -"# - .trim_start() - .to_string() - ); - } } diff --git a/cli/src/tests/corpus_test.rs b/cli/src/tests/corpus_test.rs index dae87542..a8adce5f 100644 --- a/cli/src/tests/corpus_test.rs +++ b/cli/src/tests/corpus_test.rs @@ -59,8 +59,8 @@ fn test_real_language_corpus_files() { let language = get_language(language_name); let corpus_dir = grammars_dir.join(language_name).join("corpus"); let error_corpus_file = error_corpus_dir.join(&format!("{}_errors.txt", language_name)); - let main_tests = parse_tests(&corpus_dir, true).unwrap(); - let error_tests = parse_tests(&error_corpus_file, true).unwrap_or(TestEntry::default()); + let main_tests = parse_tests(&corpus_dir).unwrap(); + let error_tests = parse_tests(&error_corpus_file).unwrap_or(TestEntry::default()); let mut tests = flatten_tests(main_tests); tests.extend(flatten_tests(error_tests)); @@ -243,7 +243,7 @@ fn test_feature_corpus_files() { let corpus_path = test_path.join("corpus.txt"); let c_code = generate_result.unwrap().1; let language = get_test_language(language_name, &c_code, Some(&test_path)); - let test = parse_tests(&corpus_path, true).unwrap(); + let test = parse_tests(&corpus_path).unwrap(); let tests = flatten_tests(test); if !tests.is_empty() { @@ -381,7 +381,7 @@ fn flatten_tests(test: TestEntry) -> Vec<(String, Vec, String, bool)> { } result.push((name, input, output, has_fields)); } - TestEntry::Group { mut name, children, .. } => { + TestEntry::Group { mut name, children } => { if !prefix.is_empty() { name.insert_str(0, " - "); name.insert_str(0, prefix); From d88dae7a3e15f8130a6b54f2f900ed2b109a7613 Mon Sep 17 00:00:00 2001 From: Ika Date: Fri, 6 Sep 2019 10:57:59 +0800 Subject: [PATCH 04/27] feat(cli): support snapshot testing with `--update` flag This PR adds an `--update` flag to the `tree-sitter test` command, which adds the ability to replace the _expected_ output in the corpus.txt with the _actual_ output produced by the parser, that is, we can now simply use this `--update` flag to write all the corresponding parser output back to the corpus.txt, and we just need to check the output without typing its actual sexp. - use the same output format as `tree-sitter parse`, except there won't be any position information printed. - the corpus.txt won't be touched if there's no difference between the _expected_ output and the _actual_ output in that file. - if there're differences between _expected_ and _actual_, _expected_ will be replaced by _actual_ and the whole file will be reformatted, i.e., all the output sexp will be formatted just like the output from `tree-sitter parse` and all the delimiters `===`/`---` will be normalized as 80-column long. - this flag also works with `--filter` flag. --- cli/src/main.rs | 4 +- cli/src/test.rs | 240 +++++++++++++++++++++++++++++++---- cli/src/tests/corpus_test.rs | 2 +- 3 files changed, 216 insertions(+), 30 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 59d04a97..84b13da8 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -82,6 +82,7 @@ fn run() -> error::Result<()> { .short("f") .takes_value(true), ) + .arg(Arg::with_name("update").long("update").short("u")) .arg(Arg::with_name("debug").long("debug").short("d")) .arg(Arg::with_name("debug-graph").long("debug-graph").short("D")), ) @@ -150,9 +151,10 @@ fn run() -> error::Result<()> { let debug = matches.is_present("debug"); let debug_graph = matches.is_present("debug-graph"); let filter = matches.value_of("filter"); + let update = matches.is_present("update"); let corpus_path = current_dir.join("corpus"); if let Some(language) = loader.languages_at_path(¤t_dir)?.first() { - test::run_tests_at_path(*language, &corpus_path, debug, debug_graph, filter)?; + test::run_tests_at_path(*language, &corpus_path, debug, debug_graph, filter, update)?; } else { eprintln!("No language found"); } diff --git a/cli/src/test.rs b/cli/src/test.rs index c0f5e6a7..c57d72ad 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -6,9 +6,10 @@ use lazy_static::lazy_static; use regex::bytes::{Regex as ByteRegex, RegexBuilder as ByteRegexBuilder}; use regex::Regex; use std::char; +use std::fmt::Write as FmtWrite; use std::fs; use std::io::{self, Write}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::str; use tree_sitter::{Language, LogType, Parser}; @@ -30,6 +31,7 @@ pub enum TestEntry { Group { name: String, children: Vec, + file_path: Option, }, Example { name: String, @@ -44,6 +46,7 @@ impl Default for TestEntry { TestEntry::Group { name: String::new(), children: Vec::new(), + file_path: None, } } } @@ -54,6 +57,7 @@ pub fn run_tests_at_path( debug: bool, debug_graph: bool, filter: Option<&str>, + update: bool, ) -> Result<()> { let test_entry = parse_tests(path)?; let mut _log_session = None; @@ -72,27 +76,37 @@ pub fn run_tests_at_path( } let mut failures = Vec::new(); - if let TestEntry::Group { children, .. } = test_entry { - for child in children { - run_tests(&mut parser, child, filter, 0, &mut failures)?; - } - } + let mut corrected_entries = Vec::new(); + run_tests(&mut parser, test_entry, filter, 0, &mut failures, update, &mut corrected_entries)?; if failures.len() > 0 { println!(""); - if failures.len() == 1 { - println!("1 failure:") - } else { - println!("{} failures:", failures.len()) - } + if update { + if failures.len() == 1 { + println!("1 update:\n") + } else { + println!("{} updates:\n", failures.len()) + } - print_diff_key(); - for (i, (name, actual, expected)) in failures.iter().enumerate() { - println!("\n {}. {}:", i + 1, name); - print_diff(actual, expected); + for (i, (name, ..)) in failures.iter().enumerate() { + println!(" {}. {}", i + 1, name); + } + Ok(()) + } else { + if failures.len() == 1 { + println!("1 failure:") + } else { + println!("{} failures:", failures.len()) + } + + print_diff_key(); + for (i, (name, actual, expected)) in failures.iter().enumerate() { + println!("\n {}. {}:", i + 1, name); + print_diff(actual, expected); + } + Error::err(String::new()) } - Error::err(String::new()) } else { Ok(()) } @@ -131,6 +145,8 @@ fn run_tests( filter: Option<&str>, mut indent_level: i32, failures: &mut Vec<(String, String, String)>, + update: bool, + corrected_entries: &mut Vec<(String, String, String)>, ) -> Result<()> { match test_entry { TestEntry::Example { @@ -141,6 +157,11 @@ fn run_tests( } => { if let Some(filter) = filter { if !name.contains(filter) { + if update { + let input = String::from_utf8(input).unwrap(); + let output = format_sexp(&output); + corrected_entries.push((name, input, output)); + } return Ok(()); } } @@ -154,25 +175,126 @@ fn run_tests( } if actual == output { println!("✓ {}", Colour::Green.paint(&name)); + if update { + let input = String::from_utf8(input).unwrap(); + let output = format_sexp(&output); + corrected_entries.push((name, input, output)); + } } else { - println!("✗ {}", Colour::Red.paint(&name)); + if update { + let input = String::from_utf8(input).unwrap(); + let output = format_sexp(&actual); + corrected_entries.push((name.clone(), input, output)); + println!("✓ {}", Colour::Blue.paint(&name)); + } else { + println!("✗ {}", Colour::Red.paint(&name)); + } failures.push((name, actual, output)); } } - TestEntry::Group { name, children } => { - for _ in 0..indent_level { - print!(" "); + TestEntry::Group { name, children, file_path } => { + if indent_level > 0 { + for _ in 0..indent_level { + print!(" "); + } + println!("{}:", name); } - println!("{}:", name); + + let failure_count = failures.len(); + indent_level += 1; for child in children { - run_tests(parser, child, filter, indent_level, failures)?; + run_tests(parser, child, filter, indent_level, failures, update, corrected_entries)?; + } + + if let Some(file_path) = file_path { + if update && failures.len() - failure_count > 0 { + write_tests(&file_path, corrected_entries)?; + } + corrected_entries.clear(); } } } Ok(()) } +fn format_sexp(sexp: &String) -> String { + let mut formatted = String::new(); + + let mut indent_level = 0; + let mut has_field = false; + let mut s_iter = sexp.split(|c| c == ' ' || c == ')'); + while let Some(s) = s_iter.next() { + if s.is_empty() { + // ")" + indent_level -= 1; + write!(formatted, ")").unwrap(); + } else if s.starts_with('(') { + if has_field { + has_field = false; + } else { + if indent_level > 0 { + writeln!(formatted, "").unwrap(); + for _ in 0..indent_level { + write!(formatted, " ").unwrap(); + } + } + indent_level += 1; + } + + // "(node_name" + write!(formatted, "{}", s).unwrap(); + + let mut c_iter = s.chars(); + c_iter.next(); + let second_char = c_iter.next().unwrap(); + if second_char == 'M' { + // "(MISSING node_name" + let s = s_iter.next().unwrap(); + write!(formatted, " {}", s).unwrap(); + } + } else if s.ends_with(':') { + // "field:" + writeln!(formatted, "").unwrap(); + for _ in 0..indent_level { + write!(formatted, " ").unwrap(); + } + write!(formatted, "{} ", s).unwrap(); + has_field = true; + indent_level += 1; + } + } + + formatted +} + +fn write_tests(file_path: &Path, corrected_entries: &Vec<(String, String, String)>) -> Result<()> { + let mut buffer = fs::File::create(file_path)?; + write_tests_to_buffer(&mut buffer, corrected_entries) +} + +fn write_tests_to_buffer( + buffer: &mut Write, + corrected_entries: &Vec<(String, String, String)>, +) -> Result<()> { + for (i, (name, input, output)) in corrected_entries.iter().enumerate() { + if i > 0 { + write!(buffer, "\n")?; + } + write!( + buffer, + "{}\n{}\n{}\n{}\n{}\n\n{}\n", + "=".repeat(80), + name, + "=".repeat(80), + input, + "-".repeat(80), + output.trim() + )?; + } + Ok(()) +} + pub fn parse_tests(path: &Path) -> io::Result { let name = path .file_stem() @@ -188,10 +310,10 @@ pub fn parse_tests(path: &Path) -> io::Result { children.push(parse_tests(&entry.path())?); } } - Ok(TestEntry::Group { name, children }) + Ok(TestEntry::Group { name, children, file_path: None }) } else { let content = fs::read_to_string(path)?; - Ok(parse_test_content(name, content)) + Ok(parse_test_content(name, content, Some(path.to_path_buf()))) } } @@ -199,7 +321,7 @@ pub fn strip_sexp_fields(sexp: String) -> String { SEXP_FIELD_REGEX.replace_all(&sexp, " (").to_string() } -fn parse_test_content(name: String, content: String) -> TestEntry { +fn parse_test_content(name: String, content: String, file_path: Option) -> TestEntry { let mut children = Vec::new(); let bytes = content.as_bytes(); let mut prev_name = String::new(); @@ -250,7 +372,7 @@ fn parse_test_content(name: String, content: String) -> TestEntry { .to_string(); prev_header_end = header_end; } - TestEntry::Group { name, children } + TestEntry::Group { name, children, file_path } } #[cfg(test)] @@ -282,6 +404,7 @@ d "# .trim() .to_string(), + None, ); assert_eq!( @@ -301,7 +424,8 @@ d output: "(d)".to_string(), has_fields: false, }, - ] + ], + file_path: None, } ); } @@ -334,6 +458,7 @@ abc "# .trim() .to_string(), + None, ); assert_eq!( @@ -353,8 +478,67 @@ abc output: "(c (d))".to_string(), has_fields: false, }, - ] + ], + file_path: None, } ); } + + #[test] + fn test_format_sexp() { + assert_eq!( + format_sexp(&"(a b: (c) (d) e: (f (g (h (MISSING i)))))".to_string()), + r#" +(a + b: (c) + (d) + e: (f + (g + (h + (MISSING i))))) +"# + .trim() + .to_string() + ); + } + + #[test] + fn test_write_tests_to_buffer() { + let mut buffer = Vec::new(); + let corrected_entries = vec![ + ( + "title 1".to_string(), + "input 1".to_string(), + "output 1".to_string(), + ), + ( + "title 2".to_string(), + "input 2".to_string(), + "output 2".to_string(), + ), + ]; + write_tests_to_buffer(&mut buffer, &corrected_entries).unwrap(); + assert_eq!( + String::from_utf8(buffer).unwrap(), + r#" +================================================================================ +title 1 +================================================================================ +input 1 +-------------------------------------------------------------------------------- + +output 1 + +================================================================================ +title 2 +================================================================================ +input 2 +-------------------------------------------------------------------------------- + +output 2 +"# + .trim_start() + .to_string() + ); + } } diff --git a/cli/src/tests/corpus_test.rs b/cli/src/tests/corpus_test.rs index a8adce5f..ed6226f2 100644 --- a/cli/src/tests/corpus_test.rs +++ b/cli/src/tests/corpus_test.rs @@ -381,7 +381,7 @@ fn flatten_tests(test: TestEntry) -> Vec<(String, Vec, String, bool)> { } result.push((name, input, output, has_fields)); } - TestEntry::Group { mut name, children } => { + TestEntry::Group { mut name, children, .. } => { if !prefix.is_empty() { name.insert_str(0, " - "); name.insert_str(0, prefix); From f191858bae95d2daab860d8104a9e9f60cd1d351 Mon Sep 17 00:00:00 2001 From: Ika Date: Thu, 3 Oct 2019 21:24:17 +0800 Subject: [PATCH 05/27] fix: handle UNEXPECTED node --- cli/src/test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/test.rs b/cli/src/test.rs index c57d72ad..544ec249 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -248,8 +248,8 @@ fn format_sexp(sexp: &String) -> String { let mut c_iter = s.chars(); c_iter.next(); let second_char = c_iter.next().unwrap(); - if second_char == 'M' { - // "(MISSING node_name" + if second_char == 'M' || second_char == 'U' { + // "(MISSING node_name" or "(UNEXPECTED 'x'" let s = s_iter.next().unwrap(); write!(formatted, " {}", s).unwrap(); } From 96f259d8c56b2cf567de042b416445f1ad2634a2 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 3 Dec 2020 09:48:20 -0800 Subject: [PATCH 06/27] Run rustfmt --- cli/src/test.rs | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/cli/src/test.rs b/cli/src/test.rs index 50c27220..c8cfe89f 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -77,7 +77,15 @@ pub fn run_tests_at_path( let mut failures = Vec::new(); let mut corrected_entries = Vec::new(); - run_tests(&mut parser, test_entry, filter, 0, &mut failures, update, &mut corrected_entries)?; + run_tests( + &mut parser, + test_entry, + filter, + 0, + &mut failures, + update, + &mut corrected_entries, + )?; if failures.len() > 0 { println!(""); @@ -210,7 +218,11 @@ fn run_tests( failures.push((name, actual, output)); } } - TestEntry::Group { name, children, file_path } => { + TestEntry::Group { + name, + children, + file_path, + } => { if indent_level > 0 { for _ in 0..indent_level { print!(" "); @@ -222,7 +234,15 @@ fn run_tests( indent_level += 1; for child in children { - run_tests(parser, child, filter, indent_level, failures, update, corrected_entries)?; + run_tests( + parser, + child, + filter, + indent_level, + failures, + update, + corrected_entries, + )?; } if let Some(file_path) = file_path { @@ -292,7 +312,7 @@ fn write_tests(file_path: &Path, corrected_entries: &Vec<(String, String, String } fn write_tests_to_buffer( - buffer: &mut Write, + buffer: &mut impl Write, corrected_entries: &Vec<(String, String, String)>, ) -> Result<()> { for (i, (name, input, output)) in corrected_entries.iter().enumerate() { @@ -328,7 +348,11 @@ pub fn parse_tests(path: &Path) -> io::Result { children.push(parse_tests(&entry.path())?); } } - Ok(TestEntry::Group { name, children, file_path: None }) + Ok(TestEntry::Group { + name, + children, + file_path: None, + }) } else { let content = fs::read_to_string(path)?; Ok(parse_test_content(name, content, Some(path.to_path_buf()))) @@ -390,7 +414,11 @@ fn parse_test_content(name: String, content: String, file_path: Option) .to_string(); prev_header_end = header_end; } - TestEntry::Group { name, children, file_path } + TestEntry::Group { + name, + children, + file_path, + } } #[cfg(test)] From 94c61de35358bacab8251c47081b85ae84d9b86c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 2 Dec 2020 16:12:01 -0800 Subject: [PATCH 07/27] Update JS error recovery test to reflect grammar change --- test/fixtures/error_corpus/javascript_errors.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/fixtures/error_corpus/javascript_errors.txt b/test/fixtures/error_corpus/javascript_errors.txt index ad71037c..4359ae68 100644 --- a/test/fixtures/error_corpus/javascript_errors.txt +++ b/test/fixtures/error_corpus/javascript_errors.txt @@ -148,7 +148,8 @@ const h = `i ${j(k} l` (lexical_declaration (variable_declarator (identifier) - (template_string (template_substitution (identifier) (ERROR))))) + (template_string (template_substitution + (augmented_assignment_expression (identifier) (MISSING identifier)))))) (lexical_declaration (variable_declarator (identifier) From 5008700735be65bb81cf3de5b292708778c3562a Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 4 Dec 2020 14:36:28 -0800 Subject: [PATCH 08/27] wasm: Look for both loadWebAssemblyModule and loadSideModule See https://github.com/emscripten-core/emscripten/pull/12969 --- lib/binding_web/binding.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/binding_web/binding.js b/lib/binding_web/binding.js index b060715f..e22d5b5b 100644 --- a/lib/binding_web/binding.js +++ b/lib/binding_web/binding.js @@ -878,8 +878,14 @@ class Language { })); } + // emscripten-core/emscripten#12969 + const loadModule = + typeof loadSideModule === 'function' + ? loadSideModule + : loadWebAssemblyModule; + return bytes - .then(bytes => loadSideModule(bytes, {loadAsync: true})) + .then(bytes => loadModule(bytes, {loadAsync: true})) .then(mod => { const symbolNames = Object.keys(mod) const functionName = symbolNames.find(key => From e5ef2f2aa3c6d2cafcdd01e81f0852cb4cd21280 Mon Sep 17 00:00:00 2001 From: Jim Hester Date: Mon, 7 Dec 2020 09:05:03 -0500 Subject: [PATCH 09/27] Add link to R tree sitter grammar --- docs/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.md b/docs/index.md index eca3f1a9..1293ec48 100644 --- a/docs/index.md +++ b/docs/index.md @@ -46,6 +46,7 @@ Parsers for these languages are fairly complete: * [Python](https://github.com/tree-sitter/tree-sitter-python) * [Ruby](https://github.com/tree-sitter/tree-sitter-ruby) * [Rust](https://github.com/tree-sitter/tree-sitter-rust) +* [R](https://github.com/r-lib/tree-sitter-r) * [SystemRDL](https://github.com/SystemRDL/tree-sitter-systemrdl) * [TOML](https://github.com/ikatyang/tree-sitter-toml) * [TypeScript](https://github.com/tree-sitter/tree-sitter-typescript) From 0f492e4254aab125d5828ed95a506ddb684c9201 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 11 Dec 2020 13:47:20 -0800 Subject: [PATCH 10/27] Include ts_tree_copy in wasm build Fixes #846 --- lib/binding_web/exports.json | 1 + lib/binding_web/test/tree-test.js | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/lib/binding_web/exports.json b/lib/binding_web/exports.json index 72105158..d0173f3a 100644 --- a/lib/binding_web/exports.json +++ b/lib/binding_web/exports.json @@ -79,6 +79,7 @@ "_ts_query_predicates_for_pattern", "_ts_query_string_count", "_ts_query_string_value_for_id", + "_ts_tree_copy", "_ts_tree_cursor_current_field_id_wasm", "_ts_tree_cursor_current_node_id_wasm", "_ts_tree_cursor_current_node_is_missing_wasm", diff --git a/lib/binding_web/test/tree-test.js b/lib/binding_web/test/tree-test.js index ccb7a830..8c04e63e 100644 --- a/lib/binding_web/test/tree-test.js +++ b/lib/binding_web/test/tree-test.js @@ -323,6 +323,31 @@ describe("Tree", () => { assert(!cursor.gotoParent()); }) }); + + describe(".copy", () => { + it("creates another tree that remains stable if the original tree is edited", () => { + input = 'abc + cde'; + tree = parser.parse(input); + assert.equal( + tree.rootNode.toString(), + "(program (expression_statement (binary_expression left: (identifier) right: (identifier))))" + ); + + const tree2 = tree.copy(); + ([input, edit] = spliceInput(input, 3, 0, '123')); + assert.equal(input, 'abc123 + cde'); + tree.edit(edit); + + const leftNode = tree.rootNode.firstChild.firstChild.firstChild; + const leftNode2 = tree2.rootNode.firstChild.firstChild.firstChild; + const rightNode = tree.rootNode.firstChild.firstChild.lastChild; + const rightNode2 = tree2.rootNode.firstChild.firstChild.lastChild; + assert.equal(leftNode.endIndex, 6) + assert.equal(leftNode2.endIndex, 3) + assert.equal(rightNode.startIndex, 9) + assert.equal(rightNode2.startIndex, 6) + }); + }); }); function spliceInput(input, startIndex, lengthRemoved, newText) { From 4336d9c8c50ca700556be2ece99bf9d4d743c13f Mon Sep 17 00:00:00 2001 From: Henrique Oliveira Pinto Date: Fri, 11 Dec 2020 17:57:15 -0500 Subject: [PATCH 11/27] Update binding_web/README.md releases link The previous link pointed to a specific version and it made me think that the project had no releases for a year! --- lib/binding_web/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/binding_web/README.md b/lib/binding_web/README.md index ba1b4cb6..dc0d44ba 100644 --- a/lib/binding_web/README.md +++ b/lib/binding_web/README.md @@ -7,7 +7,7 @@ WebAssembly bindings to the [Tree-sitter](https://github.com/tree-sitter/tree-si ### Setup -You can download the the `tree-sitter.js` and `tree-sitter.wasm` files from [the latest GitHub release](https://github.com/tree-sitter/tree-sitter/releases/tag/0.14.7) and load them using a standalone script: +You can download the the `tree-sitter.js` and `tree-sitter.wasm` files from [the latest GitHub release](https://github.com/tree-sitter/tree-sitter/releases) and load them using a standalone script: ```html