From 62f8c431aeee91894c79a5cf507399a6f764a4ba Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Thu, 16 Mar 2023 15:02:04 +0200 Subject: [PATCH 1/4] test: add `retry` and `test_with_seed` proc macros --- Cargo.lock | 11 +++ cli/Cargo.toml | 2 + cli/src/tests/proc_macro/Cargo.toml | 14 +++ cli/src/tests/proc_macro/src/lib.rs | 137 ++++++++++++++++++++++++++++ 4 files changed, 164 insertions(+) create mode 100644 cli/src/tests/proc_macro/Cargo.toml create mode 100644 cli/src/tests/proc_macro/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 7a675454..404c269f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -464,6 +464,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc_macro" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "rand", + "syn", +] + [[package]] name = "quote" version = "1.0.26" @@ -746,6 +756,7 @@ dependencies = [ "lazy_static", "log", "pretty_assertions", + "proc_macro", "rand", "regex", "regex-syntax", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 6d48e8b9..47e03284 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -69,6 +69,8 @@ version = "0.4.6" features = ["std"] [dev-dependencies] +proc_macro = { path = "src/tests/proc_macro" } + rand = "0.8" tempfile = "3" pretty_assertions = "0.7.2" diff --git a/cli/src/tests/proc_macro/Cargo.toml b/cli/src/tests/proc_macro/Cargo.toml new file mode 100644 index 00000000..a9a2b146 --- /dev/null +++ b/cli/src/tests/proc_macro/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "proc_macro" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1" +quote = "1" +rand = "0.8.5" +syn = { version = "1", features = ["full"] } diff --git a/cli/src/tests/proc_macro/src/lib.rs b/cli/src/tests/proc_macro/src/lib.rs new file mode 100644 index 00000000..d831a75b --- /dev/null +++ b/cli/src/tests/proc_macro/src/lib.rs @@ -0,0 +1,137 @@ +use proc_macro::TokenStream; +use proc_macro2::Span; +use quote::quote; +use syn::{ + parse::{Parse, ParseStream}, + parse_macro_input, Error, Expr, Ident, ItemFn, LitInt, Token, +}; + +#[proc_macro_attribute] +pub fn retry(args: TokenStream, input: TokenStream) -> TokenStream { + let count = parse_macro_input!(args as LitInt); + let input = parse_macro_input!(input as ItemFn); + let attrs = input.attrs.clone(); + let name = input.sig.ident.clone(); + + TokenStream::from(quote! { + #(#attrs),* + fn #name() { + #input + + for i in 0..=#count { + let result = std::panic::catch_unwind(|| { + #name(); + }); + + if result.is_ok() { + return; + } + + if i == #count { + std::panic::resume_unwind(result.unwrap_err()); + } + } + } + }) +} + +#[proc_macro_attribute] +pub fn test_with_seed(args: TokenStream, input: TokenStream) -> TokenStream { + struct Args { + retry: LitInt, + seed: Expr, + seed_fn: Option, + } + + impl Parse for Args { + fn parse(input: ParseStream) -> syn::Result { + let mut retry = None; + let mut seed = None; + let mut seed_fn = None; + + while !input.is_empty() { + let name = input.parse::()?; + match name.to_string().as_str() { + "retry" => { + input.parse::()?; + retry.replace(input.parse()?); + } + "seed" => { + input.parse::()?; + seed.replace(input.parse()?); + } + "seed_fn" => { + input.parse::()?; + seed_fn.replace(input.parse()?); + } + x => { + return Err(Error::new( + name.span(), + format!("Unsupported parameter `{x}`"), + )) + } + } + + if !input.is_empty() { + input.parse::()?; + } + } + + if retry.is_none() { + retry.replace(LitInt::new("0", Span::mixed_site())); + } + + Ok(Args { + retry: retry.expect("`retry` parameter is requred"), + seed: seed.expect("`initial_seed` parameter is required"), + seed_fn, + }) + } + } + + let Args { + retry, + seed, + seed_fn, + } = parse_macro_input!(args as Args); + + let seed_fn = seed_fn.iter(); + + let func = parse_macro_input!(input as ItemFn); + let attrs = func.attrs.clone(); + let name = func.sig.ident.clone(); + + // dbg!(quote::ToTokens::into_token_stream(&func)); + + TokenStream::from(quote! { + #[test] + #(#attrs),* + fn #name() { + #func + + let mut seed = #seed; + + for i in 0..=#retry { + let result = std::panic::catch_unwind(|| { + #name(seed); + }); + + if result.is_ok() { + return; + } + + if i == #retry { + std::panic::resume_unwind(result.unwrap_err()); + } + + #( + seed = #seed_fn(); + )* + + if i < #retry { + println!("\nRetry {}/{} with a new seed {}", i + 1, #retry, seed); + } + } + } + }) +} From 588549c09388dd6586cfbb06c06f8eeba436f38f Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Thu, 16 Mar 2023 15:02:59 +0200 Subject: [PATCH 2/4] test: run `test_parsing_with_a_timeout` with 10 retries --- cli/src/tests/parser_test.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cli/src/tests/parser_test.rs b/cli/src/tests/parser_test.rs index 30b12336..78c6cda4 100644 --- a/cli/src/tests/parser_test.rs +++ b/cli/src/tests/parser_test.rs @@ -8,6 +8,7 @@ use crate::{ generate::generate_parser_for_grammar, parse::{perform_edit, Edit}, }; +use proc_macro::retry; use std::{ sync::atomic::{AtomicUsize, Ordering}, thread, time, @@ -638,6 +639,7 @@ fn test_parsing_cancelled_by_another_thread() { // Timeouts #[test] +#[retry(10)] fn test_parsing_with_a_timeout() { let mut parser = Parser::new(); parser.set_language(get_language("json")).unwrap(); From 3aeef44eb651b77b7392d5ee89fa4d92273b8e0c Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Thu, 16 Mar 2023 15:08:23 +0200 Subject: [PATCH 3/4] test: run all corpus tests with 10 retries --- cli/src/tests/corpus_test.rs | 95 +++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 45 deletions(-) diff --git a/cli/src/tests/corpus_test.rs b/cli/src/tests/corpus_test.rs index 401a99a5..b818b2c1 100644 --- a/cli/src/tests/corpus_test.rs +++ b/cli/src/tests/corpus_test.rs @@ -2,6 +2,7 @@ use super::helpers::{ allocations, edits::{get_random_edit, invert_edit}, fixtures::{fixtures_dir, get_language, get_test_language}, + new_seed, random::Rand, scope_sequence::ScopeSequence, EDIT_COUNT, EXAMPLE_FILTER, ITERATION_COUNT, LANGUAGE_FILTER, LOG_ENABLED, LOG_GRAPH_ENABLED, @@ -13,70 +14,71 @@ use crate::{ test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields, TestEntry}, util, }; -use std::fs; +use proc_macro::test_with_seed; +use std::{env, fs}; use tree_sitter::{LogType, Node, Parser, Point, Range, Tree}; -#[test] -fn test_bash_corpus() { - test_language_corpus("bash"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_bash(seed: usize) { + test_language_corpus(seed, "bash"); } -#[test] -fn test_c_corpus() { - test_language_corpus("c"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_c(seed: usize) { + test_language_corpus(seed, "c"); } -#[test] -fn test_cpp_corpus() { - test_language_corpus("cpp"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_cpp(seed: usize) { + test_language_corpus(seed, "cpp"); } -#[test] -fn test_embedded_template_corpus() { - test_language_corpus("embedded-template"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_embedded_template(seed: usize) { + test_language_corpus(seed, "embedded-template"); } -#[test] -fn test_go_corpus() { - test_language_corpus("go"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_go(seed: usize) { + test_language_corpus(seed, "go"); } -#[test] -fn test_html_corpus() { - test_language_corpus("html"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_html(seed: usize) { + test_language_corpus(seed, "html"); } -#[test] -fn test_javascript_corpus() { - test_language_corpus("javascript"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_javascript(seed: usize) { + test_language_corpus(seed, "javascript"); } -#[test] -fn test_json_corpus() { - test_language_corpus("json"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_json(seed: usize) { + test_language_corpus(seed, "json"); } -#[test] -fn test_php_corpus() { - test_language_corpus("php"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_php(seed: usize) { + test_language_corpus(seed, "php"); } -#[test] -fn test_python_corpus() { - test_language_corpus("python"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_python(seed: usize) { + test_language_corpus(seed, "python"); } -#[test] -fn test_ruby_corpus() { - test_language_corpus("ruby"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_ruby(seed: usize) { + test_language_corpus(seed, "ruby"); } -#[test] -fn test_rust_corpus() { - test_language_corpus("rust"); +#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] +fn test_corpus_for_rust(seed: usize) { + test_language_corpus(seed, "rust"); } -fn test_language_corpus(language_name: &str) { +fn test_language_corpus(start_seed: usize, language_name: &str) { let grammars_dir = fixtures_dir().join("grammars"); let error_corpus_dir = fixtures_dir().join("error_corpus"); let template_corpus_dir = fixtures_dir().join("template_corpus"); @@ -100,6 +102,10 @@ fn test_language_corpus(language_name: &str) { let language = get_language(language_name); let mut failure_count = 0; + + let log_seed = env::var("TREE_SITTER_LOG_SEED").is_ok(); + + println!(); for test in tests { println!(" {} example - {}", language_name, test.name); @@ -140,7 +146,7 @@ fn test_language_corpus(language_name: &str) { drop(parser); for trial in 0..*ITERATION_COUNT { - let seed = *START_SEED + trial; + let seed = start_seed + trial; let passed = allocations::record(|| { let mut rand = Rand::new(seed); let mut log_session = None; @@ -161,7 +167,9 @@ fn test_language_corpus(language_name: &str) { perform_edit(&mut tree, &mut input, &edit); } - // println!(" seed: {}", seed); + if log_seed { + println!(" seed: {}", seed); + } if *LOG_GRAPH_ENABLED { eprintln!("{}\n", String::from_utf8_lossy(&input)); @@ -173,10 +181,7 @@ fn test_language_corpus(language_name: &str) { // Check that the new tree is consistent. check_consistent_sizes(&tree2, &input); if let Err(message) = check_changed_ranges(&tree, &tree2, &input) { - println!( - "\nUnexpected scope change in seed {}\n{}\n\n", - seed, message - ); + println!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",); return false; } @@ -211,7 +216,7 @@ fn test_language_corpus(language_name: &str) { // Check that the edited tree is consistent. check_consistent_sizes(&tree3, &input); if let Err(message) = check_changed_ranges(&tree2, &tree3, &input) { - eprintln!("Unexpected scope change in seed {}\n{}\n\n", seed, message); + println!("Unexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n"); return false; } From ddb0af95098cf04a307d5ea6e93d77a00643c5d7 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Thu, 16 Mar 2023 15:08:56 +0200 Subject: [PATCH 4/4] test: use random SEED numbers This is needed to omit occurrences of the same seed in a sequence of following seeds due to the reason of that two initial seed are very close if based on unix epoch seconds. --- cli/src/tests/helpers/mod.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cli/src/tests/helpers/mod.rs b/cli/src/tests/helpers/mod.rs index def0ea3e..54df8809 100644 --- a/cli/src/tests/helpers/mod.rs +++ b/cli/src/tests/helpers/mod.rs @@ -6,7 +6,8 @@ pub(super) mod random; pub(super) mod scope_sequence; use lazy_static::lazy_static; -use std::{env, time, usize}; +use rand::Rng; +use std::env; lazy_static! { pub static ref LOG_ENABLED: bool = env::var("TREE_SITTER_LOG").is_ok(); @@ -16,11 +17,7 @@ lazy_static! { } lazy_static! { - pub static ref START_SEED: usize = - int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| time::SystemTime::now() - .duration_since(time::UNIX_EPOCH) - .unwrap() - .as_secs() as usize,); + pub static ref START_SEED: usize = new_seed(); pub static ref EDIT_COUNT: usize = int_env_var("TREE_SITTER_EDITS").unwrap_or(3); pub static ref ITERATION_COUNT: usize = int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10); } @@ -28,3 +25,10 @@ lazy_static! { fn int_env_var(name: &'static str) -> Option { env::var(name).ok().and_then(|e| e.parse().ok()) } + +pub(crate) fn new_seed() -> usize { + int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| { + let mut rng = rand::thread_rng(); + rng.gen::() + }) +}