Merge pull request #2135 from tree-sitter/no-flaky-tests

No flaky tests - make all tests run in a reliable way; V2

Cargo.lock

@@ -464,6 +464,16 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "proc_macro"
+version = "0.1.0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "rand",
+ "syn",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.26"
@@ -746,6 +756,7 @@ dependencies = [
  "lazy_static",
  "log",
  "pretty_assertions",
+ "proc_macro",
  "rand",
  "regex",
  "regex-syntax",

cli/Cargo.toml

@@ -69,6 +69,8 @@ version = "0.4.6"
 features = ["std"]
 
 [dev-dependencies]
+proc_macro = { path = "src/tests/proc_macro" }
+
 rand = "0.8"
 tempfile = "3"
 pretty_assertions = "0.7.2"

cli/src/tests/corpus_test.rs

@@ -2,6 +2,7 @@ use super::helpers::{
     allocations,
     edits::{get_random_edit, invert_edit},
     fixtures::{fixtures_dir, get_language, get_test_language},
+    new_seed,
     random::Rand,
     scope_sequence::ScopeSequence,
     EDIT_COUNT, EXAMPLE_FILTER, ITERATION_COUNT, LANGUAGE_FILTER, LOG_ENABLED, LOG_GRAPH_ENABLED,
@@ -13,70 +14,71 @@ use crate::{
     test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields, TestEntry},
     util,
 };
-use std::fs;
+use proc_macro::test_with_seed;
+use std::{env, fs};
 use tree_sitter::{LogType, Node, Parser, Point, Range, Tree};
 
-#[test]
-fn test_bash_corpus() {
-    test_language_corpus("bash");
+#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
+fn test_corpus_for_bash(seed: usize) {
+    test_language_corpus(seed, "bash");
 }
 
-#[test]
-fn test_c_corpus() {
-    test_language_corpus("c");
+#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
+fn test_corpus_for_c(seed: usize) {
+    test_language_corpus(seed, "c");
 }
 
-#[test]
-fn test_cpp_corpus() {
-    test_language_corpus("cpp");
+#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
+fn test_corpus_for_cpp(seed: usize) {
+    test_language_corpus(seed, "cpp");
 }
 
-#[test]
-fn test_embedded_template_corpus() {
-    test_language_corpus("embedded-template");
+#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
+fn test_corpus_for_embedded_template(seed: usize) {
+    test_language_corpus(seed, "embedded-template");
 }
 
-#[test]
-fn test_go_corpus() {
-    test_language_corpus("go");
+#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
+fn test_corpus_for_go(seed: usize) {
+    test_language_corpus(seed, "go");
 }
 
-#[test]
-fn test_html_corpus() {
-    test_language_corpus("html");
+#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
+fn test_corpus_for_html(seed: usize) {
+    test_language_corpus(seed, "html");
 }
 
-#[test]
-fn test_javascript_corpus() {
-    test_language_corpus("javascript");
+#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
+fn test_corpus_for_javascript(seed: usize) {
+    test_language_corpus(seed, "javascript");
 }
 
-#[test]
-fn test_json_corpus() {
-    test_language_corpus("json");
+#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
+fn test_corpus_for_json(seed: usize) {
+    test_language_corpus(seed, "json");
 }
 
-#[test]
-fn test_php_corpus() {
-    test_language_corpus("php");
+#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
+fn test_corpus_for_php(seed: usize) {
+    test_language_corpus(seed, "php");
 }
 
-#[test]
-fn test_python_corpus() {
-    test_language_corpus("python");
+#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
+fn test_corpus_for_python(seed: usize) {
+    test_language_corpus(seed, "python");
 }
 
-#[test]
-fn test_ruby_corpus() {
-    test_language_corpus("ruby");
+#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
+fn test_corpus_for_ruby(seed: usize) {
+    test_language_corpus(seed, "ruby");
 }
 
-#[test]
-fn test_rust_corpus() {
-    test_language_corpus("rust");
+#[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)]
+fn test_corpus_for_rust(seed: usize) {
+    test_language_corpus(seed, "rust");
 }
 
-fn test_language_corpus(language_name: &str) {
+fn test_language_corpus(start_seed: usize, language_name: &str) {
     let grammars_dir = fixtures_dir().join("grammars");
     let error_corpus_dir = fixtures_dir().join("error_corpus");
     let template_corpus_dir = fixtures_dir().join("template_corpus");
@@ -100,6 +102,10 @@ fn test_language_corpus(language_name: &str) {
 
     let language = get_language(language_name);
     let mut failure_count = 0;
+
+    let log_seed = env::var("TREE_SITTER_LOG_SEED").is_ok();
+
+    println!();
     for test in tests {
         println!("  {} example - {}", language_name, test.name);
 
@@ -140,7 +146,7 @@ fn test_language_corpus(language_name: &str) {
         drop(parser);
 
         for trial in 0..*ITERATION_COUNT {
-            let seed = *START_SEED + trial;
+            let seed = start_seed + trial;
             let passed = allocations::record(|| {
                 let mut rand = Rand::new(seed);
                 let mut log_session = None;
@@ -161,7 +167,9 @@ fn test_language_corpus(language_name: &str) {
                     perform_edit(&mut tree, &mut input, &edit);
                 }
 
-                // println!("    seed: {}", seed);
+                if log_seed {
+                    println!("    seed: {}", seed);
+                }
 
                 if *LOG_GRAPH_ENABLED {
                     eprintln!("{}\n", String::from_utf8_lossy(&input));
@@ -173,10 +181,7 @@ fn test_language_corpus(language_name: &str) {
                 // Check that the new tree is consistent.
                 check_consistent_sizes(&tree2, &input);
                 if let Err(message) = check_changed_ranges(&tree, &tree2, &input) {
-                    println!(
-                        "\nUnexpected scope change in seed {}\n{}\n\n",
-                        seed, message
-                    );
+                    println!("\nUnexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n",);
                     return false;
                 }
 
@@ -211,7 +216,7 @@ fn test_language_corpus(language_name: &str) {
                 // Check that the edited tree is consistent.
                 check_consistent_sizes(&tree3, &input);
                 if let Err(message) = check_changed_ranges(&tree2, &tree3, &input) {
-                    eprintln!("Unexpected scope change in seed {}\n{}\n\n", seed, message);
+                    println!("Unexpected scope change in seed {seed} with start seed {start_seed}\n{message}\n\n");
                     return false;
                 }
 

cli/src/tests/helpers/mod.rs

@@ -6,7 +6,8 @@ pub(super) mod random;
 pub(super) mod scope_sequence;
 
 use lazy_static::lazy_static;
-use std::{env, time, usize};
+use rand::Rng;
+use std::env;
 
 lazy_static! {
     pub static ref LOG_ENABLED: bool = env::var("TREE_SITTER_LOG").is_ok();
@@ -16,11 +17,7 @@ lazy_static! {
 }
 
 lazy_static! {
-    pub static ref START_SEED: usize =
-        int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| time::SystemTime::now()
-            .duration_since(time::UNIX_EPOCH)
-            .unwrap()
-            .as_secs() as usize,);
+    pub static ref START_SEED: usize = new_seed();
     pub static ref EDIT_COUNT: usize = int_env_var("TREE_SITTER_EDITS").unwrap_or(3);
     pub static ref ITERATION_COUNT: usize = int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10);
 }
@@ -28,3 +25,10 @@ lazy_static! {
 fn int_env_var(name: &'static str) -> Option<usize> {
     env::var(name).ok().and_then(|e| e.parse().ok())
 }
+
+pub(crate) fn new_seed() -> usize {
+    int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| {
+        let mut rng = rand::thread_rng();
+        rng.gen::<usize>()
+    })
+}

cli/src/tests/parser_test.rs

@@ -8,6 +8,7 @@ use crate::{
     generate::generate_parser_for_grammar,
     parse::{perform_edit, Edit},
 };
+use proc_macro::retry;
 use std::{
     sync::atomic::{AtomicUsize, Ordering},
     thread, time,
@@ -638,6 +639,7 @@ fn test_parsing_cancelled_by_another_thread() {
 // Timeouts
 
 #[test]
+#[retry(10)]
 fn test_parsing_with_a_timeout() {
     let mut parser = Parser::new();
     parser.set_language(get_language("json")).unwrap();

cli/src/tests/proc_macro/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "proc_macro"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2 = "1"
+quote = "1"
+rand = "0.8.5"
+syn = { version = "1", features = ["full"] }

cli/src/tests/proc_macro/src/lib.rs

@@ -0,0 +1,137 @@
+use proc_macro::TokenStream;
+use proc_macro2::Span;
+use quote::quote;
+use syn::{
+    parse::{Parse, ParseStream},
+    parse_macro_input, Error, Expr, Ident, ItemFn, LitInt, Token,
+};
+
+#[proc_macro_attribute]
+pub fn retry(args: TokenStream, input: TokenStream) -> TokenStream {
+    let count = parse_macro_input!(args as LitInt);
+    let input = parse_macro_input!(input as ItemFn);
+    let attrs = input.attrs.clone();
+    let name = input.sig.ident.clone();
+
+    TokenStream::from(quote! {
+        #(#attrs),*
+        fn #name() {
+            #input
+
+            for i in 0..=#count {
+                let result = std::panic::catch_unwind(|| {
+                    #name();
+                });
+
+                if result.is_ok() {
+                    return;
+                }
+
+                if i == #count {
+                    std::panic::resume_unwind(result.unwrap_err());
+                }
+            }
+        }
+    })
+}
+
+#[proc_macro_attribute]
+pub fn test_with_seed(args: TokenStream, input: TokenStream) -> TokenStream {
+    struct Args {
+        retry: LitInt,
+        seed: Expr,
+        seed_fn: Option<Ident>,
+    }
+
+    impl Parse for Args {
+        fn parse(input: ParseStream) -> syn::Result<Self> {
+            let mut retry = None;
+            let mut seed = None;
+            let mut seed_fn = None;
+
+            while !input.is_empty() {
+                let name = input.parse::<Ident>()?;
+                match name.to_string().as_str() {
+                    "retry" => {
+                        input.parse::<Token![=]>()?;
+                        retry.replace(input.parse()?);
+                    }
+                    "seed" => {
+                        input.parse::<Token![=]>()?;
+                        seed.replace(input.parse()?);
+                    }
+                    "seed_fn" => {
+                        input.parse::<Token![=]>()?;
+                        seed_fn.replace(input.parse()?);
+                    }
+                    x => {
+                        return Err(Error::new(
+                            name.span(),
+                            format!("Unsupported parameter `{x}`"),
+                        ))
+                    }
+                }
+
+                if !input.is_empty() {
+                    input.parse::<Token![,]>()?;
+                }
+            }
+
+            if retry.is_none() {
+                retry.replace(LitInt::new("0", Span::mixed_site()));
+            }
+
+            Ok(Args {
+                retry: retry.expect("`retry` parameter is requred"),
+                seed: seed.expect("`initial_seed` parameter is required"),
+                seed_fn,
+            })
+        }
+    }
+
+    let Args {
+        retry,
+        seed,
+        seed_fn,
+    } = parse_macro_input!(args as Args);
+
+    let seed_fn = seed_fn.iter();
+
+    let func = parse_macro_input!(input as ItemFn);
+    let attrs = func.attrs.clone();
+    let name = func.sig.ident.clone();
+
+    // dbg!(quote::ToTokens::into_token_stream(&func));
+
+    TokenStream::from(quote! {
+        #[test]
+        #(#attrs),*
+        fn #name() {
+            #func
+
+            let mut seed = #seed;
+
+            for i in 0..=#retry {
+                let result = std::panic::catch_unwind(|| {
+                    #name(seed);
+                });
+
+                if result.is_ok() {
+                    return;
+                }
+
+                if i == #retry {
+                    std::panic::resume_unwind(result.unwrap_err());
+                }
+
+                #(
+                    seed = #seed_fn();
+                )*
+
+                if i < #retry {
+                    println!("\nRetry {}/{} with a new seed {}", i + 1, #retry, seed);
+                }
+            }
+        }
+    })
+}