Merge branch 'master' into wasm-language

cli/src/tests/helpers/allocations.rs

@@ -2,7 +2,7 @@ use std::{
     collections::HashMap,
     os::raw::c_void,
     sync::{
-        atomic::{AtomicBool, AtomicU64, Ordering::SeqCst},
+        atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst},
         Mutex,
     },
 };
@@ -25,8 +25,8 @@ unsafe impl Sync for Allocation {}
 #[derive(Default)]
 struct AllocationRecorder {
     enabled: AtomicBool,
-    allocation_count: AtomicU64,
-    outstanding_allocations: Mutex<HashMap<Allocation, u64>>,
+    allocation_count: AtomicUsize,
+    outstanding_allocations: Mutex<HashMap<Allocation, usize>>,
 }
 
 thread_local! {
@@ -83,6 +83,9 @@ fn record_alloc(ptr: *mut c_void) {
 }
 
 fn record_dealloc(ptr: *mut c_void) {
+    if ptr.is_null() {
+        panic!("Zero pointer deallocation!");
+    }
     RECORDER.with(|recorder| {
         if recorder.enabled.load(SeqCst) {
             recorder
@@ -107,9 +110,13 @@ unsafe extern "C" fn ts_record_calloc(count: usize, size: usize) -> *mut c_void
 }
 
 unsafe extern "C" fn ts_record_realloc(ptr: *mut c_void, size: usize) -> *mut c_void {
-    record_dealloc(ptr);
     let result = realloc(ptr, size);
-    record_alloc(result);
+    if ptr.is_null() {
+        record_alloc(result);
+    } else if ptr != result {
+        record_dealloc(ptr);
+        record_alloc(result);
+    }
     result
 }
 

cli/src/tests/helpers/dirs.rs

@@ -1,11 +1,46 @@
 lazy_static! {
-    static ref ROOT_DIR: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_owned();
-    static ref FIXTURES_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures");
-    static ref HEADER_DIR: PathBuf = ROOT_DIR.join("lib").join("include");
-    static ref GRAMMARS_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures").join("grammars");
-    static ref SCRATCH_DIR: PathBuf = {
+    pub static ref ROOT_DIR: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_owned();
+    pub static ref FIXTURES_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures");
+    pub static ref HEADER_DIR: PathBuf = ROOT_DIR.join("lib").join("include");
+    pub static ref GRAMMARS_DIR: PathBuf = ROOT_DIR.join("test").join("fixtures").join("grammars");
+    pub static ref SCRATCH_BASE_DIR: PathBuf = {
         let result = ROOT_DIR.join("target").join("scratch");
         fs::create_dir_all(&result).unwrap();
         result
     };
+    pub static ref SCRATCH_DIR: PathBuf = {
+        // https://doc.rust-lang.org/reference/conditional-compilation.html
+        let vendor = if cfg!(target_vendor = "apple") {
+            "apple"
+        } else if cfg!(target_vendor = "fortanix") {
+            "fortanix"
+        } else if cfg!(target_vendor = "pc") {
+            "pc"
+        } else {
+            "unknown"
+        };
+        let env = if cfg!(target_env = "gnu") {
+            "gnu"
+        } else if cfg!(target_env = "msvc") {
+            "msvc"
+        } else if cfg!(target_env = "musl") {
+            "musl"
+        } else if cfg!(target_env = "sgx") {
+            "sgx"
+        } else {
+            "unknown"
+        };
+        let endian = if cfg!(target_endian = "little") {
+            "little"
+        } else if cfg!(target_endian = "big") {
+            "big"
+        } else {
+            "unknown"
+        };
+
+        let machine = format!("{}-{}-{}-{}-{}", std::env::consts::ARCH, std::env::consts::OS, vendor, env, endian);
+        let result = SCRATCH_BASE_DIR.join(machine);
+        fs::create_dir_all(&result).unwrap();
+        result
+    };
 }

cli/src/tests/helpers/fixtures.rs

@@ -1,6 +1,6 @@
 use lazy_static::lazy_static;
-use std::fs;
 use std::path::{Path, PathBuf};
+use std::{env, fs};
 use tree_sitter::Language;
 use tree_sitter_highlight::HighlightConfiguration;
 use tree_sitter_loader::Loader;
@@ -9,7 +9,13 @@ use tree_sitter_tags::TagsConfiguration;
 include!("./dirs.rs");
 
 lazy_static! {
-    static ref TEST_LOADER: Loader = Loader::with_parser_lib_path(SCRATCH_DIR.join("lib"));
+    static ref TEST_LOADER: Loader = {
+        let mut loader = Loader::with_parser_lib_path(SCRATCH_DIR.clone());
+        if env::var("TREE_SITTER_GRAMMAR_DEBUG").is_ok() {
+            loader.use_debug_build(true);
+        }
+        loader
+    };
 }
 
 pub fn test_loader<'a>() -> &'a Loader {
@@ -46,9 +52,11 @@ pub fn get_highlight_config(
     let locals_query = fs::read_to_string(queries_path.join("locals.scm")).unwrap_or(String::new());
     let mut result = HighlightConfiguration::new(
         language,
+        language_name,
         &highlights_query,
         &injections_query,
         &locals_query,
+        false,
     )
     .unwrap();
     result.configure(&highlight_names);
@@ -63,11 +71,7 @@ pub fn get_tags_config(language_name: &str) -> TagsConfiguration {
     TagsConfiguration::new(language, &tags_query, &locals_query).unwrap()
 }
 
-pub fn get_test_language(
-    name: &str,
-    parser_code: &str,
-    scanner_src_path: Option<&Path>,
-) -> Language {
+pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> Language {
     let src_dir = SCRATCH_DIR.join("src").join(name);
     fs::create_dir_all(&src_dir).unwrap();
 
@@ -76,11 +80,16 @@ pub fn get_test_language(
         fs::write(&parser_path, parser_code).unwrap();
     }
 
-    if let Some(scanner_src_path) = scanner_src_path {
-        let scanner_code = fs::read_to_string(&scanner_src_path).unwrap();
-        let scanner_path = src_dir.join("scanner.c");
-        if !fs::read_to_string(&scanner_path).map_or(false, |content| content == scanner_code) {
-            fs::write(&scanner_path, scanner_code).unwrap();
+    if let Some(path) = path {
+        let scanner_path = path.join("scanner.c");
+        if scanner_path.exists() {
+            let scanner_code = fs::read_to_string(&scanner_path).unwrap();
+            let scanner_copy_path = src_dir.join("scanner.c");
+            if !fs::read_to_string(&scanner_copy_path)
+                .map_or(false, |content| content == scanner_code)
+            {
+                fs::write(&scanner_copy_path, scanner_code).unwrap();
+            }
         }
     }
 

cli/src/tests/helpers/mod.rs

@@ -6,7 +6,8 @@ pub(super) mod random;
 pub(super) mod scope_sequence;
 
 use lazy_static::lazy_static;
-use std::{env, time, usize};
+use rand::Rng;
+use std::env;
 
 lazy_static! {
     pub static ref LOG_ENABLED: bool = env::var("TREE_SITTER_LOG").is_ok();
@@ -16,11 +17,7 @@ lazy_static! {
 }
 
 lazy_static! {
-    pub static ref START_SEED: usize =
-        int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| time::SystemTime::now()
-            .duration_since(time::UNIX_EPOCH)
-            .unwrap()
-            .as_secs() as usize,);
+    pub static ref START_SEED: usize = new_seed();
     pub static ref EDIT_COUNT: usize = int_env_var("TREE_SITTER_EDITS").unwrap_or(3);
     pub static ref ITERATION_COUNT: usize = int_env_var("TREE_SITTER_ITERATIONS").unwrap_or(10);
 }
@@ -28,3 +25,10 @@ lazy_static! {
 fn int_env_var(name: &'static str) -> Option<usize> {
     env::var(name).ok().and_then(|e| e.parse().ok())
 }
+
+pub(crate) fn new_seed() -> usize {
+    int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| {
+        let mut rng = rand::thread_rng();
+        rng.gen::<usize>()
+    })
+}

cli/src/tests/helpers/query_helpers.rs

@@ -1,6 +1,8 @@
 use rand::prelude::Rng;
 use std::{cmp::Ordering, fmt::Write, ops::Range};
-use tree_sitter::{Node, Point, Tree, TreeCursor};
+use tree_sitter::{
+    Language, Node, Parser, Point, Query, QueryCapture, QueryCursor, QueryMatch, Tree, TreeCursor,
+};
 
 #[derive(Debug)]
 pub struct Pattern {
@@ -304,3 +306,56 @@ fn compare_depth_first(a: Node, b: Node) -> Ordering {
     let b = b.byte_range();
     a.start.cmp(&b.start).then_with(|| b.end.cmp(&a.end))
 }
+
+pub fn assert_query_matches(
+    language: Language,
+    query: &Query,
+    source: &str,
+    expected: &[(usize, Vec<(&str, &str)>)],
+) {
+    let mut parser = Parser::new();
+    parser.set_language(language).unwrap();
+    let tree = parser.parse(source, None).unwrap();
+    let mut cursor = QueryCursor::new();
+    let matches = cursor.matches(&query, tree.root_node(), source.as_bytes());
+    pretty_assertions::assert_eq!(collect_matches(matches, &query, source), expected);
+    pretty_assertions::assert_eq!(cursor.did_exceed_match_limit(), false);
+}
+
+pub fn collect_matches<'a>(
+    matches: impl Iterator<Item = QueryMatch<'a, 'a>>,
+    query: &'a Query,
+    source: &'a str,
+) -> Vec<(usize, Vec<(&'a str, &'a str)>)> {
+    matches
+        .map(|m| {
+            (
+                m.pattern_index,
+                format_captures(m.captures.iter().cloned(), query, source),
+            )
+        })
+        .collect()
+}
+
+pub fn collect_captures<'a>(
+    captures: impl Iterator<Item = (QueryMatch<'a, 'a>, usize)>,
+    query: &'a Query,
+    source: &'a str,
+) -> Vec<(&'a str, &'a str)> {
+    format_captures(captures.map(|(m, i)| m.captures[i]), query, source)
+}
+
+fn format_captures<'a>(
+    captures: impl Iterator<Item = QueryCapture<'a>>,
+    query: &'a Query,
+    source: &'a str,
+) -> Vec<(&'a str, &'a str)> {
+    captures
+        .map(|capture| {
+            (
+                query.capture_names()[capture.index as usize],
+                capture.node.utf8_text(source.as_bytes()).unwrap(),
+            )
+        })
+        .collect()
+}