diff --git a/cli/src/tests/corpuses.rs b/cli/src/tests/corpuses.rs index 76ed02d0..9b60d685 100644 --- a/cli/src/tests/corpuses.rs +++ b/cli/src/tests/corpuses.rs @@ -1,6 +1,7 @@ use super::allocations; use super::fixtures::{fixtures_dir, get_language, get_test_language}; use super::random::Rand; +use super::scope_sequence::ScopeSequence; use crate::generate; use crate::test::{parse_tests, print_diff, print_diff_key, TestEntry}; use crate::util; @@ -125,7 +126,11 @@ fn test_real_language_corpus_files() { // Check that the new tree is consistent. check_consistent_sizes(&tree2, &input); - check_changed_ranges(&tree, &tree2, &input); + if let Err(message) = check_changed_ranges(&tree, &tree2, &input) { + println!("\nUnexpected scope change in trial {}\n{}\n\n", trial, message); + failure_count += 1; + break; + } // Undo all of the edits and re-parse again. while let Some(edit) = undo_stack.pop() { @@ -139,19 +144,26 @@ fn test_real_language_corpus_files() { .parse_utf8(&mut |i, _| input.get(i..).unwrap_or(&[]), Some(&tree2)) .unwrap(); - // Check that the edited tree is consistent. - check_consistent_sizes(&tree3, &input); - check_changed_ranges(&tree2, &tree3, &input); - // Verify that the final tree matches the expectation from the corpus. let actual_output = tree3.root_node().to_sexp(); if actual_output != expected_output { - println!("Incorrect parse for {} - {} - trial {}", language_name, example_name, trial); + println!( + "Incorrect parse for {} - {} - trial {}", + language_name, example_name, trial + ); print_diff_key(); print_diff(&actual_output, &expected_output); println!(""); failure_count += 1; - // break; + break; + } + + // Check that the edited tree is consistent. + check_consistent_sizes(&tree3, &input); + if let Err(message) = check_changed_ranges(&tree2, &tree3, &input) { + eprintln!("Unexpected scope change in trial {}\n{}\n\n", trial, message); + failure_count += 1; + break; } drop(tree); @@ -348,7 +360,12 @@ fn position_for_offset(input: &Vec, offset: usize) -> Point { fn check_consistent_sizes(tree: &Tree, input: &Vec) {} -fn check_changed_ranges(old_tree: &Tree, new_tree: &Tree, input: &Vec) {} +fn check_changed_ranges(old_tree: &Tree, new_tree: &Tree, input: &Vec) -> Result<(), String> { + let changed_ranges = old_tree.changed_ranges(new_tree); + let old_scope_sequence = ScopeSequence::new(old_tree); + let new_scope_sequence = ScopeSequence::new(new_tree); + old_scope_sequence.check_changes(&new_scope_sequence, &input, &changed_ranges) +} fn get_parser(session: &mut Option, log_filename: &str) -> Parser { let mut parser = Parser::new(); diff --git a/cli/src/tests/mod.rs b/cli/src/tests/mod.rs index 3a0c607a..fa841382 100644 --- a/cli/src/tests/mod.rs +++ b/cli/src/tests/mod.rs @@ -3,3 +3,4 @@ mod corpuses; mod fixtures; mod random; mod parser_api; +mod scope_sequence; diff --git a/cli/src/tests/scope_sequence.rs b/cli/src/tests/scope_sequence.rs new file mode 100644 index 00000000..685fe91f --- /dev/null +++ b/cli/src/tests/scope_sequence.rs @@ -0,0 +1,97 @@ +use tree_sitter::{Point, Range, Tree}; + +#[derive(Debug)] +pub struct ScopeSequence(Vec); + +type ScopeStack = Vec<&'static str>; + +impl ScopeSequence { + pub fn new(tree: &Tree) -> Self { + let mut result = ScopeSequence(Vec::new()); + let mut scope_stack = Vec::new(); + + let mut cursor = tree.walk(); + let mut visited_children = false; + loop { + let node = cursor.node(); + for _ in result.0.len()..node.start_byte() { + result.0.push(scope_stack.clone()); + } + if visited_children { + for _ in result.0.len()..node.end_byte() { + result.0.push(scope_stack.clone()); + } + scope_stack.pop(); + if cursor.goto_next_sibling() { + visited_children = false; + } else if !cursor.goto_parent() { + break; + } + } else { + scope_stack.push(cursor.node().kind()); + if !cursor.goto_first_child() { + visited_children = true; + } + } + } + + result + } + + pub fn check_changes( + &self, + other: &ScopeSequence, + text: &Vec, + known_changed_ranges: &Vec, + ) -> Result<(), String> { + if self.0.len() != text.len() { + panic!( + "Inconsistent scope sequence: {:?}", + self.0.iter().zip(text.iter().map(|c| *c as char)).collect::>() + ); + } + + assert_eq!(self.0.len(), other.0.len()); + let mut position = Point { row: 0, column: 0 }; + for (i, stack) in self.0.iter().enumerate() { + let other_stack = &other.0[i]; + if *stack != *other_stack { + let containing_range = known_changed_ranges + .iter() + .find(|range| range.start_point <= position && position < range.end_point); + if containing_range.is_none() { + let line = &text[(i - position.column)..] + .split(|c| *c == '\n' as u8) + .next() + .unwrap(); + return Err(format!( + concat!( + "Position: {}\n", + "Byte offset: {}\n", + "Line: {}\n", + "{}^\n", + "Old scopes: {:?}\n", + "New scopes: {:?}\n", + "Invalidated ranges: {:?}", + ), + position, + i, + String::from_utf8_lossy(line), + String::from(" ").repeat(position.column + "Line: ".len()), + stack, + other_stack, + known_changed_ranges, + )); + } + } + + if text[i] == '\n' as u8 { + position.row += 1; + position.column = 0; + } else { + position.column += 1; + } + } + Ok(()) + } +} diff --git a/lib/binding/helper.c b/lib/binding/helper.c new file mode 100644 index 00000000..4275e445 --- /dev/null +++ b/lib/binding/helper.c @@ -0,0 +1,17 @@ +#if defined(TREE_SITTER_TEST) + +void ts_record_free(void *); + +void rust_tree_sitter_free(void *p) { + ts_record_free(p); +} + +#else + +void free(void *); + +void rust_tree_sitter_free(void *p) { + free(p); +} + +#endif diff --git a/lib/binding/lib.rs b/lib/binding/lib.rs index 1f29e28a..150dfcf4 100644 --- a/lib/binding/lib.rs +++ b/lib/binding/lib.rs @@ -436,7 +436,7 @@ impl Tree { ffi::ts_tree_get_changed_ranges(self.0, other.0, &mut count as *mut _ as *mut u32); let ranges = slice::from_raw_parts(ptr, count); let result = ranges.into_iter().map(|r| r.clone().into()).collect(); - free(ptr as *mut c_void); + free_ptr(ptr as *mut c_void); result } } @@ -576,7 +576,7 @@ impl<'tree> Node<'tree> { .to_str() .unwrap() .to_string(); - unsafe { free(c_string as *mut c_void) }; + unsafe { free_ptr(c_string as *mut c_void) }; result } @@ -882,5 +882,6 @@ impl

PropertySheet

{ } extern "C" { - fn free(pointer: *mut c_void); + #[link_name = "rust_tree_sitter_free"] + fn free_ptr(ptr: *mut c_void); } diff --git a/lib/build.rs b/lib/build.rs index df66ee7c..7cca001c 100644 --- a/lib/build.rs +++ b/lib/build.rs @@ -41,6 +41,7 @@ fn main() { .include("include") .include("utf8proc") .file(src_path.join("lib.c")) + .file(Path::new("binding").join("helper.c")) .compile("tree-sitter"); } diff --git a/script/test.cmd b/script/test.cmd index e62eed0e..ef4ce02e 100644 --- a/script/test.cmd +++ b/script/test.cmd @@ -4,4 +4,4 @@ set TREE_SITTER_TEST=1 set RUST_TEST_THREADS=1 set RUST_BACKTRACE=full -cargo test "%~1" +cargo test "%~1" -- --nocapture