From 6b8483c53c9ac765fbf1114ff902a946b9353f4c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Sat, 2 Feb 2019 21:37:54 -0800 Subject: [PATCH] Start work on porting included range unit tests --- cli/src/tests/parser_test.rs | 144 ++++++++++++++++++++++++++++++++++- lib/binding/lib.rs | 26 ++++++- 2 files changed, 168 insertions(+), 2 deletions(-) diff --git a/cli/src/tests/parser_test.rs b/cli/src/tests/parser_test.rs index 43fbbc1b..8a11d22a 100644 --- a/cli/src/tests/parser_test.rs +++ b/cli/src/tests/parser_test.rs @@ -1,6 +1,6 @@ use super::helpers::fixtures::get_language; use std::thread; -use tree_sitter::{InputEdit, Language, LogType, Parser, Point}; +use tree_sitter::{InputEdit, Language, LogType, Parser, Point, Range}; #[test] fn test_basic_parsing() { @@ -260,6 +260,148 @@ fn test_parsing_on_multiple_threads() { assert_eq!(child_count_differences, &[1, 2, 3, 4]); } +// Included Ranges + +#[test] +fn test_parsing_with_one_included_range() { + let source_code = "hi"; + + let mut parser = Parser::new(); + parser.set_language(get_language("html")).unwrap(); + let html_tree = parser.parse_str(source_code, None).unwrap(); + let script_content_node = html_tree.root_node().child(1).unwrap().child(1).unwrap(); + assert_eq!(script_content_node.kind(), "raw_text"); + + parser.set_included_ranges(&[script_content_node.range()]); + parser.set_language(get_language("javascript")).unwrap(); + let js_tree = parser.parse_str(source_code, None).unwrap(); + + assert_eq!( + js_tree.root_node().to_sexp(), + concat!( + "(program (expression_statement (call_expression", + " (member_expression (identifier) (property_identifier))", + " (arguments (string)))))", + ) + ); + assert_eq!( + js_tree.root_node().start_position(), + Point::new(0, source_code.find("console").unwrap()) + ); +} + +#[test] +fn test_parsing_with_multiple_included_ranges() { + let source_code = "html `
Hello, ${name.toUpperCase()}, it's ${now()}.
`"; + + let mut parser = Parser::new(); + parser.set_language(get_language("javascript")).unwrap(); + let js_tree = parser.parse_str(source_code, None).unwrap(); + let template_string_node = js_tree + .root_node() + .descendant_for_byte_range( + source_code.find("
").unwrap(), + source_code.find("Hello").unwrap(), + ) + .unwrap(); + assert_eq!(template_string_node.kind(), "template_string"); + + let open_quote_node = template_string_node.child(0).unwrap(); + let interpolation_node1 = template_string_node.child(1).unwrap(); + let interpolation_node2 = template_string_node.child(2).unwrap(); + let close_quote_node = template_string_node.child(3).unwrap(); + + parser.set_language(get_language("html")).unwrap(); + parser.set_included_ranges(&[ + Range { + start_byte: open_quote_node.end_byte(), + start_point: open_quote_node.end_position(), + end_byte: interpolation_node1.start_byte(), + end_point: interpolation_node1.start_position(), + }, + Range { + start_byte: interpolation_node1.end_byte(), + start_point: interpolation_node1.end_position(), + end_byte: interpolation_node2.start_byte(), + end_point: interpolation_node2.start_position(), + }, + Range { + start_byte: interpolation_node2.end_byte(), + start_point: interpolation_node2.end_position(), + end_byte: close_quote_node.start_byte(), + end_point: close_quote_node.start_position(), + }, + ]); + let html_tree = parser.parse_str(source_code, None).unwrap(); + + assert_eq!( + html_tree.root_node().to_sexp(), + concat!( + "(fragment (element", + " (start_tag (tag_name))", + " (text)", + " (element (start_tag (tag_name)) (end_tag (tag_name)))", + " (text)", + " (end_tag (tag_name))))", + ) + ); + + let div_element_node = html_tree.root_node().child(0).unwrap(); + let hello_text_node = div_element_node.child(1).unwrap(); + let b_element_node = div_element_node.child(2).unwrap(); + let b_start_tag_node = b_element_node.child(0).unwrap(); + let b_end_tag_node = b_element_node.child(1).unwrap(); + + assert_eq!(hello_text_node.kind(), "text"); + assert_eq!( + hello_text_node.start_byte(), + source_code.find("Hello").unwrap() + ); + assert_eq!(hello_text_node.end_byte(), source_code.find("").unwrap()); + + assert_eq!(b_start_tag_node.kind(), "start_tag"); + assert_eq!( + b_start_tag_node.start_byte(), + source_code.find("").unwrap() + ); + assert_eq!( + b_start_tag_node.end_byte(), + source_code.find("${now()}").unwrap() + ); + + assert_eq!(b_end_tag_node.kind(), "end_tag"); + assert_eq!( + b_end_tag_node.start_byte(), + source_code.find("").unwrap() + ); + assert_eq!( + b_end_tag_node.end_byte(), + source_code.find(".
").unwrap() + ); +} + +#[test] +fn test_parsing_utf16_code_with_errors_at_the_end_of_an_included_range() { + let source_code = ""; + let utf16_source_code: Vec = source_code.as_bytes().iter().map(|c| *c as u16).collect(); + + let start_byte = 2 * source_code.find("a.").unwrap(); + let end_byte = 2 * source_code.find("").unwrap(); + + let mut parser = Parser::new(); + parser.set_language(get_language("javascript")).unwrap(); + parser.set_included_ranges(&[Range { + start_byte, + end_byte, + start_point: Point::new(0, start_byte), + end_point: Point::new(0, end_byte), + }]); + let tree = parser + .parse_utf16(&mut |i, _| &utf16_source_code[i..], None) + .unwrap(); + assert_eq!(tree.root_node().to_sexp(), "(program (ERROR (identifier)))"); +} + fn rust() -> Language { get_language("rust") } diff --git a/lib/binding/lib.rs b/lib/binding/lib.rs index 150dfcf4..9e04ed35 100644 --- a/lib/binding/lib.rs +++ b/lib/binding/lib.rs @@ -261,7 +261,7 @@ impl Parser { ) -> Option { self.parse_utf16_ptr( &mut |byte, position| { - let slice = input(byte, position); + let slice = input(byte / 2, position); (slice.as_ptr(), slice.len()) }, old_tree, @@ -570,6 +570,30 @@ impl<'tree> Node<'tree> { Self::new(unsafe { ffi::ts_node_prev_named_sibling(self.0) }) } + pub fn descendant_for_byte_range(&self, start: usize, end: usize) -> Option { + Self::new(unsafe { + ffi::ts_node_descendant_for_byte_range(self.0, start as u32, end as u32) + }) + } + + pub fn named_descendant_for_byte_range(&self, start: usize, end: usize) -> Option { + Self::new(unsafe { + ffi::ts_node_named_descendant_for_byte_range(self.0, start as u32, end as u32) + }) + } + + pub fn descendant_for_point_range(&self, start: Point, end: Point) -> Option { + Self::new(unsafe { + ffi::ts_node_descendant_for_point_range(self.0, start.into(), end.into()) + }) + } + + pub fn named_descendant_for_point_range(&self, start: Point, end: Point) -> Option { + Self::new(unsafe { + ffi::ts_node_named_descendant_for_point_range(self.0, start.into(), end.into()) + }) + } + pub fn to_sexp(&self) -> String { let c_string = unsafe { ffi::ts_node_string(self.0) }; let result = unsafe { CStr::from_ptr(c_string) }