use super::helpers::edits::ReadRecorder; use super::helpers::fixtures::{get_language, get_test_language}; use crate::generate::generate_parser_for_grammar; use crate::parse::{perform_edit, Edit}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::{thread, time}; use tree_sitter::{InputEdit, LogType, Parser, Point, Range}; #[test] fn test_basic_parsing() { let mut parser = Parser::new(); parser.set_language(get_language("rust")).unwrap(); let tree = parser .parse( " struct Stuff {} fn main() {} ", None, ) .unwrap(); let root_node = tree.root_node(); assert_eq!(root_node.kind(), "source_file"); assert_eq!( root_node.to_sexp(), "(source_file (struct_item (type_identifier) (field_declaration_list)) (function_item (identifier) (parameters) (block)))" ); let struct_node = root_node.child(0).unwrap(); assert_eq!(struct_node.kind(), "struct_item"); } #[test] fn test_parsing_with_logging() { let mut parser = Parser::new(); parser.set_language(get_language("rust")).unwrap(); let mut messages = Vec::new(); parser.set_logger(Some(Box::new(|log_type, message| { messages.push((log_type, message.to_string())); }))); parser .parse( " struct Stuff {} fn main() {} ", None, ) .unwrap(); assert!(messages.contains(&( LogType::Parse, "reduce sym:struct_item, child_count:3".to_string() ))); assert!(messages.contains(&(LogType::Lex, "skip character:' '".to_string()))); for (_, m) in &messages { assert!(!m.contains("row:0")); } } #[test] #[cfg(unix)] fn test_parsing_with_debug_graph_enabled() { use std::io::{BufRead, BufReader, Seek}; let has_zero_indexed_row = |s: &str| s.contains("position: 0,"); let mut parser = Parser::new(); parser.set_language(get_language("javascript")).unwrap(); let mut debug_graph_file = tempfile::tempfile().unwrap(); parser.print_dot_graphs(&debug_graph_file); parser.parse("const zero = 0", None).unwrap(); debug_graph_file.seek(std::io::SeekFrom::Start(0)).unwrap(); let log_reader = BufReader::new(debug_graph_file) .lines() .map(|l| l.expect("Failed to read line from graph log")); for line in log_reader { assert!( !has_zero_indexed_row(&line), "Graph log output includes zero-indexed row: {}", line ); } } #[test] fn test_parsing_with_custom_utf8_input() { let mut parser = Parser::new(); parser.set_language(get_language("rust")).unwrap(); let lines = &["pub fn foo() {", " 1", "}"]; let tree = parser .parse_with( &mut |_, position| { let row = position.row; let column = position.column; if row < lines.len() { if column < lines[row].as_bytes().len() { &lines[row].as_bytes()[column..] } else { "\n".as_bytes() } } else { &[] } }, None, ) .unwrap(); let root = tree.root_node(); assert_eq!(root.to_sexp(), "(source_file (function_item (visibility_modifier) (identifier) (parameters) (block (integer_literal))))"); assert_eq!(root.kind(), "source_file"); assert_eq!(root.has_error(), false); assert_eq!(root.child(0).unwrap().kind(), "function_item"); } #[test] fn test_parsing_with_custom_utf16_input() { let mut parser = Parser::new(); parser.set_language(get_language("rust")).unwrap(); let lines: Vec> = ["pub fn foo() {", " 1", "}"] .iter() .map(|s| s.encode_utf16().collect()) .collect(); let tree = parser .parse_utf16_with( &mut |_, position| { let row = position.row; let column = position.column; if row < lines.len() { if column < lines[row].len() { &lines[row][column..] } else { &[10] } } else { &[] } }, None, ) .unwrap(); let root = tree.root_node(); assert_eq!(root.to_sexp(), "(source_file (function_item (visibility_modifier) (identifier) (parameters) (block (integer_literal))))"); assert_eq!(root.kind(), "source_file"); assert_eq!(root.has_error(), false); assert_eq!(root.child(0).unwrap().kind(), "function_item"); } // Incremental parsing #[test] fn test_parsing_after_editing_beginning_of_code() { let mut parser = Parser::new(); parser.set_language(get_language("javascript")).unwrap(); let mut code = b"123 + 456 * (10 + x);".to_vec(); let mut tree = parser.parse(&code, None).unwrap(); assert_eq!( tree.root_node().to_sexp(), concat!( "(program (expression_statement (binary_expression ", "(number) ", "(binary_expression (number) (parenthesized_expression (binary_expression (number) (identifier)))))))", ) ); perform_edit( &mut tree, &mut code, &Edit { position: 3, deleted_length: 0, inserted_text: b" || 5".to_vec(), }, ); let mut recorder = ReadRecorder::new(&code); let tree = parser .parse_with(&mut |i, _| recorder.read(i), Some(&tree)) .unwrap(); assert_eq!( tree.root_node().to_sexp(), concat!( "(program (expression_statement (binary_expression ", "(number) ", "(binary_expression ", "(number) ", "(binary_expression (number) (parenthesized_expression (binary_expression (number) (identifier))))))))", ) ); assert_eq!(recorder.strings_read(), vec!["123 || 5 "]); } #[test] fn test_parsing_after_editing_end_of_code() { let mut parser = Parser::new(); parser.set_language(get_language("javascript")).unwrap(); let mut code = b"x * (100 + abc);".to_vec(); let mut tree = parser.parse(&code, None).unwrap(); assert_eq!( tree.root_node().to_sexp(), concat!( "(program (expression_statement (binary_expression ", "(identifier) ", "(parenthesized_expression (binary_expression (number) (identifier))))))", ) ); let position = code.len() - 2; perform_edit( &mut tree, &mut code, &Edit { position, deleted_length: 0, inserted_text: b".d".to_vec(), }, ); let mut recorder = ReadRecorder::new(&code); let tree = parser .parse_with(&mut |i, _| recorder.read(i), Some(&tree)) .unwrap(); assert_eq!( tree.root_node().to_sexp(), concat!( "(program (expression_statement (binary_expression ", "(identifier) ", "(parenthesized_expression (binary_expression (number) (member_expression (identifier) (property_identifier)))))))" ) ); assert_eq!(recorder.strings_read(), vec![" * ", "abc.d)",]); } // Thread safety #[test] fn test_parsing_on_multiple_threads() { // Parse this source file so that each thread has a non-trivial amount of // work to do. let this_file_source = include_str!("parser_test.rs"); let mut parser = Parser::new(); parser.set_language(get_language("rust")).unwrap(); let tree = parser.parse(this_file_source, None).unwrap(); let mut parse_threads = Vec::new(); for thread_id in 1..5 { let mut tree_clone = tree.clone(); parse_threads.push(thread::spawn(move || { // For each thread, prepend a different number of declarations to the // source code. let mut prepend_line_count = 0; let mut prepended_source = String::new(); for _ in 0..thread_id { prepend_line_count += 2; prepended_source += "struct X {}\n\n"; } tree_clone.edit(&InputEdit { start_byte: 0, old_end_byte: 0, new_end_byte: prepended_source.len(), start_position: Point::new(0, 0), old_end_position: Point::new(0, 0), new_end_position: Point::new(prepend_line_count, 0), }); prepended_source += this_file_source; // Reparse using the old tree as a starting point. let mut parser = Parser::new(); parser.set_language(get_language("rust")).unwrap(); parser.parse(&prepended_source, Some(&tree_clone)).unwrap() })); } // Check that the trees have the expected relationship to one another. let trees = parse_threads .into_iter() .map(|thread| thread.join().unwrap()); let child_count_differences = trees .map(|t| t.root_node().child_count() - tree.root_node().child_count()) .collect::>(); assert_eq!(child_count_differences, &[1, 2, 3, 4]); } #[test] fn test_parsing_cancelled_by_another_thread() { let cancellation_flag = Box::new(AtomicUsize::new(0)); let mut parser = Parser::new(); parser.set_language(get_language("javascript")).unwrap(); unsafe { parser.set_cancellation_flag(Some(&cancellation_flag)) }; // Long input - parsing succeeds let tree = parser.parse_with( &mut |offset, _| { if offset == 0 { b" [" } else if offset >= 20000 { b"" } else { b"0," } }, None, ); assert!(tree.is_some()); let cancel_thread = thread::spawn(move || { thread::sleep(time::Duration::from_millis(100)); cancellation_flag.store(1, Ordering::SeqCst); }); // Infinite input let tree = parser.parse_with( &mut |offset, _| { thread::yield_now(); thread::sleep(time::Duration::from_millis(10)); if offset == 0 { b" [" } else { b"0," } }, None, ); // Parsing returns None because it was cancelled. cancel_thread.join().unwrap(); assert!(tree.is_none()); } // Timeouts #[test] fn test_parsing_with_a_timeout() { let mut parser = Parser::new(); parser.set_language(get_language("json")).unwrap(); // Parse an infinitely-long array, but pause after 100 microseconds of processing. parser.set_timeout_micros(100); let start_time = time::Instant::now(); let tree = parser.parse_with( &mut |offset, _| { if offset == 0 { b" [" } else { b",0" } }, None, ); assert!(tree.is_none()); assert!(start_time.elapsed().as_micros() < 500); // Continue parsing, but pause after 300 microseconds of processing. parser.set_timeout_micros(1000); let start_time = time::Instant::now(); let tree = parser.parse_with( &mut |offset, _| { if offset == 0 { b" [" } else { b",0" } }, None, ); assert!(tree.is_none()); assert!(start_time.elapsed().as_micros() > 500); assert!(start_time.elapsed().as_micros() < 1500); // Finish parsing parser.set_timeout_micros(0); let tree = parser .parse_with( &mut |offset, _| { if offset > 5000 { b"" } else if offset == 5000 { b"]" } else { b",0" } }, None, ) .unwrap(); assert_eq!(tree.root_node().child(0).unwrap().kind(), "array"); } #[test] fn test_parsing_with_a_timeout_and_a_reset() { let mut parser = Parser::new(); parser.set_language(get_language("json")).unwrap(); parser.set_timeout_micros(5); let tree = parser.parse( "[\"ok\", 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]", None, ); assert!(tree.is_none()); // Without calling reset, the parser continues from where it left off, so // it does not see the changes to the beginning of the source code. parser.set_timeout_micros(0); let tree = parser.parse( "[null, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]", None, ).unwrap(); assert_eq!( tree.root_node() .named_child(0) .unwrap() .named_child(0) .unwrap() .kind(), "string" ); parser.set_timeout_micros(5); let tree = parser.parse( "[\"ok\", 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]", None, ); assert!(tree.is_none()); // By calling reset, we force the parser to start over from scratch so // that it sees the changes to the beginning of the source code. parser.set_timeout_micros(0); parser.reset(); let tree = parser.parse( "[null, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]", None, ).unwrap(); assert_eq!( tree.root_node() .named_child(0) .unwrap() .named_child(0) .unwrap() .kind(), "null" ); } // Included Ranges #[test] fn test_parsing_with_one_included_range() { let source_code = "hi"; let mut parser = Parser::new(); parser.set_language(get_language("html")).unwrap(); let html_tree = parser.parse(source_code, None).unwrap(); let script_content_node = html_tree.root_node().child(1).unwrap().child(1).unwrap(); assert_eq!(script_content_node.kind(), "raw_text"); parser.set_included_ranges(&[script_content_node.range()]); parser.set_language(get_language("javascript")).unwrap(); let js_tree = parser.parse(source_code, None).unwrap(); assert_eq!( js_tree.root_node().to_sexp(), concat!( "(program (expression_statement (call_expression", " (member_expression (identifier) (property_identifier))", " (arguments (string)))))", ) ); assert_eq!( js_tree.root_node().start_position(), Point::new(0, source_code.find("console").unwrap()) ); } #[test] fn test_parsing_with_multiple_included_ranges() { let source_code = "html `
Hello, ${name.toUpperCase()}, it's ${now()}.
`"; let mut parser = Parser::new(); parser.set_language(get_language("javascript")).unwrap(); let js_tree = parser.parse(source_code, None).unwrap(); let template_string_node = js_tree .root_node() .descendant_for_byte_range( source_code.find("
").unwrap(), source_code.find("Hello").unwrap(), ) .unwrap(); assert_eq!(template_string_node.kind(), "template_string"); let open_quote_node = template_string_node.child(0).unwrap(); let interpolation_node1 = template_string_node.child(1).unwrap(); let interpolation_node2 = template_string_node.child(2).unwrap(); let close_quote_node = template_string_node.child(3).unwrap(); parser.set_language(get_language("html")).unwrap(); parser.set_included_ranges(&[ Range { start_byte: open_quote_node.end_byte(), start_point: open_quote_node.end_position(), end_byte: interpolation_node1.start_byte(), end_point: interpolation_node1.start_position(), }, Range { start_byte: interpolation_node1.end_byte(), start_point: interpolation_node1.end_position(), end_byte: interpolation_node2.start_byte(), end_point: interpolation_node2.start_position(), }, Range { start_byte: interpolation_node2.end_byte(), start_point: interpolation_node2.end_position(), end_byte: close_quote_node.start_byte(), end_point: close_quote_node.start_position(), }, ]); let html_tree = parser.parse(source_code, None).unwrap(); assert_eq!( html_tree.root_node().to_sexp(), concat!( "(fragment (element", " (start_tag (tag_name))", " (text)", " (element (start_tag (tag_name)) (end_tag (tag_name)))", " (text)", " (end_tag (tag_name))))", ) ); let div_element_node = html_tree.root_node().child(0).unwrap(); let hello_text_node = div_element_node.child(1).unwrap(); let b_element_node = div_element_node.child(2).unwrap(); let b_start_tag_node = b_element_node.child(0).unwrap(); let b_end_tag_node = b_element_node.child(1).unwrap(); assert_eq!(hello_text_node.kind(), "text"); assert_eq!( hello_text_node.start_byte(), source_code.find("Hello").unwrap() ); assert_eq!(hello_text_node.end_byte(), source_code.find("").unwrap()); assert_eq!(b_start_tag_node.kind(), "start_tag"); assert_eq!( b_start_tag_node.start_byte(), source_code.find("").unwrap() ); assert_eq!( b_start_tag_node.end_byte(), source_code.find("${now()}").unwrap() ); assert_eq!(b_end_tag_node.kind(), "end_tag"); assert_eq!( b_end_tag_node.start_byte(), source_code.find("").unwrap() ); assert_eq!( b_end_tag_node.end_byte(), source_code.find(".
").unwrap() ); } #[test] fn test_parsing_utf16_code_with_errors_at_the_end_of_an_included_range() { let source_code = ""; let utf16_source_code: Vec = source_code.as_bytes().iter().map(|c| *c as u16).collect(); let start_byte = 2 * source_code.find("a.").unwrap(); let end_byte = 2 * source_code.find("").unwrap(); let mut parser = Parser::new(); parser.set_language(get_language("javascript")).unwrap(); parser.set_included_ranges(&[Range { start_byte, end_byte, start_point: Point::new(0, start_byte), end_point: Point::new(0, end_byte), }]); let tree = parser.parse_utf16(&utf16_source_code, None).unwrap(); assert_eq!(tree.root_node().to_sexp(), "(program (ERROR (identifier)))"); } #[test] fn test_parsing_with_external_scanner_that_uses_included_range_boundaries() { let source_code = "a <%= b() %> c <% d() %>"; let range1_start_byte = source_code.find(" b() ").unwrap(); let range1_end_byte = range1_start_byte + " b() ".len(); let range2_start_byte = source_code.find(" d() ").unwrap(); let range2_end_byte = range2_start_byte + " d() ".len(); let mut parser = Parser::new(); parser.set_language(get_language("javascript")).unwrap(); parser.set_included_ranges(&[ Range { start_byte: range1_start_byte, end_byte: range1_end_byte, start_point: Point::new(0, range1_start_byte), end_point: Point::new(0, range1_end_byte), }, Range { start_byte: range2_start_byte, end_byte: range2_end_byte, start_point: Point::new(0, range2_start_byte), end_point: Point::new(0, range2_end_byte), }, ]); let tree = parser.parse(source_code, None).unwrap(); let root = tree.root_node(); let statement1 = root.child(0).unwrap(); let statement2 = root.child(1).unwrap(); assert_eq!( root.to_sexp(), concat!( "(program", " (expression_statement (call_expression (identifier) (arguments)))", " (expression_statement (call_expression (identifier) (arguments))))" ) ); assert_eq!(statement1.start_byte(), source_code.find("b()").unwrap()); assert_eq!(statement1.end_byte(), source_code.find(" %> c").unwrap()); assert_eq!(statement2.start_byte(), source_code.find("d()").unwrap()); assert_eq!(statement2.end_byte(), source_code.len() - " %>".len()); } #[test] fn test_parsing_with_a_newly_excluded_range() { let mut source_code = String::from("
<%= something %>
"); // Parse HTML including the template directive, which will cause an error let mut parser = Parser::new(); parser.set_language(get_language("html")).unwrap(); let mut first_tree = parser.parse(&source_code, None).unwrap(); // Insert code at the beginning of the document. let prefix = "a very very long line of plain text. "; first_tree.edit(&InputEdit { start_byte: 0, old_end_byte: 0, new_end_byte: prefix.len(), start_position: Point::new(0, 0), old_end_position: Point::new(0, 0), new_end_position: Point::new(0, prefix.len()), }); source_code.insert_str(0, prefix); // Parse the HTML again, this time *excluding* the template directive // (which has moved since the previous parse). let directive_start = source_code.find("<%=").unwrap(); let directive_end = source_code.find("").unwrap(); let source_code_end = source_code.len(); parser.set_included_ranges(&[ Range { start_byte: 0, end_byte: directive_start, start_point: Point::new(0, 0), end_point: Point::new(0, directive_start), }, Range { start_byte: directive_end, end_byte: source_code_end, start_point: Point::new(0, directive_end), end_point: Point::new(0, source_code_end), }, ]); let tree = parser.parse(&source_code, Some(&first_tree)).unwrap(); assert_eq!( tree.root_node().to_sexp(), concat!( "(fragment (text) (element", " (start_tag (tag_name))", " (element (start_tag (tag_name)) (end_tag (tag_name)))", " (end_tag (tag_name))))" ) ); assert_eq!( tree.changed_ranges(&first_tree), vec![ // The first range that has changed syntax is the range of the newly-inserted text. Range { start_byte: 0, end_byte: prefix.len(), start_point: Point::new(0, 0), end_point: Point::new(0, prefix.len()), }, // Even though no edits were applied to the outer `div` element, // its contents have changed syntax because a range of text that // was previously included is now excluded. Range { start_byte: directive_start, end_byte: directive_end, start_point: Point::new(0, directive_start), end_point: Point::new(0, directive_end), }, ] ); } #[test] fn test_parsing_with_a_newly_included_range() { let source_code = "
<%= foo() %>
<%= bar() %>"; let first_code_start_index = source_code.find(" foo").unwrap(); let first_code_end_index = first_code_start_index + 7; let second_code_start_index = source_code.find(" bar").unwrap(); let second_code_end_index = second_code_start_index + 7; let ranges = [ Range { start_byte: first_code_start_index, end_byte: first_code_end_index, start_point: Point::new(0, first_code_start_index), end_point: Point::new(0, first_code_end_index), }, Range { start_byte: second_code_start_index, end_byte: second_code_end_index, start_point: Point::new(0, second_code_start_index), end_point: Point::new(0, second_code_end_index), }, ]; // Parse only the first code directive as JavaScript let mut parser = Parser::new(); parser.set_language(get_language("javascript")).unwrap(); parser.set_included_ranges(&ranges[0..1]); let first_tree = parser.parse(source_code, None).unwrap(); assert_eq!( first_tree.root_node().to_sexp(), concat!( "(program", " (expression_statement (call_expression (identifier) (arguments))))", ) ); // Parse both the code directives as JavaScript, using the old tree as a reference. parser.set_included_ranges(&ranges); let tree = parser.parse(&source_code, Some(&first_tree)).unwrap(); assert_eq!( tree.root_node().to_sexp(), concat!( "(program", " (expression_statement (call_expression (identifier) (arguments)))", " (expression_statement (call_expression (identifier) (arguments))))", ) ); assert_eq!( tree.changed_ranges(&first_tree), vec![Range { start_byte: first_code_end_index + 1, end_byte: second_code_end_index + 1, start_point: Point::new(0, first_code_end_index + 1), end_point: Point::new(0, second_code_end_index + 1), }] ); } #[test] fn test_parsing_with_included_ranges_and_missing_tokens() { let (parser_name, parser_code) = generate_parser_for_grammar( r#"{ "name": "test_leading_missing_token", "rules": { "program": { "type": "SEQ", "members": [ {"type": "SYMBOL", "name": "A"}, {"type": "SYMBOL", "name": "b"}, {"type": "SYMBOL", "name": "c"}, {"type": "SYMBOL", "name": "A"}, {"type": "SYMBOL", "name": "b"}, {"type": "SYMBOL", "name": "c"} ] }, "A": {"type": "SYMBOL", "name": "a"}, "a": {"type": "STRING", "value": "a"}, "b": {"type": "STRING", "value": "b"}, "c": {"type": "STRING", "value": "c"} } }"#, ) .unwrap(); let mut parser = Parser::new(); parser .set_language(get_test_language(&parser_name, &parser_code, None)) .unwrap(); // There's a missing `a` token at the beginning of the code. It must be inserted // at the beginning of the first included range, not at {0, 0}. let source_code = "__bc__bc__"; parser.set_included_ranges(&[ Range { start_byte: 2, end_byte: 4, start_point: Point::new(0, 2), end_point: Point::new(0, 4), }, Range { start_byte: 6, end_byte: 8, start_point: Point::new(0, 6), end_point: Point::new(0, 8), }, ]); let tree = parser.parse(source_code, None).unwrap(); let root = tree.root_node(); assert_eq!( root.to_sexp(), "(program (A (MISSING a)) (b) (c) (A (MISSING a)) (b) (c))" ); assert_eq!(root.start_byte(), 2); assert_eq!(root.child(3).unwrap().start_byte(), 4); }