traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Introduce language ref-count management C APIs, remove Copy impl for Language in Rust

Browse source
This commit is contained in:
Max Brunsfeld 2023-11-27 15:50:08 -08:00
parent 660481dbf7
commit da16cb1459
27 changed files with 363 additions and 314 deletions
Download patch file Download diff file Expand all files Collapse all files

72
cli/src/tests/parser_test.rs
View file

@ -20,7 +20,7 @@ use tree_sitter_proc_macro::retry;
#[test]
fn test_parsing_simple_string() {
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
parser.set_language(&get_language("rust")).unwrap();
let tree = parser
.parse(
@ -51,7 +51,7 @@ fn test_parsing_simple_string() {
#[test]
fn test_parsing_with_logging() {
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
parser.set_language(&get_language("rust")).unwrap();
let mut messages = Vec::new();
parser.set_logger(Some(Box::new(|log_type, message| {
@ -92,7 +92,7 @@ fn test_parsing_with_debug_graph_enabled() {
let has_zero_indexed_row = |s: &str| s.contains("position: 0,");
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_language(&get_language("javascript")).unwrap();
let mut debug_graph_file = tempfile::tempfile().unwrap();
parser.print_dot_graphs(&debug_graph_file);
@ -114,7 +114,7 @@ fn test_parsing_with_debug_graph_enabled() {
#[test]
fn test_parsing_with_custom_utf8_input() {
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
parser.set_language(&get_language("rust")).unwrap();
let lines = &["pub fn foo() {", " 1", "}"];
@ -157,7 +157,7 @@ fn test_parsing_with_custom_utf8_input() {
#[test]
fn test_parsing_with_custom_utf16_input() {
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
parser.set_language(&get_language("rust")).unwrap();
let lines: Vec<Vec<u16>> = ["pub fn foo() {", " 1", "}"]
.iter()
@ -196,7 +196,7 @@ fn test_parsing_with_custom_utf16_input() {
#[test]
fn test_parsing_with_callback_returning_owned_strings() {
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
parser.set_language(&get_language("rust")).unwrap();
let text = b"pub fn foo() { 1 }";
@ -217,7 +217,7 @@ fn test_parsing_with_callback_returning_owned_strings() {
#[test]
fn test_parsing_text_with_byte_order_mark() {
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
parser.set_language(&get_language("rust")).unwrap();
// Parse UTF16 text with a BOM
let tree = parser
@ -276,7 +276,7 @@ fn test_parsing_text_with_byte_order_mark() {
#[test]
fn test_parsing_invalid_chars_at_eof() {
let mut parser = Parser::new();
parser.set_language(get_language("json")).unwrap();
parser.set_language(&get_language("json")).unwrap();
let tree = parser.parse(b"\xdf", None).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
@ -287,7 +287,7 @@ fn test_parsing_invalid_chars_at_eof() {
#[test]
fn test_parsing_unexpected_null_characters_within_source() {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_language(&get_language("javascript")).unwrap();
let tree = parser.parse(b"var \0 something;", None).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
@ -298,7 +298,7 @@ fn test_parsing_unexpected_null_characters_within_source() {
#[test]
fn test_parsing_ends_when_input_callback_returns_empty() {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_language(&get_language("javascript")).unwrap();
let mut i = 0;
let source = b"abcdefghijklmnoqrs";
let tree = parser
@ -322,7 +322,7 @@ fn test_parsing_ends_when_input_callback_returns_empty() {
#[test]
fn test_parsing_after_editing_beginning_of_code() {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_language(&get_language("javascript")).unwrap();
let mut code = b"123 + 456 * (10 + x);".to_vec();
let mut tree = parser.parse(&code, None).unwrap();
@ -370,7 +370,7 @@ fn test_parsing_after_editing_beginning_of_code() {
#[test]
fn test_parsing_after_editing_end_of_code() {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_language(&get_language("javascript")).unwrap();
let mut code = b"x * (100 + abc);".to_vec();
let mut tree = parser.parse(&code, None).unwrap();
@ -418,7 +418,7 @@ fn test_parsing_after_editing_end_of_code() {
#[test]
fn test_parsing_empty_file_with_reused_tree() {
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
parser.set_language(&get_language("rust")).unwrap();
let tree = parser.parse("", None);
parser.parse("", tree.as_ref());
@ -437,7 +437,7 @@ fn test_parsing_after_editing_tree_that_depends_on_column_values() {
let mut parser = Parser::new();
parser
.set_language(get_test_language(&grammar_name, &parser_code, Some(&dir)))
.set_language(&get_test_language(&grammar_name, &parser_code, Some(&dir)))
.unwrap();
let mut code = b"
@ -507,7 +507,7 @@ h + i
#[test]
fn test_parsing_after_detecting_error_in_the_middle_of_a_string_token() {
let mut parser = Parser::new();
parser.set_language(get_language("python")).unwrap();
parser.set_language(&get_language("python")).unwrap();
let mut source = b"a = b, 'c, d'".to_vec();
let tree = parser.parse(&source, None).unwrap();
@ -551,7 +551,7 @@ fn test_parsing_on_multiple_threads() {
let this_file_source = include_str!("parser_test.rs");
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
parser.set_language(&get_language("rust")).unwrap();
let tree = parser.parse(this_file_source, None).unwrap();
let mut parse_threads = Vec::new();
@ -579,7 +579,7 @@ fn test_parsing_on_multiple_threads() {
// Reparse using the old tree as a starting point.
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
parser.set_language(&get_language("rust")).unwrap();
parser.parse(&prepended_source, Some(&tree_clone)).unwrap()
}));
}
@ -600,7 +600,7 @@ fn test_parsing_cancelled_by_another_thread() {
let cancellation_flag = std::sync::Arc::new(AtomicUsize::new(0));
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_language(&get_language("javascript")).unwrap();
unsafe { parser.set_cancellation_flag(Some(&cancellation_flag)) };
// Long input - parsing succeeds
@ -649,7 +649,7 @@ fn test_parsing_cancelled_by_another_thread() {
#[retry(10)]
fn test_parsing_with_a_timeout() {
let mut parser = Parser::new();
parser.set_language(get_language("json")).unwrap();
parser.set_language(&get_language("json")).unwrap();
// Parse an infinitely-long array, but pause after 1ms of processing.
parser.set_timeout_micros(1000);
@ -711,7 +711,7 @@ fn test_parsing_with_a_timeout() {
#[retry(10)]
fn test_parsing_with_a_timeout_and_a_reset() {
let mut parser = Parser::new();
parser.set_language(get_language("json")).unwrap();
parser.set_language(&get_language("json")).unwrap();
parser.set_timeout_micros(5);
let tree = parser.parse(
@ -768,7 +768,7 @@ fn test_parsing_with_a_timeout_and_a_reset() {
fn test_parsing_with_a_timeout_and_implicit_reset() {
allocations::record(|| {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_language(&get_language("javascript")).unwrap();
parser.set_timeout_micros(5);
let tree = parser.parse(
@ -779,7 +779,7 @@ fn test_parsing_with_a_timeout_and_implicit_reset() {
// Changing the parser's language implicitly resets, discarding
// the previous partial parse.
parser.set_language(get_language("json")).unwrap();
parser.set_language(&get_language("json")).unwrap();
parser.set_timeout_micros(0);
let tree = parser.parse(
"[null, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]",
@ -802,7 +802,7 @@ fn test_parsing_with_a_timeout_and_implicit_reset() {
fn test_parsing_with_timeout_and_no_completion() {
allocations::record(|| {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_language(&get_language("javascript")).unwrap();
parser.set_timeout_micros(5);
let tree = parser.parse(
@ -822,7 +822,7 @@ fn test_parsing_with_one_included_range() {
let source_code = "<span>hi</span><script>console.log('sup');</script>";
let mut parser = Parser::new();
parser.set_language(get_language("html")).unwrap();
parser.set_language(&get_language("html")).unwrap();
let html_tree = parser.parse(source_code, None).unwrap();
let script_content_node = html_tree.root_node().child(1).unwrap().child(1).unwrap();
assert_eq!(script_content_node.kind(), "raw_text");
@ -830,7 +830,7 @@ fn test_parsing_with_one_included_range() {
parser
.set_included_ranges(&[script_content_node.range()])
.unwrap();
parser.set_language(get_language("javascript")).unwrap();
parser.set_language(&get_language("javascript")).unwrap();
let js_tree = parser.parse(source_code, None).unwrap();
assert_eq!(
@ -853,7 +853,7 @@ fn test_parsing_with_multiple_included_ranges() {
let source_code = "html `<div>Hello, ${name.toUpperCase()}, it's <b>${now()}</b>.</div>`";
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_language(&get_language("javascript")).unwrap();
let js_tree = parser.parse(source_code, None).unwrap();
let template_string_node = js_tree
.root_node()
@ -869,7 +869,7 @@ fn test_parsing_with_multiple_included_ranges() {
let interpolation_node2 = template_string_node.child(2).unwrap();
let close_quote_node = template_string_node.child(3).unwrap();
parser.set_language(get_language("html")).unwrap();
parser.set_language(&get_language("html")).unwrap();
let html_ranges = &[
Range {
start_byte: open_quote_node.end_byte(),
@ -948,7 +948,7 @@ fn test_parsing_with_included_range_containing_mismatched_positions() {
let source_code = "<div>test</div>{_ignore_this_part_}";
let mut parser = Parser::new();
parser.set_language(get_language("html")).unwrap();
parser.set_language(&get_language("html")).unwrap();
let end_byte = source_code.find("{_ignore_this_part_").unwrap();
@ -1029,7 +1029,7 @@ fn test_parsing_utf16_code_with_errors_at_the_end_of_an_included_range() {
let end_byte = 2 * source_code.find("</script>").unwrap();
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_language(&get_language("javascript")).unwrap();
parser
.set_included_ranges(&[Range {
start_byte,
@ -1051,7 +1051,7 @@ fn test_parsing_with_external_scanner_that_uses_included_range_boundaries() {
let range2_end_byte = range2_start_byte + " d() ".len();
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_language(&get_language("javascript")).unwrap();
parser
.set_included_ranges(&[
Range {
@ -1095,7 +1095,7 @@ fn test_parsing_with_a_newly_excluded_range() {
// Parse HTML including the template directive, which will cause an error
let mut parser = Parser::new();
parser.set_language(get_language("html")).unwrap();
parser.set_language(&get_language("html")).unwrap();
let mut first_tree = parser
.parse_with(&mut chunked_input(&source_code, 3), None)
.unwrap();
@ -1182,7 +1182,7 @@ fn test_parsing_with_a_newly_included_range() {
// Parse only the first code directive as JavaScript
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_language(&get_language("javascript")).unwrap();
parser
.set_included_ranges(&[simple_range(range1_start, range1_end)])
.unwrap();
@ -1274,7 +1274,7 @@ fn test_parsing_with_included_ranges_and_missing_tokens() {
let mut parser = Parser::new();
parser
.set_language(get_test_language(&parser_name, &parser_code, None))
.set_language(&get_test_language(&parser_name, &parser_code, None))
.unwrap();
// There's a missing `a` token at the beginning of the code. It must be inserted
@ -1331,7 +1331,7 @@ fn test_grammars_that_can_hang_on_eof() {
let mut parser = Parser::new();
parser
.set_language(get_test_language(&parser_name, &parser_code, None))
.set_language(&get_test_language(&parser_name, &parser_code, None))
.unwrap();
parser.parse("\"", None).unwrap();
@ -1356,7 +1356,7 @@ fn test_grammars_that_can_hang_on_eof() {
.unwrap();
parser
.set_language(get_test_language(&parser_name, &parser_code, None))
.set_language(&get_test_language(&parser_name, &parser_code, None))
.unwrap();
parser.parse("\"", None).unwrap();
@ -1381,7 +1381,7 @@ fn test_grammars_that_can_hang_on_eof() {
.unwrap();
parser
.set_language(get_test_language(&parser_name, &parser_code, None))
.set_language(&get_test_language(&parser_name, &parser_code, None))
.unwrap();
parser.parse("\"", None).unwrap();
}