traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

binding: Make parse methods more convenient

Browse source 
* Rename parse_str to parse and make it polymorphic.
* Rename parse_utf8 to parse_with, since it is now the callback-based
  version of parse
* Add a parse_utf16 method analogous to parse
* Rename existing parse_utf16 method to parse_utf16_with

This brings in the changes from tree-sitter/rust-tree-sitter#5
This commit is contained in:
Max Brunsfeld 2019-02-05 10:59:31 -08:00
parent 49392c8502
commit 50281637d7
10 changed files with 89 additions and 169 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cli/src/parse.rs
View file

@ -33,7 +33,7 @@ pub fn parse_file_at_path(
let time = Instant::now();
let tree = parser
.parse_utf8(&mut |byte, _| &source_code[byte..], None)
.parse(&source_code, None)
.expect("Incompatible language version");
let duration = time.elapsed();
let duration_ms = duration.as_secs() * 1000 + duration.subsec_nanos() as u64 / 1000000;

4
cli/src/test.rs
View file

@ -141,9 +141,7 @@ fn run_tests(
return Ok(());
}
}
let tree = parser
.parse_utf8(&mut |byte_offset, _| &input[byte_offset..], None)
.unwrap();
let tree = parser.parse(&input, None).unwrap();
let actual = tree.root_node().to_sexp();
for _ in 0..indent_level {
print!(" ");

16
cli/src/tests/corpus_test.rs
View file

@ -75,7 +75,7 @@ fn test_real_language_corpus_files() {
let mut log_session = None;
let mut parser = get_parser(&mut log_session, "log.html");
parser.set_language(language).unwrap();
let tree = parser.parse_utf8(&mut |i, _| &input[i..], None).unwrap();
let tree = parser.parse(&input, None).unwrap();
let actual_output = tree.root_node().to_sexp();
drop(tree);
drop(parser);
@ -91,9 +91,7 @@ fn test_real_language_corpus_files() {
let mut parser = Parser::new();
parser.set_language(language).unwrap();
let tree = parser
.parse_utf8(&mut |i, _| input.get(i..).unwrap_or(&[]), None)
.unwrap();
let tree = parser.parse(&input, None).unwrap();
drop(parser);
for trial in 1..=TRIAL_COUNT {
@ -122,9 +120,7 @@ fn test_real_language_corpus_files() {
eprintln!("{}\n", String::from_utf8_lossy(&input));
}
let mut tree2 = parser
.parse_utf8(&mut |i, _| input.get(i..).unwrap_or(&[]), Some(&tree))
.unwrap();
let mut tree2 = parser.parse(&input, Some(&tree)).unwrap();
// Check that the new tree is consistent.
check_consistent_sizes(&tree2, &input);
@ -145,9 +141,7 @@ fn test_real_language_corpus_files() {
eprintln!("{}\n", String::from_utf8_lossy(&input));
}
let tree3 = parser
.parse_utf8(&mut |i, _| input.get(i..).unwrap_or(&[]), Some(&tree2))
.unwrap();
let tree3 = parser.parse(&input, Some(&tree2)).unwrap();
// Verify that the final tree matches the expectation from the corpus.
let actual_output = tree3.root_node().to_sexp();
@ -254,7 +248,7 @@ fn test_feature_corpus_files() {
let mut log_session = None;
let mut parser = get_parser(&mut log_session, "log.html");
parser.set_language(language).unwrap();
let tree = parser.parse_utf8(&mut |i, _| &input[i..], None).unwrap();
let tree = parser.parse(&input, None).unwrap();
let actual_output = tree.root_node().to_sexp();
drop(tree);
drop(parser);

4
cli/src/tests/node_test.rs
View file

@ -248,7 +248,7 @@ fn test_node_named_child_with_aliases_and_extras() {
.set_language(get_test_language(&parser_name, &parser_code, None))
.unwrap();
let tree = parser.parse_str("b ... b ... c", None).unwrap();
let tree = parser.parse("b ... b ... c", None).unwrap();
let root = tree.root_node();
assert_eq!(root.to_sexp(), "(a (b) (comment) (B) (comment) (C))");
assert_eq!(root.named_child_count(), 5);
@ -360,5 +360,5 @@ fn get_all_nodes(tree: &Tree) -> Vec<Node> {
fn parse_json_example() -> Tree {
let mut parser = Parser::new();
parser.set_language(get_language("json")).unwrap();
parser.parse_str(JSON_EXAMPLE, None).unwrap()
parser.parse(JSON_EXAMPLE, None).unwrap()
}

62
cli/src/tests/parser_test.rs
View file

@ -10,7 +10,7 @@ fn test_basic_parsing() {
parser.set_language(get_language("rust")).unwrap();
let tree = parser
.parse_str(
.parse(
"
struct Stuff {}
fn main() {}
@ -42,7 +42,7 @@ fn test_parsing_with_logging() {
})));
parser
.parse_str(
.parse(
"
struct Stuff {}
fn main() {}
@ -66,7 +66,7 @@ fn test_parsing_with_custom_utf8_input() {
let lines = &["pub fn foo() {", " 1", "}"];
let tree = parser
.parse_utf8(
.parse_with(
&mut |_, position| {
let row = position.row;
let column = position.column;
@ -102,7 +102,7 @@ fn test_parsing_with_custom_utf16_input() {
.collect();
let tree = parser
.parse_utf16(
.parse_utf16_with(
&mut |_, position| {
let row = position.row;
let column = position.column;
@ -135,7 +135,7 @@ fn test_parsing_after_editing_beginning_of_code() {
parser.set_language(get_language("javascript")).unwrap();
let mut code = b"123 + 456 * (10 + x);".to_vec();
let mut tree = parser.parse_utf8(&mut |i, _| &code[i..], None).unwrap();
let mut tree = parser.parse(&code, None).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
concat!(
@ -157,7 +157,7 @@ fn test_parsing_after_editing_beginning_of_code() {
let mut recorder = ReadRecorder::new(&code);
let tree = parser
.parse_utf8(&mut |i, _| recorder.read(i), Some(&tree))
.parse_with(&mut |i, _| recorder.read(i), Some(&tree))
.unwrap();
assert_eq!(
tree.root_node().to_sexp(),
@ -179,13 +179,13 @@ fn test_parsing_after_editing_end_of_code() {
parser.set_language(get_language("javascript")).unwrap();
let mut code = b"x * (100 + abc);".to_vec();
let mut tree = parser.parse_utf8(&mut |i, _| &code[i..], None).unwrap();
let mut tree = parser.parse(&code, None).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
concat!(
"(program (expression_statement (binary_expression ",
"(identifier) ",
"(parenthesized_expression (binary_expression (number) (identifier))))))",
"(identifier) ",
"(parenthesized_expression (binary_expression (number) (identifier))))))",
)
);
@ -202,7 +202,7 @@ fn test_parsing_after_editing_end_of_code() {
let mut recorder = ReadRecorder::new(&code);
let tree = parser
.parse_utf8(&mut |i, _| recorder.read(i), Some(&tree))
.parse_with(&mut |i, _| recorder.read(i), Some(&tree))
.unwrap();
assert_eq!(
tree.root_node().to_sexp(),
@ -226,7 +226,7 @@ fn test_parsing_on_multiple_threads() {
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
let tree = parser.parse_str(this_file_source, None).unwrap();
let tree = parser.parse(this_file_source, None).unwrap();
let mut parse_threads = Vec::new();
for thread_id in 1..5 {
@ -254,9 +254,7 @@ fn test_parsing_on_multiple_threads() {
// Reparse using the old tree as a starting point.
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
parser
.parse_str(&prepended_source, Some(&tree_clone))
.unwrap()
parser.parse(&prepended_source, Some(&tree_clone)).unwrap()
}));
}
@ -281,7 +279,7 @@ fn test_parsing_with_an_operation_limit() {
// Start parsing from an infinite input. Parsing should abort after 5 "operations".
parser.set_operation_limit(5);
let mut call_count = 0;
let tree = parser.parse_utf8(
let tree = parser.parse_with(
&mut |_, _| {
if call_count == 0 {
call_count += 1;
@ -301,7 +299,7 @@ fn test_parsing_with_an_operation_limit() {
call_count = 0;
parser.set_operation_limit(20);
let tree = parser
.parse_utf8(
.parse_with(
&mut |_, _| {
if call_count == 0 {
call_count += 1;
@ -325,27 +323,27 @@ fn test_parsing_with_a_reset_after_reaching_an_operation_limit() {
parser.set_language(get_language("json")).unwrap();
parser.set_operation_limit(3);
let tree = parser.parse_str("[1234, 5, 6, 7, 8]", None);
let tree = parser.parse("[1234, 5, 6, 7, 8]", None);
assert!(tree.is_none());
// Without calling reset, the parser continues from where it left off, so
// it does not see the changes to the beginning of the source code.
parser.set_operation_limit(usize::MAX);
let tree = parser.parse_str("[null, 5, 6, 4, 5]", None).unwrap();
let tree = parser.parse("[null, 5, 6, 4, 5]", None).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(value (array (number) (number) (number) (number) (number)))"
);
parser.set_operation_limit(3);
let tree = parser.parse_str("[1234, 5, 6, 7, 8]", None);
let tree = parser.parse("[1234, 5, 6, 7, 8]", None);
assert!(tree.is_none());
// By calling reset, we force the parser to start over from scratch so
// that it sees the changes to the beginning of the source code.
parser.set_operation_limit(usize::MAX);
parser.reset();
let tree = parser.parse_str("[null, 5, 6, 4, 5]", None).unwrap();
let tree = parser.parse("[null, 5, 6, 4, 5]", None).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
"(value (array (null) (number) (number) (number) (number)))"
@ -360,13 +358,13 @@ fn test_parsing_with_one_included_range() {
let mut parser = Parser::new();
parser.set_language(get_language("html")).unwrap();
let html_tree = parser.parse_str(source_code, None).unwrap();
let html_tree = parser.parse(source_code, None).unwrap();
let script_content_node = html_tree.root_node().child(1).unwrap().child(1).unwrap();
assert_eq!(script_content_node.kind(), "raw_text");
parser.set_included_ranges(&[script_content_node.range()]);
parser.set_language(get_language("javascript")).unwrap();
let js_tree = parser.parse_str(source_code, None).unwrap();
let js_tree = parser.parse(source_code, None).unwrap();
assert_eq!(
js_tree.root_node().to_sexp(),
@ -388,7 +386,7 @@ fn test_parsing_with_multiple_included_ranges() {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
let js_tree = parser.parse_str(source_code, None).unwrap();
let js_tree = parser.parse(source_code, None).unwrap();
let template_string_node = js_tree
.root_node()
.descendant_for_byte_range(
@ -424,7 +422,7 @@ fn test_parsing_with_multiple_included_ranges() {
end_point: close_quote_node.start_position(),
},
]);
let html_tree = parser.parse_str(source_code, None).unwrap();
let html_tree = parser.parse(source_code, None).unwrap();
assert_eq!(
html_tree.root_node().to_sexp(),
@ -488,9 +486,7 @@ fn test_parsing_utf16_code_with_errors_at_the_end_of_an_included_range() {
start_point: Point::new(0, start_byte),
end_point: Point::new(0, end_byte),
}]);
let tree = parser
.parse_utf16(&mut |i, _| &utf16_source_code[i..], None)
.unwrap();
let tree = parser.parse_utf16(&utf16_source_code, None).unwrap();
assert_eq!(tree.root_node().to_sexp(), "(program (ERROR (identifier)))");
}
@ -519,7 +515,7 @@ fn test_parsing_with_external_scanner_that_uses_included_range_boundaries() {
},
]);
let tree = parser.parse_str(source_code, None).unwrap();
let tree = parser.parse(source_code, None).unwrap();
let root = tree.root_node();
let statement1 = root.child(0).unwrap();
let statement2 = root.child(1).unwrap();
@ -546,7 +542,7 @@ fn test_parsing_with_a_newly_excluded_range() {
// Parse HTML including the template directive, which will cause an error
let mut parser = Parser::new();
parser.set_language(get_language("html")).unwrap();
let mut first_tree = parser.parse_str(&source_code, None).unwrap();
let mut first_tree = parser.parse(&source_code, None).unwrap();
// Insert code at the beginning of the document.
let prefix = "a very very long line of plain text. ";
@ -579,7 +575,7 @@ fn test_parsing_with_a_newly_excluded_range() {
end_point: Point::new(0, source_code_end),
},
]);
let tree = parser.parse_str(&source_code, Some(&first_tree)).unwrap();
let tree = parser.parse(&source_code, Some(&first_tree)).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
@ -640,7 +636,7 @@ fn test_parsing_with_a_newly_included_range() {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
parser.set_included_ranges(&ranges[0..1]);
let first_tree = parser.parse_str(source_code, None).unwrap();
let first_tree = parser.parse(source_code, None).unwrap();
assert_eq!(
first_tree.root_node().to_sexp(),
concat!(
@ -651,7 +647,7 @@ fn test_parsing_with_a_newly_included_range() {
// Parse both the code directives as JavaScript, using the old tree as a reference.
parser.set_included_ranges(&ranges);
let tree = parser.parse_str(&source_code, Some(&first_tree)).unwrap();
let tree = parser.parse(&source_code, Some(&first_tree)).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
concat!(
@ -721,7 +717,7 @@ fn test_parsing_with_included_ranges_and_missing_tokens() {
},
]);
let tree = parser.parse_str(source_code, None).unwrap();
let tree = parser.parse(source_code, None).unwrap();
let root = tree.root_node();
assert_eq!(
root.to_sexp(),

4
cli/src/tests/properties_test.rs
View file

@ -38,7 +38,7 @@ fn test_walk_with_properties_with_nth_child() {
let mut parser = Parser::new();
parser.set_language(language).unwrap();
let tree = parser.parse_str(source_code, None).unwrap();
let tree = parser.parse(source_code, None).unwrap();
let mut cursor = tree.walk_with_properties(&property_sheet, source_code);
assert_eq!(cursor.node().kind(), "program");
@ -96,7 +96,7 @@ fn test_walk_with_properties_with_regexes() {
let mut parser = Parser::new();
parser.set_language(language).unwrap();
let tree = parser.parse_str(source_code, None).unwrap();
let tree = parser.parse(source_code, None).unwrap();
let mut cursor = tree.walk_with_properties(&property_sheet, source_code);
assert_eq!(cursor.node().kind(), "program");

14
cli/src/tests/tree_test.rs
View file

@ -7,7 +7,7 @@ use tree_sitter::{InputEdit, Parser, Point, Range, Tree};
fn test_tree_edit() {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
let tree = parser.parse_str(" abc !== def", None).unwrap();
let tree = parser.parse(" abc !== def", None).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
@ -194,7 +194,7 @@ fn test_tree_walk() {
parser.set_language(get_language("rust")).unwrap();
let tree = parser
.parse_str(
.parse(
"
struct Stuff {
a: A;
@ -228,7 +228,7 @@ fn test_tree_walk() {
fn test_tree_node_equality() {
let mut parser = Parser::new();
parser.set_language(get_language("rust")).unwrap();
let tree = parser.parse_str("struct A {}", None).unwrap();
let tree = parser.parse("struct A {}", None).unwrap();
let node1 = tree.root_node();
let node2 = tree.root_node();
assert_eq!(node1, node2);
@ -242,9 +242,7 @@ fn test_get_changed_ranges() {
let mut parser = Parser::new();
parser.set_language(get_language("javascript")).unwrap();
let tree = parser
.parse_utf8(&mut |i, _| &source_code[i..], None)
.unwrap();
let tree = parser.parse(&source_code, None).unwrap();
assert_eq!(
tree.root_node().to_sexp(),
@ -370,9 +368,7 @@ fn get_changed_ranges(
edit: Edit,
) -> Vec<Range> {
perform_edit(tree, source_code, &edit);
let new_tree = parser
.parse_utf8(&mut |i, _| &source_code[i..], Some(tree))
.unwrap();
let new_tree = parser.parse(&source_code, Some(tree)).unwrap();
let result = tree.changed_ranges(&new_tree);
*tree = new_tree;
result