diff --git a/cli/src/generate/mod.rs b/cli/src/generate/mod.rs index 127e956e..b00379af 100644 --- a/cli/src/generate/mod.rs +++ b/cli/src/generate/mod.rs @@ -58,7 +58,7 @@ pub fn generate_parser_in_directory( Ok(()) } -pub fn generate_parser_for_grammar(grammar_json: &String) -> Result<(String, String)> { +pub fn generate_parser_for_grammar(grammar_json: &str) -> Result<(String, String)> { let grammar_json = JSON_COMMENT_REGEX.replace_all(grammar_json, "\n"); generate_parser_for_grammar_with_opts(&grammar_json, true, Vec::new()) } diff --git a/cli/src/tests/corpus_test.rs b/cli/src/tests/corpus_test.rs index f1990963..c2e8b6c8 100644 --- a/cli/src/tests/corpus_test.rs +++ b/cli/src/tests/corpus_test.rs @@ -239,7 +239,7 @@ fn test_feature_corpus_files() { } else { let corpus_path = test_path.join("corpus.txt"); let c_code = generate_result.unwrap().1; - let language = get_test_language(language_name, c_code, &test_path); + let language = get_test_language(language_name, &c_code, Some(&test_path)); let test = parse_tests(&corpus_path).unwrap(); let tests = flatten_tests(test); diff --git a/cli/src/tests/helpers/fixtures.rs b/cli/src/tests/helpers/fixtures.rs index 981f0ab6..a5ea9ed0 100644 --- a/cli/src/tests/helpers/fixtures.rs +++ b/cli/src/tests/helpers/fixtures.rs @@ -20,7 +20,7 @@ pub fn get_language(name: &str) -> Language { .unwrap() } -pub fn get_test_language(name: &str, parser_code: String, path: &Path) -> Language { +pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) -> Language { let parser_c_path = SCRATCH_DIR.join(&format!("{}-parser.c", name)); if !fs::read_to_string(&parser_c_path) .map(|content| content == parser_code) @@ -28,12 +28,14 @@ pub fn get_test_language(name: &str, parser_code: String, path: &Path) -> Langua { fs::write(&parser_c_path, parser_code).unwrap(); } - let scanner_path = path.join("scanner.c"); - let scanner_path = if scanner_path.exists() { - Some(scanner_path) - } else { - None - }; + let scanner_path = path.and_then(|p| { + let result = p.join("scanner.c"); + if result.exists() { + Some(result) + } else { + None + } + }); TEST_LOADER .load_language_from_sources(name, &HEADER_DIR, &parser_c_path, &scanner_path) .unwrap() diff --git a/cli/src/tests/parser_test.rs b/cli/src/tests/parser_test.rs index 94694a32..6790d37f 100644 --- a/cli/src/tests/parser_test.rs +++ b/cli/src/tests/parser_test.rs @@ -1,4 +1,5 @@ -use super::helpers::fixtures::get_language; +use super::helpers::fixtures::{get_language, get_test_language}; +use crate::generate::generate_parser_for_grammar; use std::{thread, usize}; use tree_sitter::{InputEdit, LogType, Parser, Point, Range}; @@ -270,15 +271,18 @@ fn test_parsing_with_an_operation_limit() { // Start parsing from an infinite input. Parsing should abort after 5 "operations". parser.set_operation_limit(5); let mut call_count = 0; - let tree = parser.parse_utf8(&mut |_, _| { - if call_count == 0 { - call_count += 1; - b"[0" - } else { - call_count += 1; - b", 0" - } - }, None); + let tree = parser.parse_utf8( + &mut |_, _| { + if call_count == 0 { + call_count += 1; + b"[0" + } else { + call_count += 1; + b", 0" + } + }, + None, + ); assert!(tree.is_none()); assert!(call_count >= 3); assert!(call_count <= 8); @@ -286,15 +290,23 @@ fn test_parsing_with_an_operation_limit() { // Resume parsing from the previous state. call_count = 0; parser.set_operation_limit(20); - let tree = parser.parse_utf8(&mut |_, _| { - if call_count == 0 { - call_count += 1; - b"]" - } else { - b"" - } - }, None).unwrap(); - assert_eq!(tree.root_node().to_sexp(), "(value (array (number) (number) (number)))"); + let tree = parser + .parse_utf8( + &mut |_, _| { + if call_count == 0 { + call_count += 1; + b"]" + } else { + b"" + } + }, + None, + ) + .unwrap(); + assert_eq!( + tree.root_node().to_sexp(), + "(value (array (number) (number) (number)))" + ); } #[test] @@ -641,13 +653,70 @@ fn test_parsing_with_a_newly_included_range() { assert_eq!( tree.changed_ranges(&first_tree), - vec![ - Range { - start_byte: first_code_end_index + 1, - end_byte: second_code_end_index + 1, - start_point: Point::new(0, first_code_end_index + 1), - end_point: Point::new(0, second_code_end_index + 1), - } - ] + vec![Range { + start_byte: first_code_end_index + 1, + end_byte: second_code_end_index + 1, + start_point: Point::new(0, first_code_end_index + 1), + end_point: Point::new(0, second_code_end_index + 1), + }] ); } + +#[test] +fn test_parsing_with_included_ranges_and_missing_tokens() { + let (parser_name, parser_code) = generate_parser_for_grammar( + r#"{ + "name": "test_leading_missing_token", + "rules": { + "program": { + "type": "SEQ", + "members": [ + {"type": "SYMBOL", "name": "A"}, + {"type": "SYMBOL", "name": "b"}, + {"type": "SYMBOL", "name": "c"}, + {"type": "SYMBOL", "name": "A"}, + {"type": "SYMBOL", "name": "b"}, + {"type": "SYMBOL", "name": "c"} + ] + }, + "A": {"type": "SYMBOL", "name": "a"}, + "a": {"type": "STRING", "value": "a"}, + "b": {"type": "STRING", "value": "b"}, + "c": {"type": "STRING", "value": "c"} + } + }"#, + ) + .unwrap(); + + let mut parser = Parser::new(); + parser + .set_language(get_test_language(&parser_name, &parser_code, None)) + .unwrap(); + + // There's a missing `a` token at the beginning of the code. It must be inserted + // at the beginning of the first included range, not at {0, 0}. + let source_code = "__bc__bc__"; + parser.set_included_ranges(&[ + Range { + start_byte: 2, + end_byte: 4, + start_point: Point::new(0, 2), + end_point: Point::new(0, 4), + }, + Range { + start_byte: 6, + end_byte: 8, + start_point: Point::new(0, 6), + end_point: Point::new(0, 8), + }, + ]); + + let tree = parser.parse_str(source_code, None).unwrap(); + let root = tree.root_node(); + assert_eq!( + root.to_sexp(), + "(program (A (MISSING)) (b) (c) (A (MISSING)) (b) (c))" + ); + assert_eq!(root.start_byte(), 2); + assert_eq!(root.child(3).unwrap().start_byte(), 4); +}