From 94dc703bfcac0176feff732d12a930db8a70b6ea Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 4 Aug 2017 15:21:53 -0700 Subject: [PATCH] Require that grammars' start rules be visible --- include/tree_sitter/compiler.h | 1 + .../prepare_grammar/intern_symbols.cc | 3 +++ src/runtime/document.c | 5 +--- src/runtime/get_changed_ranges.c | 13 ++--------- test/fixtures/error_corpus/json_errors.txt | 13 +++++++---- .../test_grammars/inline_rules/grammar.json | 1 + .../invisible_start_rule/expected_error.txt | 1 + .../invisible_start_rule/grammar.json | 23 +++++++++++++++++++ test/integration/real_grammars.cc | 14 +---------- test/runtime/document_test.cc | 20 ++++++++-------- test/runtime/node_test.cc | 6 ++--- test/runtime/parser_test.cc | 20 ++++++++-------- 12 files changed, 64 insertions(+), 56 deletions(-) create mode 100644 test/fixtures/test_grammars/invisible_start_rule/expected_error.txt create mode 100644 test/fixtures/test_grammars/invisible_start_rule/grammar.json diff --git a/include/tree_sitter/compiler.h b/include/tree_sitter/compiler.h index eda0e51a..30f4e99c 100644 --- a/include/tree_sitter/compiler.h +++ b/include/tree_sitter/compiler.h @@ -16,6 +16,7 @@ typedef enum { TSCompileErrorTypeParseConflict, TSCompileErrorTypeEpsilonRule, TSCompileErrorTypeInvalidTokenContents, + TSCompileErrorTypeInvalidRuleName, } TSCompileErrorType; typedef struct { diff --git a/src/compiler/prepare_grammar/intern_symbols.cc b/src/compiler/prepare_grammar/intern_symbols.cc index 7bb2a80b..0d515fad 100644 --- a/src/compiler/prepare_grammar/intern_symbols.cc +++ b/src/compiler/prepare_grammar/intern_symbols.cc @@ -109,6 +109,9 @@ pair intern_symbols(const InputGrammar &grammar) }); } + if (grammar.variables[0].name[0] == '_') { + return {result, CompileError(TSCompileErrorTypeInvalidRuleName, "A grammar's start rule must be visible.")}; + } for (auto &variable : grammar.variables) { auto new_rule = interner.apply(variable.rule); diff --git a/src/runtime/document.c b/src/runtime/document.c index ed4b2b62..d7154a78 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -159,10 +159,7 @@ void ts_document_invalidate(TSDocument *self) { } TSNode ts_document_root_node(const TSDocument *self) { - TSNode result = ts_node_make(self->tree, 0, 0, 0); - while (result.data && !((Tree *)result.data)->visible) - result = ts_node_named_child(result, 0); - return result; + return ts_node_make(self->tree, 0, 0, 0); } uint32_t ts_document_parse_count(const TSDocument *self) { diff --git a/src/runtime/get_changed_ranges.c b/src/runtime/get_changed_ranges.c index af45a8cc..75f32763 100644 --- a/src/runtime/get_changed_ranges.c +++ b/src/runtime/get_changed_ranges.c @@ -148,20 +148,11 @@ static Iterator iterator_new(TreePath *path, Tree *tree, const TSLanguage *langu .child_index = 0, .structural_child_index = 0, })); - - Iterator self = { + return (Iterator) { .path = *path, .language = language, - .visible_depth = 0, + .visible_depth = 1, }; - - if (tree->visible) { - self.visible_depth++; - } else { - iterator_descend(&self, length_zero()); - } - - return self; } Length iterator_start_position(Iterator *self) { diff --git a/test/fixtures/error_corpus/json_errors.txt b/test/fixtures/error_corpus/json_errors.txt index 37b82e14..90fa3d40 100644 --- a/test/fixtures/error_corpus/json_errors.txt +++ b/test/fixtures/error_corpus/json_errors.txt @@ -25,10 +25,11 @@ errors inside arrays [1, , 2] --- -(array + +(value (array (number) (ERROR) - (number)) + (number))) ========================================== errors inside objects @@ -38,7 +39,9 @@ errors inside objects --- -(object (pair (string) (number)) (ERROR (UNEXPECTED 'o'))) +(value (object + (pair (string) (number)) + (ERROR (UNEXPECTED 'o')))) ========================================== errors inside nested objects @@ -48,9 +51,9 @@ errors inside nested objects --- -(object +(value (object (pair (string) (object (pair (string) (number)) (ERROR (number)))) (ERROR) - (pair (string) (number))) + (pair (string) (number)))) diff --git a/test/fixtures/test_grammars/inline_rules/grammar.json b/test/fixtures/test_grammars/inline_rules/grammar.json index 7825314b..ea545147 100644 --- a/test/fixtures/test_grammars/inline_rules/grammar.json +++ b/test/fixtures/test_grammars/inline_rules/grammar.json @@ -17,6 +17,7 @@ "name": "statement" } }, + "statement": { "type": "SEQ", "members": [ diff --git a/test/fixtures/test_grammars/invisible_start_rule/expected_error.txt b/test/fixtures/test_grammars/invisible_start_rule/expected_error.txt new file mode 100644 index 00000000..27a2eea6 --- /dev/null +++ b/test/fixtures/test_grammars/invisible_start_rule/expected_error.txt @@ -0,0 +1 @@ +A grammar's start rule must be visible. \ No newline at end of file diff --git a/test/fixtures/test_grammars/invisible_start_rule/grammar.json b/test/fixtures/test_grammars/invisible_start_rule/grammar.json new file mode 100644 index 00000000..809624bd --- /dev/null +++ b/test/fixtures/test_grammars/invisible_start_rule/grammar.json @@ -0,0 +1,23 @@ +{ + "name": "invisible_start_rule", + + "rules": { + "_value": { + "type": "CHOICE", + "members": [ + {"type": "SYMBOL", "name": "a"}, + {"type": "SYMBOL", "name": "b"} + ] + }, + + "a": { + "type": "STRING", + "value": "a" + }, + + "b": { + "type": "STRING", + "value": "b" + } + } +} \ No newline at end of file diff --git a/test/integration/real_grammars.cc b/test/integration/real_grammars.cc index 3647795c..f494dcdc 100644 --- a/test/integration/real_grammars.cc +++ b/test/integration/real_grammars.cc @@ -14,19 +14,7 @@ static void assert_correct_tree_size(TSDocument *document, string content) { TSNode root_node = ts_document_root_node(document); - size_t expected_size = content.size(); - - // In the JSON grammar, the start rule (`_value`) is hidden, so the node - // returned from `ts_document_root_node` (e.g. an `object` node), does not - // actually point to the root of the tree. In this weird case, trailing - // whitespace is not included in the root node's size. - // - // TODO: Fix this inconsistency. Maybe disallow the start rule being hidden? - if (ts_document_language(document) == load_real_language("json") && - string(ts_node_type(root_node, document)) != "ERROR") - expected_size = content.find_last_not_of("\n ") + 1; - - AssertThat(ts_node_end_byte(root_node), Equals(expected_size)); + AssertThat(ts_node_end_byte(root_node), Equals(content.size())); assert_consistent_tree_sizes(root_node); } diff --git a/test/runtime/document_test.cc b/test/runtime/document_test.cc index 3ab638b8..054d7df3 100644 --- a/test/runtime/document_test.cc +++ b/test/runtime/document_test.cc @@ -52,7 +52,7 @@ describe("Document", [&]() { root = ts_document_root_node(document); assert_node_string_equals( root, - "(object (pair (string) (array (number) (number))))"); + "(value (object (pair (string) (array (number) (number)))))"); }); after_each([&]() { @@ -71,7 +71,7 @@ describe("Document", [&]() { root = ts_document_root_node(document); assert_node_string_equals( root, - "(array (true) (false))"); + "(value (array (true) (false)))"); }); it("handles truncated UTF16 data", [&]() { @@ -96,14 +96,14 @@ describe("Document", [&]() { ts_document_parse(document); TSNode root = ts_document_root_node(document); - AssertThat(ts_node_end_point(root), Equals({0, 13})); + AssertThat(ts_node_end_point(root), Equals({0, 14})); input.measure_columns_in_bytes = true; ts_document_set_input(document, input); ts_document_invalidate(document); ts_document_parse(document); root = ts_document_root_node(document); - AssertThat(ts_node_end_point(root), Equals({0, 26})); + AssertThat(ts_node_end_point(root), Equals({0, 28})); }); it("allows the input to be retrieved later", [&]() { @@ -135,7 +135,7 @@ describe("Document", [&]() { TSNode new_root = ts_document_root_node(document); assert_node_string_equals( new_root, - "(object (pair (string) (array (null) (number))))"); + "(value (object (pair (string) (array (null) (number)))))"); AssertThat(spy_input->strings_read(), Equals(vector({" [null, 2" }))); }); @@ -147,7 +147,7 @@ describe("Document", [&]() { AssertThat(ts_node_end_char(new_root), Equals(1)); assert_node_string_equals( new_root, - "(number)"); + "(value (number))"); }); it("reads from the new input correctly when the old input was blank", [&]() { @@ -165,7 +165,7 @@ describe("Document", [&]() { AssertThat(ts_node_end_char(new_root), Equals(1)); assert_node_string_equals( new_root, - "(number)"); + "(value (number))"); }); }); @@ -181,7 +181,7 @@ describe("Document", [&]() { root = ts_document_root_node(document); assert_node_string_equals( root, - "(object (pair (string) (array (number) (number))))"); + "(value (object (pair (string) (array (number) (number)))))"); }); it("clears out any previous tree", [&]() { @@ -405,7 +405,7 @@ describe("Document", [&]() { root = ts_document_root_node(document); assert_node_string_equals( root, - "(array (number) (null) (ERROR (UNEXPECTED 'e')) (number))"); + "(value (array (number) (null) (ERROR (UNEXPECTED 'e')) (number)))"); ts_document_invalidate(document); @@ -432,7 +432,7 @@ describe("Document", [&]() { root = ts_document_root_node(document); assert_node_string_equals( root, - "(array (number) (null) (number))"); + "(value (array (number) (null) (number)))"); }); }); }); diff --git a/test/runtime/node_test.cc b/test/runtime/node_test.cc index d458f8b3..a18aed6d 100644 --- a/test/runtime/node_test.cc +++ b/test/runtime/node_test.cc @@ -72,7 +72,7 @@ describe("Node", [&]() { ts_document_set_language(document, load_real_language("json")); ts_document_set_input_string(document, json_string.c_str()); ts_document_parse(document); - root_node = ts_document_root_node(document); + root_node = ts_node_child(ts_document_root_node(document), 0); }); after_each([&]() { @@ -161,7 +161,7 @@ describe("Node", [&]() { AssertThat(ts_node_parent(number_node), Equals(root_node)); AssertThat(ts_node_parent(false_node), Equals(root_node)); AssertThat(ts_node_parent(object_node), Equals(root_node)); - AssertThat(ts_node_parent(root_node).data, Equals(nullptr)); + AssertThat(ts_node_parent(ts_document_root_node(document)).data, Equals(nullptr)); }); it("works correctly when the node contains aliased children and extras", [&]() { @@ -293,7 +293,7 @@ describe("Node", [&]() { AssertThat(ts_node_parent(child5), Equals(root_node)); AssertThat(ts_node_parent(child6), Equals(root_node)); AssertThat(ts_node_parent(child7), Equals(root_node)); - AssertThat(ts_node_parent(root_node).data, Equals(nullptr)); + AssertThat(ts_node_parent(ts_document_root_node(document)).data, Equals(nullptr)); }); }); diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index a7557685..2f60c0f0 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -87,9 +87,9 @@ describe("Parser", [&]() { set_text(" [123, @@@@@, true]"); assert_root_node( - "(array (number) (ERROR (UNEXPECTED '@')) (true))"); + "(value (array (number) (ERROR (UNEXPECTED '@')) (true)))"); - TSNode error = ts_node_named_child(root, 1); + TSNode error = ts_node_named_child(ts_node_child(root, 0), 1); AssertThat(ts_node_type(error, document), Equals("ERROR")); AssertThat(get_node_text(error), Equals(", @@@@@")); AssertThat(ts_node_child_count(error), Equals(2)); @@ -100,7 +100,7 @@ describe("Parser", [&]() { TSNode garbage = ts_node_child(error, 1); AssertThat(get_node_text(garbage), Equals("@@@@@")); - TSNode node_after_error = ts_node_named_child(root, 2); + TSNode node_after_error = ts_node_next_named_sibling(error); AssertThat(ts_node_type(node_after_error, document), Equals("true")); AssertThat(get_node_text(node_after_error), Equals("true")); }); @@ -112,9 +112,9 @@ describe("Parser", [&]() { set_text(" [123, faaaaalse, true]"); assert_root_node( - "(array (number) (ERROR (UNEXPECTED 'a')) (true))"); + "(value (array (number) (ERROR (UNEXPECTED 'a')) (true)))"); - TSNode error = ts_node_named_child(root, 1); + TSNode error = ts_node_named_child(ts_node_child(root, 0), 1); AssertThat(ts_node_type(error, document), Equals("ERROR")); AssertThat(ts_node_child_count(error), Equals(2)); @@ -126,7 +126,7 @@ describe("Parser", [&]() { AssertThat(ts_node_type(garbage, document), Equals("ERROR")); AssertThat(get_node_text(garbage), Equals("faaaaalse")); - TSNode last = ts_node_named_child(root, 2); + TSNode last = ts_node_next_named_sibling(error); AssertThat(ts_node_type(last, document), Equals("true")); AssertThat(ts_node_start_byte(last), Equals(strlen(" [123, faaaaalse, "))); }); @@ -138,14 +138,14 @@ describe("Parser", [&]() { set_text(" [123, true false, true]"); assert_root_node( - "(array (number) (true) (ERROR (false)) (true))"); + "(value (array (number) (true) (ERROR (false)) (true)))"); - TSNode error = ts_node_named_child(root, 2); + TSNode error = ts_node_named_child(ts_node_child(root, 0), 2); AssertThat(ts_node_type(error, document), Equals("ERROR")); AssertThat(get_node_text(error), Equals("false")); AssertThat(ts_node_child_count(error), Equals(1)); - TSNode last = ts_node_named_child(root, 1); + TSNode last = ts_node_next_named_sibling(error); AssertThat(ts_node_type(last, document), Equals("true")); AssertThat(get_node_text(last), Equals("true")); }); @@ -157,7 +157,7 @@ describe("Parser", [&]() { set_text(" [123, \"hi\n, true]"); assert_root_node( - "(array (number) (ERROR (UNEXPECTED '\\n')) (true))"); + "(value (array (number) (ERROR (UNEXPECTED '\\n')) (true)))"); }); });