From c5ac02c571eceed6819e3e8e265b1dc225fce666 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 29 Aug 2014 13:22:03 -0700 Subject: [PATCH] Fix size calculation for error nodes --- spec/runtime/document_spec.cc | 131 ++++++++++++++++++++++++++-------- spec/runtime/node_spec.cc | 22 ------ src/runtime/parser.c | 8 ++- 3 files changed, 107 insertions(+), 54 deletions(-) diff --git a/spec/runtime/document_spec.cc b/spec/runtime/document_spec.cc index b3a28178..84578cba 100644 --- a/spec/runtime/document_spec.cc +++ b/spec/runtime/document_spec.cc @@ -1,6 +1,5 @@ #include "runtime/runtime_spec_helper.h" #include "runtime/helpers/spy_reader.h" -#include "runtime/tree.h" extern "C" const TSLanguage * ts_language_json(); @@ -17,37 +16,50 @@ describe("Document", [&]() { ts_document_free(doc); }); - describe("when the language is not set", [&]() { - describe("setting the input", [&]() { - SpyReader *reader; + describe("set_input", [&]() { + SpyReader *reader; + before_each([&]() { + reader = new SpyReader("{ \"key\": [1, 2] }", 5); + }); + + after_each([&]() { + delete reader; + }); + + describe("when the language is set", [&]() { before_each([&]() { - reader = new SpyReader("{ \"key\": [1, 2] }", 5); + ts_document_set_language(doc, ts_language_json()); + }); + + it("parses the document", [&]() { + SpyReader *reader = new SpyReader("{ \"key\": [1, 2] }", 5); ts_document_set_input(doc, reader->input); + + AssertThat(string(ts_node_string(ts_document_root_node(doc))), Equals( + "(DOCUMENT (object (string) (array (number) (number))))" + )); }); - after_each([&]() { - delete reader; + it("reads the entire input", [&]() { + SpyReader *reader = new SpyReader("{ \"key\": [1, 2] }", 5); + ts_document_set_input(doc, reader->input); + AssertThat(reader->strings_read, Equals(vector({ + "{ \"key\": [1, 2] }" + }))); }); + }); + describe("when the language is not set", [&]() { it("does not try to parse the document", [&]() { + ts_document_set_input(doc, reader->input); + AssertThat(ts_document_root_node(doc), Equals(nullptr)); }); - - describe("setting the language", [&]() { - before_each([&]() { - ts_document_set_language(doc, ts_language_json()); - }); - - it("parses the document", [&]() { - AssertThat(string(ts_node_string(ts_document_root_node(doc))), Equals( - "(DOCUMENT (object (string) (array (number) (number))))")); - }); - }); }); }); - describe("setting the language and input", [&]() { + describe("edit", [&]() { SpyReader *reader; before_each([&]() { @@ -60,17 +72,6 @@ describe("Document", [&]() { delete reader; }); - it("parses the input", [&]() { - AssertThat(string(ts_node_string(ts_document_root_node(doc))), Equals( - "(DOCUMENT (object (string) (array (number) (number))))")); - }); - - it("reads the entire input", [&]() { - AssertThat(reader->strings_read, Equals(vector({ - "{ \"key\": [1, 2] }" - }))); - }); - describe("modifying the end of the input", [&]() { before_each([&]() { size_t position(string("{ \"key\": [1, 2]").length()); @@ -111,6 +112,76 @@ describe("Document", [&]() { }); }); }); + + describe("parsing", [&]() { + TSNode *root; + + describe("error handling", [&]() { + before_each([&]() { + ts_document_set_language(doc, ts_language_json()); + }); + + describe("when the error occurs at the beginning of a token", [&]() { + it("computes the error node's size and position correctly", [&]() { + ts_document_set_input_string(doc, " [123, @@@@@, true]"); + AssertThat(ts_node_string(ts_document_root_node(doc)), Equals( + "(DOCUMENT (array (number) (ERROR '@') (true)))")); + + root = ts_document_root_node(doc); + TSNode *array = ts_node_child(root, 0); + TSNode *error = ts_node_child(array, 1); + + AssertThat(ts_node_name(error), Equals("error")); + AssertThat(ts_node_pos(error), Equals(string(" [123,").length())) + AssertThat(ts_node_size(error), Equals(string(" @@@@@").length())) + + ts_node_release(error); + ts_node_release(array); + ts_node_release(error); + }); + }); + + describe("when the error occurs in the middle of a token", [&]() { + it("computes the error node's size and position correctly", [&]() { + ts_document_set_input_string(doc, " [123, total nonsense, true]"); + AssertThat(ts_node_string(ts_document_root_node(doc)), Equals( + "(DOCUMENT (array (number) (ERROR 'o') (true)))")); + + root = ts_document_root_node(doc); + TSNode *array = ts_node_child(root, 0); + TSNode *error = ts_node_child(array, 1); + + AssertThat(ts_node_name(error), Equals("error")); + AssertThat(ts_node_pos(error), Equals(string(" [123,").length())) + AssertThat(ts_node_size(error), Equals(string(" total nonsense").length())) + + ts_node_release(error); + ts_node_release(array); + ts_node_release(error); + }); + }); + + describe("when the error occurs after one or more tokens", [&]() { + it("computes the error node's size and position correctly", [&]() { + ts_document_set_input_string(doc, " [123, true false, true]"); + AssertThat(ts_node_string(ts_document_root_node(doc)), Equals( + "(DOCUMENT (array (number) (ERROR 'f') (true)))")); + + root = ts_document_root_node(doc); + TSNode *array = ts_node_child(root, 0); + TSNode *error = ts_node_child(array, 1); + + AssertThat(ts_node_name(error), Equals("error")); + AssertThat(ts_node_pos(error), Equals(string(" [123,").length())) + AssertThat(ts_node_size(error), Equals(string(" true false").length())) + + ts_node_release(error); + ts_node_release(array); + ts_node_release(error); + }); + }); + }); + }); }); END_TEST diff --git a/spec/runtime/node_spec.cc b/spec/runtime/node_spec.cc index bc740515..c0fa30d2 100644 --- a/spec/runtime/node_spec.cc +++ b/spec/runtime/node_spec.cc @@ -152,28 +152,6 @@ describe("Node", []() { ts_node_release(node); }); }); - - describe("error nodes", [&]() { - before_each([&]() { - ts_document_set_input_string(document, " [123, total nonsense, true]"); - AssertThat(ts_node_string(ts_document_root_node(document)), Equals( - "(DOCUMENT (array (number) (ERROR 'o') (true)))")); - - root = ts_document_root_node(document); - }); - - it("computes their size and position correctly", [&]() { - TSNode *array = ts_node_child(root, 0); - TSNode *error = ts_node_child(array, 1); - - AssertThat(ts_node_name(error), Equals("error")); - AssertThat(ts_node_pos(error), Equals(string(" [123,").length())) - AssertThat(ts_node_size(error), Equals(string(" total nonsense").length())) - - ts_node_release(array); - ts_node_release(error); - }); - }); }); END_TEST diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 58365ae9..ee555fab 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -132,7 +132,7 @@ static void lex(TSParser *parser, TSStateId lex_state) { static int handle_error(TSParser *parser) { TSTree *error = parser->lookahead; ts_tree_retain(error); - size_t start_position = ts_lexer_position(&parser->lexer); + size_t last_token_end = parser->lexer.token_end_position; for (;;) { @@ -155,6 +155,7 @@ static int handle_error(TSParser *parser) { * Unwind the parse stack until a state is found in which an error is * expected and the current lookahead token is expected afterwards. */ + size_t error_start_pos = last_token_end; for (size_t i = parser->stack.size - 1; i + 1 > 0; i--) { TSStateId state = parser->stack.entries[i].state; TSParseAction action_on_error = @@ -167,13 +168,16 @@ static int handle_error(TSParser *parser) { if (action_after_error.type != TSParseActionTypeError) { DEBUG_PARSE("RECOVER %u", state_after_error); - error->size = ts_lexer_position(&parser->lexer) - start_position + 1; + error->size = ts_lexer_position(&parser->lexer) - error_start_pos - 1; ts_stack_shrink(&parser->stack, i + 1); ts_stack_push(&parser->stack, state_after_error, error); ts_tree_release(error); return 1; } } + + TSTree *removed_tree = parser->stack.entries[i].node; + error_start_pos -= ts_tree_total_size(removed_tree); } } }