traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix size calculation for error nodes

Browse source
This commit is contained in:
Max Brunsfeld 2014-08-29 13:22:03 -07:00
parent 604b149c4b
commit c5ac02c571
3 changed files with 107 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

131
spec/runtime/document_spec.cc
View file

@ -1,6 +1,5 @@
#include "runtime/runtime_spec_helper.h"
#include "runtime/helpers/spy_reader.h"
#include "runtime/tree.h"
extern "C" const TSLanguage * ts_language_json();
@ -17,37 +16,50 @@ describe("Document", [&]() {
ts_document_free(doc);
});
describe("when the language is not set", [&]() {
describe("setting the input", [&]() {
SpyReader *reader;
describe("set_input", [&]() {
SpyReader *reader;
before_each([&]() {
reader = new SpyReader("{ \"key\": [1, 2] }", 5);
});
after_each([&]() {
delete reader;
});
describe("when the language is set", [&]() {
before_each([&]() {
reader = new SpyReader("{ \"key\": [1, 2] }", 5);
ts_document_set_language(doc, ts_language_json());
});
it("parses the document", [&]() {
SpyReader *reader = new SpyReader("{ \"key\": [1, 2] }", 5);
ts_document_set_input(doc, reader->input);
AssertThat(string(ts_node_string(ts_document_root_node(doc))), Equals(
"(DOCUMENT (object (string) (array (number) (number))))"
));
});
after_each([&]() {
delete reader;
it("reads the entire input", [&]() {
SpyReader *reader = new SpyReader("{ \"key\": [1, 2] }", 5);
ts_document_set_input(doc, reader->input);
AssertThat(reader->strings_read, Equals(vector<string>({
"{ \"key\": [1, 2] }"
})));
});
});
describe("when the language is not set", [&]() {
it("does not try to parse the document", [&]() {
ts_document_set_input(doc, reader->input);
AssertThat(ts_document_root_node(doc), Equals<TSNode *>(nullptr));
});
describe("setting the language", [&]() {
before_each([&]() {
ts_document_set_language(doc, ts_language_json());
});
it("parses the document", [&]() {
AssertThat(string(ts_node_string(ts_document_root_node(doc))), Equals(
"(DOCUMENT (object (string) (array (number) (number))))"));
});
});
});
});
describe("setting the language and input", [&]() {
describe("edit", [&]() {
SpyReader *reader;
before_each([&]() {
@ -60,17 +72,6 @@ describe("Document", [&]() {
delete reader;
});
it("parses the input", [&]() {
AssertThat(string(ts_node_string(ts_document_root_node(doc))), Equals(
"(DOCUMENT (object (string) (array (number) (number))))"));
});
it("reads the entire input", [&]() {
AssertThat(reader->strings_read, Equals(vector<string>({
"{ \"key\": [1, 2] }"
})));
});
describe("modifying the end of the input", [&]() {
before_each([&]() {
size_t position(string("{ \"key\": [1, 2]").length());
@ -111,6 +112,76 @@ describe("Document", [&]() {
});
});
});
describe("parsing", [&]() {
TSNode *root;
describe("error handling", [&]() {
before_each([&]() {
ts_document_set_language(doc, ts_language_json());
});
describe("when the error occurs at the beginning of a token", [&]() {
it("computes the error node's size and position correctly", [&]() {
ts_document_set_input_string(doc, " [123, @@@@@, true]");
AssertThat(ts_node_string(ts_document_root_node(doc)), Equals(
"(DOCUMENT (array (number) (ERROR '@') (true)))"));
root = ts_document_root_node(doc);
TSNode *array = ts_node_child(root, 0);
TSNode *error = ts_node_child(array, 1);
AssertThat(ts_node_name(error), Equals("error"));
AssertThat(ts_node_pos(error), Equals(string(" [123,").length()))
AssertThat(ts_node_size(error), Equals(string(" @@@@@").length()))
ts_node_release(error);
ts_node_release(array);
ts_node_release(error);
});
});
describe("when the error occurs in the middle of a token", [&]() {
it("computes the error node's size and position correctly", [&]() {
ts_document_set_input_string(doc, " [123, total nonsense, true]");
AssertThat(ts_node_string(ts_document_root_node(doc)), Equals(
"(DOCUMENT (array (number) (ERROR 'o') (true)))"));
root = ts_document_root_node(doc);
TSNode *array = ts_node_child(root, 0);
TSNode *error = ts_node_child(array, 1);
AssertThat(ts_node_name(error), Equals("error"));
AssertThat(ts_node_pos(error), Equals(string(" [123,").length()))
AssertThat(ts_node_size(error), Equals(string(" total nonsense").length()))
ts_node_release(error);
ts_node_release(array);
ts_node_release(error);
});
});
describe("when the error occurs after one or more tokens", [&]() {
it("computes the error node's size and position correctly", [&]() {
ts_document_set_input_string(doc, " [123, true false, true]");
AssertThat(ts_node_string(ts_document_root_node(doc)), Equals(
"(DOCUMENT (array (number) (ERROR 'f') (true)))"));
root = ts_document_root_node(doc);
TSNode *array = ts_node_child(root, 0);
TSNode *error = ts_node_child(array, 1);
AssertThat(ts_node_name(error), Equals("error"));
AssertThat(ts_node_pos(error), Equals(string(" [123,").length()))
AssertThat(ts_node_size(error), Equals(string(" true false").length()))
ts_node_release(error);
ts_node_release(array);
ts_node_release(error);
});
});
});
});
});
END_TEST

22
spec/runtime/node_spec.cc
View file

@ -152,28 +152,6 @@ describe("Node", []() {
ts_node_release(node);
});
});
describe("error nodes", [&]() {
before_each([&]() {
ts_document_set_input_string(document, " [123, total nonsense, true]");
AssertThat(ts_node_string(ts_document_root_node(document)), Equals(
"(DOCUMENT (array (number) (ERROR 'o') (true)))"));
root = ts_document_root_node(document);
});
it("computes their size and position correctly", [&]() {
TSNode *array = ts_node_child(root, 0);
TSNode *error = ts_node_child(array, 1);
AssertThat(ts_node_name(error), Equals("error"));
AssertThat(ts_node_pos(error), Equals(string(" [123,").length()))
AssertThat(ts_node_size(error), Equals(string(" total nonsense").length()))
ts_node_release(array);
ts_node_release(error);
});
});
});
END_TEST

8
src/runtime/parser.c
View file

@ -132,7 +132,7 @@ static void lex(TSParser *parser, TSStateId lex_state) {
static int handle_error(TSParser *parser) {
TSTree *error = parser->lookahead;
ts_tree_retain(error);
size_t start_position = ts_lexer_position(&parser->lexer);
size_t last_token_end = parser->lexer.token_end_position;
for (;;) {
@ -155,6 +155,7 @@ static int handle_error(TSParser *parser) {
* Unwind the parse stack until a state is found in which an error is
* expected and the current lookahead token is expected afterwards.
*/
size_t error_start_pos = last_token_end;
for (size_t i = parser->stack.size - 1; i + 1 > 0; i--) {
TSStateId state = parser->stack.entries[i].state;
TSParseAction action_on_error =
@ -167,13 +168,16 @@ static int handle_error(TSParser *parser) {
if (action_after_error.type != TSParseActionTypeError) {
DEBUG_PARSE("RECOVER %u", state_after_error);
error->size = ts_lexer_position(&parser->lexer) - start_position + 1;
error->size = ts_lexer_position(&parser->lexer) - error_start_pos - 1;
ts_stack_shrink(&parser->stack, i + 1);
ts_stack_push(&parser->stack, state_after_error, error);
ts_tree_release(error);
return 1;
}
}
TSTree *removed_tree = parser->stack.entries[i].node;
error_start_pos -= ts_tree_total_size(removed_tree);
}
}
}