traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix infinite loop in certain cases w/ unterminated tokens

Browse source
This commit is contained in:
Max Brunsfeld 2014-09-03 00:38:44 -07:00
parent 7d81126df3
commit 77529ace3d
2 changed files with 47 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

9
include/tree_sitter/parser.h
View file

@ -47,10 +47,6 @@ static inline TSTree *ts_lexer_accept(TSLexer *lexer, TSSymbol symbol,
return lexer->accept_fn(lexer, symbol, is_hidden);
}
static inline int ts_lexer_is_done(const TSLexer *lexer) {
return lexer->chunk_size == 0 && lexer->position_in_chunk > 0;
}
typedef unsigned short TSStateId;
typedef enum {
@ -111,11 +107,10 @@ struct TSLanguage {
#define ADVANCE(state_index) \
{ \
DEBUG_LEX("ADVANCE %d", state_index); \
if (ts_lexer_is_done(lexer)) { \
if (!ts_lexer_advance(lexer)) { \
DEBUG_LEX("END"); \
return ts_lexer_accept(lexer, ts_builtin_sym_error, 0); \
return ts_lexer_accept(lexer, ts_builtin_sym_end, 0); \
} \
ts_lexer_advance(lexer); \
lex_state = state_index; \
goto next_state; \
}

58
spec/runtime/document_spec.cc
View file

@ -76,41 +76,73 @@ describe("Document", [&]() {
delete reader;
});
describe("modifying the end of the input", [&]() {
before_each([&]() {
size_t position(string("{ \"key\": [1, 2]").length());
string inserted_text(", \"key2\": 4");
auto insert_text = [&](size_t position, string text) {
reader->content.insert(position, text);
ts_document_edit(doc, { position, 0, text.length() });
};
reader->content.insert(position, inserted_text);
ts_document_edit(doc, { position, 0, inserted_text.length() });
});
auto delete_text = [&](size_t position, size_t length) {
reader->content.erase(position, length);
ts_document_edit(doc, { position, length, 0 });
};
describe("inserting tokens near the end of the input", [&]() {
it("updates the parse tree", [&]() {
insert_text(
string("{ \"key\": [1, 2]").length(),
", \"key2\": 4");
AssertThat(string(ts_node_string(ts_document_root_node(doc))), Equals(
"(DOCUMENT (object (string) (array (number) (number)) (string) (number)))"));
});
it("re-reads only the changed portion of the input", [&]() {
insert_text(
string("{ \"key\": [1, 2]").length(),
", \"key2\": 4");
AssertThat(reader->strings_read.size(), Equals<size_t>(2));
AssertThat(reader->strings_read[1], Equals(", \"key2\": 4 }"));
});
});
describe("modifying the beginning of the input", [&]() {
before_each([&]() {
size_t position(string("{ ").length());
string inserted_text("\"key2\": 4, ");
describe("inserting text into an existing token", [&]() {
it("updates the parse three", [&]() {
insert_text(
string("{ \"key").length(),
"12345");
reader->content.insert(position, inserted_text);
ts_document_edit(doc, { position, 0, inserted_text.length() });
AssertThat(string(ts_node_string(ts_document_root_node(doc))), Equals(
"(DOCUMENT (object (string) (array (number) (number))))"));
});
});
describe("deleting an important part of a token", [&]() {
it("updates the parse tree, creating an error", [&]() {
delete_text(
string("{ \"key").length(),
1);
AssertThat(string(ts_node_string(ts_document_root_node(doc))), Equals(
"(DOCUMENT (end))"));
});
});
describe("inserting tokens near the beginning of the input", [&]() {
it("updates the parse tree", [&]() {
insert_text(
string("{ ").length(),
"\"key2\": 4, ");
AssertThat(string(ts_node_string(ts_document_root_node(doc))), Equals(
"(DOCUMENT (object (string) (number) (string) (array (number) (number))))"));
});
it_skip("re-reads only the changed portion of the input", [&]() {
insert_text(
string("{ ").length(),
"\"key2\": 4, ");
AssertThat(reader->strings_read.size(), Equals<size_t>(2));
AssertThat(reader->strings_read[1], Equals("\"key2\": 4, "));
});