From a98d449d886193c8d12a1b254867ab28925db175 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 1 May 2017 13:04:06 -0700 Subject: [PATCH] Add an option to immediately halt on syntax error --- include/tree_sitter/runtime.h | 9 +++++++ src/runtime/document.c | 32 +++++++++++++++------- src/runtime/lexer.c | 4 +++ src/runtime/lexer.h | 1 + src/runtime/parser.c | 50 +++++++++++++++++++++++++++++------ src/runtime/parser.h | 2 +- test/runtime/document_test.cc | 30 +++++++++++++++++++++ 7 files changed, 110 insertions(+), 18 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 6f37c9a3..95da0787 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -117,6 +117,15 @@ void ts_document_print_debugging_graphs(TSDocument *, bool); void ts_document_edit(TSDocument *, TSInputEdit); void ts_document_parse(TSDocument *); void ts_document_parse_and_get_changed_ranges(TSDocument *, TSRange **, uint32_t *); + +typedef struct { + TSRange **changed_ranges; + uint32_t *changed_range_count; + bool halt_on_error; +} TSParseOptions; + +void ts_document_parse_with_options(TSDocument *, TSParseOptions); + void ts_document_invalidate(TSDocument *); TSNode ts_document_root_node(const TSDocument *); uint32_t ts_document_parse_count(const TSDocument *); diff --git a/src/runtime/document.c b/src/runtime/document.c index ff09610d..6bcc5fbc 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -99,10 +99,28 @@ void ts_document_edit(TSDocument *self, TSInputEdit edit) { ts_tree_edit(self->tree, &edit); } +void ts_document_parse(TSDocument *self) { + return ts_document_parse_with_options(self, (TSParseOptions){ + .halt_on_error = false, + .changed_ranges = NULL, + .changed_range_count = NULL, + }); +} + void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges, uint32_t *range_count) { - if (ranges) *ranges = NULL; - if (range_count) *range_count = 0; + return ts_document_parse_with_options(self, (TSParseOptions){ + .halt_on_error = false, + .changed_ranges = ranges, + .changed_range_count = range_count, + }); +} + +void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) { + if (options.changed_ranges && options.changed_range_count) { + *options.changed_ranges = NULL; + *options.changed_range_count = 0; + } if (!self->input.read || !self->parser.language) return; @@ -111,17 +129,17 @@ void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges if (reusable_tree && !reusable_tree->has_changes) return; - Tree *tree = parser_parse(&self->parser, self->input, reusable_tree); + Tree *tree = parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error); if (self->tree) { Tree *old_tree = self->tree; self->tree = tree; - if (ranges && range_count) { + if (options.changed_ranges && options.changed_range_count) { tree_path_init(&self->parser.tree_path1, old_tree); tree_path_init(&self->parser.tree_path2, tree); tree_path_get_changes(&self->parser.tree_path1, &self->parser.tree_path2, - ranges, range_count); + options.changed_ranges, options.changed_range_count); } ts_tree_release(old_tree); @@ -132,10 +150,6 @@ void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges self->valid = true; } -void ts_document_parse(TSDocument *self) { - ts_document_parse_and_get_changed_ranges(self, NULL, NULL); -} - void ts_document_invalidate(TSDocument *self) { self->valid = false; } diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c index 5646e101..7e0ef51f 100644 --- a/src/runtime/lexer.c +++ b/src/runtime/lexer.c @@ -145,3 +145,7 @@ void ts_lexer_start(Lexer *self) { if (!self->lookahead_size) ts_lexer__get_lookahead(self); } + +void ts_lexer_advance_to_end(Lexer *self) { + while (self->data.lookahead != 0) ts_lexer__advance(self, false); +} diff --git a/src/runtime/lexer.h b/src/runtime/lexer.h index 0cf6c252..f6f127f5 100644 --- a/src/runtime/lexer.h +++ b/src/runtime/lexer.h @@ -32,6 +32,7 @@ void ts_lexer_init(Lexer *); void ts_lexer_set_input(Lexer *, TSInput); void ts_lexer_reset(Lexer *, Length); void ts_lexer_start(Lexer *); +void ts_lexer_advance_to_end(Lexer *); #ifdef __cplusplus } diff --git a/src/runtime/parser.c b/src/runtime/parser.c index af65c7ea..e6c2c1dd 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -151,21 +151,28 @@ static bool parser__can_reuse(Parser *self, TSStateId state, Tree *tree, return tree->child_count > 1 && tree->error_cost == 0; } -static bool parser__condense_stack(Parser *self) { - bool result = false; +typedef int CondenseResult; +static int CondenseResultMadeChange = 1; +static int CondenseResultAllVersionsHadError = 2; + +static CondenseResult parser__condense_stack(Parser *self) { + CondenseResult result = 0; + bool has_version_without_errors = false; + for (StackVersion i = 0; i < ts_stack_version_count(self->stack); i++) { if (ts_stack_is_halted(self->stack, i)) { ts_stack_remove_version(self->stack, i); - result = true; + result |= CondenseResultMadeChange; i--; continue; } ErrorStatus error_status = ts_stack_error_status(self->stack, i); + if (error_status.count == 0) has_version_without_errors = true; for (StackVersion j = 0; j < i; j++) { if (ts_stack_merge(self->stack, j, i)) { - result = true; + result |= CondenseResultMadeChange; i--; break; } @@ -174,18 +181,20 @@ static bool parser__condense_stack(Parser *self) { ts_stack_error_status(self->stack, j))) { case -1: ts_stack_remove_version(self->stack, j); - result = true; + result |= CondenseResultMadeChange; i--; j--; break; case 1: ts_stack_remove_version(self->stack, i); - result = true; + result |= CondenseResultMadeChange; i--; break; } } } + + if (!has_version_without_errors) result |= CondenseResultAllVersionsHadError; return result; } @@ -1183,7 +1192,7 @@ void parser_destroy(Parser *self) { parser_set_language(self, NULL); } -Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree) { +Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_error) { parser__start(self, input, old_tree); StackVersion version = STACK_VERSION_NONE; @@ -1213,7 +1222,32 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree) { self->reusable_node = reusable_node; - if (parser__condense_stack(self)) { + CondenseResult condense_result = parser__condense_stack(self); + if (halt_on_error && (condense_result & CondenseResultAllVersionsHadError)) { + LOG("halting_parse"); + + ts_lexer_advance_to_end(&self->lexer); + Length remaining_length = length_sub( + self->lexer.current_position, + ts_stack_top_position(self->stack, 0) + ); + + Tree *filler_node = ts_tree_make_error(remaining_length, length_zero(), 0); + filler_node->visible = false; + parser__push(self, 0, filler_node, 0); + + TreeArray children = array_new(); + Tree *root_error = ts_tree_make_error_node(&children); + parser__push(self, 0, root_error, 0); + + TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, ts_builtin_sym_end); + Tree *eof = ts_tree_make_leaf(ts_builtin_sym_end, length_zero(), length_zero(), metadata); + parser__accept(self, 0, eof); + ts_tree_release(eof); + break; + } + + if (condense_result & CondenseResultMadeChange) { LOG("condense"); LOG_STACK(); } diff --git a/src/runtime/parser.h b/src/runtime/parser.h index a7b8dde3..15c010cf 100644 --- a/src/runtime/parser.h +++ b/src/runtime/parser.h @@ -31,7 +31,7 @@ typedef struct { bool parser_init(Parser *); void parser_destroy(Parser *); -Tree *parser_parse(Parser *, TSInput, Tree *); +Tree *parser_parse(Parser *, TSInput, Tree *, bool halt_on_error); void parser_set_language(Parser *, const TSLanguage *); #ifdef __cplusplus diff --git a/test/runtime/document_test.cc b/test/runtime/document_test.cc index 71d7b8c7..960fe047 100644 --- a/test/runtime/document_test.cc +++ b/test/runtime/document_test.cc @@ -367,6 +367,36 @@ describe("Document", [&]() { }))); }); }); + + describe("parse_with_options(options)", [&]() { + it("halts as soon as an error is found if the halt_on_error flag is set", [&]() { + string input_string = "[1, null, error, 3]"; + ts_document_set_language(document, load_real_language("json")); + ts_document_set_input_string(document, input_string.c_str()); + + TSParseOptions options; + options.changed_ranges = nullptr; + + options.halt_on_error = false; + ts_document_parse_with_options(document, options); + root = ts_document_root_node(document); + assert_node_string_equals( + root, + "(array (number) (null) (ERROR (UNEXPECTED 'e')) (number))"); + + ts_document_invalidate(document); + + options.halt_on_error = true; + ts_document_parse_with_options(document, options); + root = ts_document_root_node(document); + assert_node_string_equals( + root, + "(ERROR (number) (null) (UNEXPECTED 'e'))"); + + AssertThat(ts_node_end_char(root), Equals(input_string.size())); + AssertThat(ts_node_end_byte(root), Equals(input_string.size())); + }); + }); }); END_TEST