From 6938b288a540233877cb747d381a85b9289df7b2 Mon Sep 17 00:00:00 2001 From: joshvera Date: Fri, 24 Mar 2017 14:51:37 -0400 Subject: [PATCH 01/19] Make external scanner symbol map unique --- src/compiler/generate_code/c_code.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index 76d82ce2..07e27baf 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -295,7 +295,10 @@ class CCodeGenerator { } void add_external_scanner_symbol_map() { - line("TSSymbol ts_external_scanner_symbol_map[EXTERNAL_TOKEN_COUNT] = {"); + string language_function_name = "tree_sitter_" + name; + string external_scanner_name = language_function_name + "_external_scanner"; + + line("TSSymbol " + external_scanner_name + "_symbol_map[EXTERNAL_TOKEN_COUNT] = {"); indent([&]() { for (size_t i = 0; i < syntax_grammar.external_tokens.size(); i++) { line("[" + external_token_id(i) + "] = " + symbol_id(Symbol::external(i)) + ","); @@ -381,7 +384,7 @@ class CCodeGenerator { } else { indent([&]() { line("(const bool *)ts_external_scanner_states,"); - line("ts_external_scanner_symbol_map,"); + line(external_scanner_name + "_symbol_map,"); line(external_scanner_name + "_create,"); line(external_scanner_name + "_destroy,"); line(external_scanner_name + "_reset,"); From f76935cc7e8a2f1b5a4f568df056594be94273ac Mon Sep 17 00:00:00 2001 From: joshvera Date: Fri, 24 Mar 2017 18:38:21 -0400 Subject: [PATCH 02/19] just make it static --- src/compiler/generate_code/c_code.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index 07e27baf..2d6c22c9 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -295,10 +295,7 @@ class CCodeGenerator { } void add_external_scanner_symbol_map() { - string language_function_name = "tree_sitter_" + name; - string external_scanner_name = language_function_name + "_external_scanner"; - - line("TSSymbol " + external_scanner_name + "_symbol_map[EXTERNAL_TOKEN_COUNT] = {"); + line("static TSSymbol ts_external_scanner_symbol_map[EXTERNAL_TOKEN_COUNT] = {"); indent([&]() { for (size_t i = 0; i < syntax_grammar.external_tokens.size(); i++) { line("[" + external_token_id(i) + "] = " + symbol_id(Symbol::external(i)) + ","); @@ -384,7 +381,7 @@ class CCodeGenerator { } else { indent([&]() { line("(const bool *)ts_external_scanner_states,"); - line(external_scanner_name + "_symbol_map,"); + line("ts_external_scanner_symbol_map,"); line(external_scanner_name + "_create,"); line(external_scanner_name + "_destroy,"); line(external_scanner_name + "_reset,"); From 4b1f69142d2c75354eb7012908418ec5e3260586 Mon Sep 17 00:00:00 2001 From: Rob Rix Date: Wed, 12 Apr 2017 09:46:01 -0400 Subject: [PATCH 03/19] Define a symbol type enum. --- include/tree_sitter/runtime.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index cc551ebb..37f99e54 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -20,6 +20,12 @@ typedef enum { TSInputEncodingUTF16, } TSInputEncoding; +typedef enum { + TSSymbolTypeRegular, + TSSymbolTypeAnonymous, + TSSymbolTypeAuxiliary, +} TSSymbolType; + typedef struct { void *payload; const char *(*read)(void *payload, uint32_t *bytes_read); From 3a888b16233e00be445fbf5e994a92a103e324df Mon Sep 17 00:00:00 2001 From: Rob Rix Date: Wed, 12 Apr 2017 09:47:51 -0400 Subject: [PATCH 04/19] Define a function providing the type of a given symbol. --- include/tree_sitter/runtime.h | 1 + src/runtime/language.c | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 37f99e54..6f37c9a3 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -123,6 +123,7 @@ uint32_t ts_document_parse_count(const TSDocument *); uint32_t ts_language_symbol_count(const TSLanguage *); const char *ts_language_symbol_name(const TSLanguage *, TSSymbol); +TSSymbolType ts_language_symbol_type(const TSLanguage *, TSSymbol); uint32_t ts_language_version(const TSLanguage *); #ifdef __cplusplus diff --git a/src/runtime/language.c b/src/runtime/language.c index 05361a81..8ca4de50 100644 --- a/src/runtime/language.c +++ b/src/runtime/language.c @@ -54,3 +54,14 @@ const char *ts_language_symbol_name(const TSLanguage *language, TSSymbol symbol) else return language->symbol_names[symbol]; } + +TSSymbolType ts_language_symbol_type(const TSLanguage *language, TSSymbol symbol) { + TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); + if (metadata.named) { + return TSSymbolTypeRegular; + } else if (metadata.visible) { + return TSSymbolTypeAnonymous; + } else { + return TSSymbolTypeAuxiliary; + } +} From 91558f0a0e282aac314d75e2dd5d28176499fe07 Mon Sep 17 00:00:00 2001 From: Timothy Clem Date: Mon, 20 Mar 2017 16:54:19 -0700 Subject: [PATCH 05/19] utf8proc_iterate can set codepoint_ref to -1 and returns negative error --- src/runtime/lexer.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c index 123a29fd..5646e101 100644 --- a/src/runtime/lexer.c +++ b/src/runtime/lexer.c @@ -36,11 +36,17 @@ static void ts_lexer__get_lookahead(Lexer *self) { const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk; uint32_t size = self->chunk_size - position_in_chunk + 1; - if (self->input.encoding == TSInputEncodingUTF8) - self->lookahead_size = - utf8proc_iterate(chunk, size, &self->data.lookahead); - else + if (self->input.encoding == TSInputEncodingUTF8) { + int64_t lookahead_size = utf8proc_iterate(chunk, size, &self->data.lookahead); + if (lookahead_size < 0) { + self->lookahead_size = 1; + } else { + self->lookahead_size = lookahead_size; + } + } + else { self->lookahead_size = utf16_iterate(chunk, size, &self->data.lookahead); + } } static void ts_lexer__advance(void *payload, bool skip) { From 37f2a4745ff17c09c8b1dc316e8c0599c8190219 Mon Sep 17 00:00:00 2001 From: Timothy Clem Date: Tue, 21 Mar 2017 09:58:35 -0700 Subject: [PATCH 06/19] Test demonstrating non-UT8 input failure --- test/runtime/parser_test.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 7dfcf26b..7e409d45 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -473,6 +473,16 @@ describe("Parser", [&]() { AssertThat(ts_node_end_char(root), Equals(strlen("'OOO - DD';"))); AssertThat(ts_node_end_byte(root), Equals(strlen("'\u03A9\u03A9\u03A9 \u2014 \u0394\u0394';"))); }); + + it("handles non-UTF8 characters", [&]() { + // ts_document_set_logger(document, stderr_logger_new(true)); + ts_document_print_debugging_graphs(document, true); + ts_document_set_language(document, load_real_language("javascript")); + ts_document_set_input_string(document, "cons\xeb\x00e=ls\x83l6hi');\x0a"); + ts_document_parse(document); + + AssertThat(ts_node_end_byte(root), Equals(strlen("cons\xeb\x00e=ls\x83l6hi');\x0a"))); + }); }); }); From 03a555a86ed79f102ba3a71005b5ad6e6b8c8c8d Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 21 Mar 2017 11:05:32 -0700 Subject: [PATCH 07/19] Finish test for invalid UTF8 handling Signed-off-by: Tim Clem --- test/runtime/parser_test.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/runtime/parser_test.cc b/test/runtime/parser_test.cc index 7e409d45..e390b164 100644 --- a/test/runtime/parser_test.cc +++ b/test/runtime/parser_test.cc @@ -475,13 +475,14 @@ describe("Parser", [&]() { }); it("handles non-UTF8 characters", [&]() { - // ts_document_set_logger(document, stderr_logger_new(true)); - ts_document_print_debugging_graphs(document, true); + const char *string = "cons\xeb\x00e=ls\x83l6hi');\x0a"; + ts_document_set_language(document, load_real_language("javascript")); - ts_document_set_input_string(document, "cons\xeb\x00e=ls\x83l6hi');\x0a"); + ts_document_set_input_string(document, string); ts_document_parse(document); - AssertThat(ts_node_end_byte(root), Equals(strlen("cons\xeb\x00e=ls\x83l6hi');\x0a"))); + TSNode root = ts_document_root_node(document); + AssertThat(ts_node_end_byte(root), Equals(strlen(string))); }); }); }); From 704c2d5907fd56c0b85eb5602d722556a278dbc1 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 21 Mar 2017 11:05:48 -0700 Subject: [PATCH 08/19] Fix lookahead_char type in ts_tree_make_error function --- src/runtime/tree.c | 2 +- src/runtime/tree.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 49f81e9c..195b6260 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -99,7 +99,7 @@ TreeArray ts_tree_array_remove_trailing_extras(TreeArray *self) { return result; } -Tree *ts_tree_make_error(Length size, Length padding, char lookahead_char) { +Tree *ts_tree_make_error(Length size, Length padding, int32_t lookahead_char) { Tree *result = ts_tree_make_leaf(ts_builtin_sym_error, padding, size, (TSSymbolMetadata){ .visible = true, .named = true, diff --git a/src/runtime/tree.h b/src/runtime/tree.h index c08ba24b..f205af97 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -76,7 +76,7 @@ Tree *ts_tree_make_leaf(TSSymbol, Length, Length, TSSymbolMetadata); Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, TSSymbolMetadata); Tree *ts_tree_make_copy(Tree *child); Tree *ts_tree_make_error_node(TreeArray *); -Tree *ts_tree_make_error(Length, Length, char); +Tree *ts_tree_make_error(Length, Length, int32_t); void ts_tree_retain(Tree *tree); void ts_tree_release(Tree *tree); bool ts_tree_eq(const Tree *tree1, const Tree *tree2); From 82cb1c9806cee57cebe361abed5bc870e3d07b65 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 21 Mar 2017 11:12:08 -0700 Subject: [PATCH 09/19] Handle invalid UTF8 in encoding test helpers Signed-off-by: Tim Clem --- test/helpers/encoding_helpers.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/helpers/encoding_helpers.cc b/test/helpers/encoding_helpers.cc index 8ef9fec1..1169bb2d 100644 --- a/test/helpers/encoding_helpers.cc +++ b/test/helpers/encoding_helpers.cc @@ -4,10 +4,16 @@ #include "utf8proc.h" static inline int string_iterate(TSInputEncoding encoding, const uint8_t *string, size_t length, int32_t *code_point) { - if (encoding == TSInputEncodingUTF8) - return utf8proc_iterate(string, length, code_point); - else + if (encoding == TSInputEncodingUTF8) { + int32_t character_size = utf8proc_iterate(string, length, code_point); + if (character_size < 0) { + return 1; + } else { + return character_size; + } + } else { return utf16_iterate(string, length, code_point); + } } size_t string_char_count(TSInputEncoding encoding, const std::string &input) { From a98d449d886193c8d12a1b254867ab28925db175 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 1 May 2017 13:04:06 -0700 Subject: [PATCH 10/19] Add an option to immediately halt on syntax error --- include/tree_sitter/runtime.h | 9 +++++++ src/runtime/document.c | 32 +++++++++++++++------- src/runtime/lexer.c | 4 +++ src/runtime/lexer.h | 1 + src/runtime/parser.c | 50 +++++++++++++++++++++++++++++------ src/runtime/parser.h | 2 +- test/runtime/document_test.cc | 30 +++++++++++++++++++++ 7 files changed, 110 insertions(+), 18 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 6f37c9a3..95da0787 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -117,6 +117,15 @@ void ts_document_print_debugging_graphs(TSDocument *, bool); void ts_document_edit(TSDocument *, TSInputEdit); void ts_document_parse(TSDocument *); void ts_document_parse_and_get_changed_ranges(TSDocument *, TSRange **, uint32_t *); + +typedef struct { + TSRange **changed_ranges; + uint32_t *changed_range_count; + bool halt_on_error; +} TSParseOptions; + +void ts_document_parse_with_options(TSDocument *, TSParseOptions); + void ts_document_invalidate(TSDocument *); TSNode ts_document_root_node(const TSDocument *); uint32_t ts_document_parse_count(const TSDocument *); diff --git a/src/runtime/document.c b/src/runtime/document.c index ff09610d..6bcc5fbc 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -99,10 +99,28 @@ void ts_document_edit(TSDocument *self, TSInputEdit edit) { ts_tree_edit(self->tree, &edit); } +void ts_document_parse(TSDocument *self) { + return ts_document_parse_with_options(self, (TSParseOptions){ + .halt_on_error = false, + .changed_ranges = NULL, + .changed_range_count = NULL, + }); +} + void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges, uint32_t *range_count) { - if (ranges) *ranges = NULL; - if (range_count) *range_count = 0; + return ts_document_parse_with_options(self, (TSParseOptions){ + .halt_on_error = false, + .changed_ranges = ranges, + .changed_range_count = range_count, + }); +} + +void ts_document_parse_with_options(TSDocument *self, TSParseOptions options) { + if (options.changed_ranges && options.changed_range_count) { + *options.changed_ranges = NULL; + *options.changed_range_count = 0; + } if (!self->input.read || !self->parser.language) return; @@ -111,17 +129,17 @@ void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges if (reusable_tree && !reusable_tree->has_changes) return; - Tree *tree = parser_parse(&self->parser, self->input, reusable_tree); + Tree *tree = parser_parse(&self->parser, self->input, reusable_tree, options.halt_on_error); if (self->tree) { Tree *old_tree = self->tree; self->tree = tree; - if (ranges && range_count) { + if (options.changed_ranges && options.changed_range_count) { tree_path_init(&self->parser.tree_path1, old_tree); tree_path_init(&self->parser.tree_path2, tree); tree_path_get_changes(&self->parser.tree_path1, &self->parser.tree_path2, - ranges, range_count); + options.changed_ranges, options.changed_range_count); } ts_tree_release(old_tree); @@ -132,10 +150,6 @@ void ts_document_parse_and_get_changed_ranges(TSDocument *self, TSRange **ranges self->valid = true; } -void ts_document_parse(TSDocument *self) { - ts_document_parse_and_get_changed_ranges(self, NULL, NULL); -} - void ts_document_invalidate(TSDocument *self) { self->valid = false; } diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c index 5646e101..7e0ef51f 100644 --- a/src/runtime/lexer.c +++ b/src/runtime/lexer.c @@ -145,3 +145,7 @@ void ts_lexer_start(Lexer *self) { if (!self->lookahead_size) ts_lexer__get_lookahead(self); } + +void ts_lexer_advance_to_end(Lexer *self) { + while (self->data.lookahead != 0) ts_lexer__advance(self, false); +} diff --git a/src/runtime/lexer.h b/src/runtime/lexer.h index 0cf6c252..f6f127f5 100644 --- a/src/runtime/lexer.h +++ b/src/runtime/lexer.h @@ -32,6 +32,7 @@ void ts_lexer_init(Lexer *); void ts_lexer_set_input(Lexer *, TSInput); void ts_lexer_reset(Lexer *, Length); void ts_lexer_start(Lexer *); +void ts_lexer_advance_to_end(Lexer *); #ifdef __cplusplus } diff --git a/src/runtime/parser.c b/src/runtime/parser.c index af65c7ea..e6c2c1dd 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -151,21 +151,28 @@ static bool parser__can_reuse(Parser *self, TSStateId state, Tree *tree, return tree->child_count > 1 && tree->error_cost == 0; } -static bool parser__condense_stack(Parser *self) { - bool result = false; +typedef int CondenseResult; +static int CondenseResultMadeChange = 1; +static int CondenseResultAllVersionsHadError = 2; + +static CondenseResult parser__condense_stack(Parser *self) { + CondenseResult result = 0; + bool has_version_without_errors = false; + for (StackVersion i = 0; i < ts_stack_version_count(self->stack); i++) { if (ts_stack_is_halted(self->stack, i)) { ts_stack_remove_version(self->stack, i); - result = true; + result |= CondenseResultMadeChange; i--; continue; } ErrorStatus error_status = ts_stack_error_status(self->stack, i); + if (error_status.count == 0) has_version_without_errors = true; for (StackVersion j = 0; j < i; j++) { if (ts_stack_merge(self->stack, j, i)) { - result = true; + result |= CondenseResultMadeChange; i--; break; } @@ -174,18 +181,20 @@ static bool parser__condense_stack(Parser *self) { ts_stack_error_status(self->stack, j))) { case -1: ts_stack_remove_version(self->stack, j); - result = true; + result |= CondenseResultMadeChange; i--; j--; break; case 1: ts_stack_remove_version(self->stack, i); - result = true; + result |= CondenseResultMadeChange; i--; break; } } } + + if (!has_version_without_errors) result |= CondenseResultAllVersionsHadError; return result; } @@ -1183,7 +1192,7 @@ void parser_destroy(Parser *self) { parser_set_language(self, NULL); } -Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree) { +Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_error) { parser__start(self, input, old_tree); StackVersion version = STACK_VERSION_NONE; @@ -1213,7 +1222,32 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree) { self->reusable_node = reusable_node; - if (parser__condense_stack(self)) { + CondenseResult condense_result = parser__condense_stack(self); + if (halt_on_error && (condense_result & CondenseResultAllVersionsHadError)) { + LOG("halting_parse"); + + ts_lexer_advance_to_end(&self->lexer); + Length remaining_length = length_sub( + self->lexer.current_position, + ts_stack_top_position(self->stack, 0) + ); + + Tree *filler_node = ts_tree_make_error(remaining_length, length_zero(), 0); + filler_node->visible = false; + parser__push(self, 0, filler_node, 0); + + TreeArray children = array_new(); + Tree *root_error = ts_tree_make_error_node(&children); + parser__push(self, 0, root_error, 0); + + TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, ts_builtin_sym_end); + Tree *eof = ts_tree_make_leaf(ts_builtin_sym_end, length_zero(), length_zero(), metadata); + parser__accept(self, 0, eof); + ts_tree_release(eof); + break; + } + + if (condense_result & CondenseResultMadeChange) { LOG("condense"); LOG_STACK(); } diff --git a/src/runtime/parser.h b/src/runtime/parser.h index a7b8dde3..15c010cf 100644 --- a/src/runtime/parser.h +++ b/src/runtime/parser.h @@ -31,7 +31,7 @@ typedef struct { bool parser_init(Parser *); void parser_destroy(Parser *); -Tree *parser_parse(Parser *, TSInput, Tree *); +Tree *parser_parse(Parser *, TSInput, Tree *, bool halt_on_error); void parser_set_language(Parser *, const TSLanguage *); #ifdef __cplusplus diff --git a/test/runtime/document_test.cc b/test/runtime/document_test.cc index 71d7b8c7..960fe047 100644 --- a/test/runtime/document_test.cc +++ b/test/runtime/document_test.cc @@ -367,6 +367,36 @@ describe("Document", [&]() { }))); }); }); + + describe("parse_with_options(options)", [&]() { + it("halts as soon as an error is found if the halt_on_error flag is set", [&]() { + string input_string = "[1, null, error, 3]"; + ts_document_set_language(document, load_real_language("json")); + ts_document_set_input_string(document, input_string.c_str()); + + TSParseOptions options; + options.changed_ranges = nullptr; + + options.halt_on_error = false; + ts_document_parse_with_options(document, options); + root = ts_document_root_node(document); + assert_node_string_equals( + root, + "(array (number) (null) (ERROR (UNEXPECTED 'e')) (number))"); + + ts_document_invalidate(document); + + options.halt_on_error = true; + ts_document_parse_with_options(document, options); + root = ts_document_root_node(document); + assert_node_string_equals( + root, + "(ERROR (number) (null) (UNEXPECTED 'e'))"); + + AssertThat(ts_node_end_char(root), Equals(input_string.size())); + AssertThat(ts_node_end_byte(root), Equals(input_string.size())); + }); + }); }); END_TEST From 74f5ceddf78258e4d2fff050e92493cd5e3596be Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 1 May 2017 14:25:25 -0700 Subject: [PATCH 11/19] Fix parsing of valid code with halt_on_error flag set Signed-off-by: Tim Clem --- src/runtime/parser.c | 6 +++++- test/runtime/document_test.cc | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index e6c2c1dd..f4f01a03 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -194,7 +194,10 @@ static CondenseResult parser__condense_stack(Parser *self) { } } - if (!has_version_without_errors) result |= CondenseResultAllVersionsHadError; + if (!has_version_without_errors && ts_stack_version_count(self->stack) > 0) { + result |= CondenseResultAllVersionsHadError; + } + return result; } @@ -1225,6 +1228,7 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err CondenseResult condense_result = parser__condense_stack(self); if (halt_on_error && (condense_result & CondenseResultAllVersionsHadError)) { LOG("halting_parse"); + LOG_STACK(); ts_lexer_advance_to_end(&self->lexer); Length remaining_length = length_sub( diff --git a/test/runtime/document_test.cc b/test/runtime/document_test.cc index 960fe047..7757823c 100644 --- a/test/runtime/document_test.cc +++ b/test/runtime/document_test.cc @@ -396,6 +396,21 @@ describe("Document", [&]() { AssertThat(ts_node_end_char(root), Equals(input_string.size())); AssertThat(ts_node_end_byte(root), Equals(input_string.size())); }); + + it("can parse valid code with the halt_on_error flag set", [&]() { + string input_string = "[1, null, 3]"; + ts_document_set_language(document, load_real_language("json")); + ts_document_set_input_string(document, input_string.c_str()); + + TSParseOptions options; + options.changed_ranges = nullptr; + options.halt_on_error = true; + ts_document_parse_with_options(document, options); + root = ts_document_root_node(document); + assert_node_string_equals( + root, + "(array (number) (null) (number))"); + }); }); }); From e8a9bb7a51d4248606e7435fb82da6f4242c5f05 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 1 May 2017 14:41:55 -0700 Subject: [PATCH 12/19] :art: Extract parser__halt_parse function --- src/runtime/parser.c | 46 ++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index f4f01a03..ef9cd31c 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -991,6 +991,30 @@ static void parser__handle_error(Parser *self, StackVersion version, } } +static void parser__halt_parse(Parser *self) { + LOG("halting_parse"); + LOG_STACK(); + + ts_lexer_advance_to_end(&self->lexer); + Length remaining_length = length_sub( + self->lexer.current_position, + ts_stack_top_position(self->stack, 0) + ); + + Tree *filler_node = ts_tree_make_error(remaining_length, length_zero(), 0); + filler_node->visible = false; + parser__push(self, 0, filler_node, 0); + + TreeArray children = array_new(); + Tree *root_error = ts_tree_make_error_node(&children); + parser__push(self, 0, root_error, 0); + + TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, ts_builtin_sym_end); + Tree *eof = ts_tree_make_leaf(ts_builtin_sym_end, length_zero(), length_zero(), metadata); + parser__accept(self, 0, eof); + ts_tree_release(eof); +} + static void parser__recover(Parser *self, StackVersion version, TSStateId state, Tree *lookahead) { if (lookahead->symbol == ts_builtin_sym_end) { @@ -1227,27 +1251,7 @@ Tree *parser_parse(Parser *self, TSInput input, Tree *old_tree, bool halt_on_err CondenseResult condense_result = parser__condense_stack(self); if (halt_on_error && (condense_result & CondenseResultAllVersionsHadError)) { - LOG("halting_parse"); - LOG_STACK(); - - ts_lexer_advance_to_end(&self->lexer); - Length remaining_length = length_sub( - self->lexer.current_position, - ts_stack_top_position(self->stack, 0) - ); - - Tree *filler_node = ts_tree_make_error(remaining_length, length_zero(), 0); - filler_node->visible = false; - parser__push(self, 0, filler_node, 0); - - TreeArray children = array_new(); - Tree *root_error = ts_tree_make_error_node(&children); - parser__push(self, 0, root_error, 0); - - TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, ts_builtin_sym_end); - Tree *eof = ts_tree_make_leaf(ts_builtin_sym_end, length_zero(), length_zero(), metadata); - parser__accept(self, 0, eof); - ts_tree_release(eof); + parser__halt_parse(self); break; } From 7b401de5a6b1d3ccc41a0c82f24aa5b086be73e9 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 3 May 2017 09:57:09 -0700 Subject: [PATCH 13/19] Don't use pointer equality to compare external token states --- src/runtime/stack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/stack.c b/src/runtime/stack.c index ff6f7e1f..ecbaeba5 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -504,7 +504,7 @@ bool ts_stack_merge(Stack *self, StackVersion version, StackVersion new_version) new_node->position.chars == node->position.chars && new_node->error_count == node->error_count && new_node->error_cost == node->error_cost && - new_head->external_token_state == head->external_token_state) { + ts_external_token_state_eq(new_head->external_token_state, head->external_token_state)) { for (uint32_t j = 0; j < new_node->link_count; j++) stack_node_add_link(node, new_node->links[j]); if (new_head->push_count > head->push_count) From f829e81f306fbf7a90405f71b5008fa3aff14ee0 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 3 May 2017 10:32:12 -0700 Subject: [PATCH 14/19] Add MIT license --- LICENSE | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..b6d8763b --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright 2014 Max Brunsfeld + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From dee86f908a226eef59fddaaf2c2edd0da13c421f Mon Sep 17 00:00:00 2001 From: Phil Turnbull Date: Wed, 7 Jun 2017 17:05:39 -0400 Subject: [PATCH 15/19] Correctly check type is ParseActionTypeRecover --- src/compiler/parse_table.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/parse_table.cc b/src/compiler/parse_table.cc index 37707ed0..cf91154f 100644 --- a/src/compiler/parse_table.cc +++ b/src/compiler/parse_table.cc @@ -151,7 +151,7 @@ bool ParseState::has_shift_action() const { void ParseState::each_referenced_state(function fn) { for (auto &entry : terminal_entries) for (ParseAction &action : entry.second.actions) - if (action.type == ParseActionTypeShift || ParseActionTypeRecover) + if (action.type == ParseActionTypeShift || action.type == ParseActionTypeRecover) fn(&action.state_index); for (auto &entry : nonterminal_entries) fn(&entry.second); From 6897530c4720000b19dd4566bfc3f840b4a56f95 Mon Sep 17 00:00:00 2001 From: Phil Turnbull Date: Wed, 7 Jun 2017 17:07:00 -0400 Subject: [PATCH 16/19] Check for invalid state indexes Some ParseActions have a state-id of -1 which can cause an out-of-bounds read when removing duplicate parse states. This was found by AddressSanitizer: ==90699==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x6320000187f8 at pc 0x0001071220a9 bp 0x7fff595fd440 sp 0x7fff595fd438 READ of size 8 at 0x6320000187f8 thread T0 #0 0x1071220a8 in tree_sitter::build_tables::ParseTableBuilder::remove_duplicate_parse_states()::'lambda0'(unsigned long*)::operator()(unsigned long*) const build_parse_table.cc:398 #1 0x107121fa5 in void std::__1::__invoke_void_return_wrapper::__call(tree_sitter::build_tables::ParseTableBuilder::remove_duplicate_parse_states()::'lambda0'(unsigned long*)&&&, unsigned long*&&) __functional_base:416 ... 0x6320000187f8 is located 8 bytes to the left of 88264-byte region [0x632000018800,0x63200002e0c8) allocated by thread T0 here: #0 0x107b1576b in wrap__Znwm (libclang_rt.asan_osx_dynamic.dylib:x86_64h+0x6076b) #1 0x10711da2c in std::__1::vector >::allocate(unsigned long) new:169 #2 0x10711d8fb in std::__1::vector >::vector(unsigned long) vector:1074 #3 0x107112f5c in std::__1::vector >::vector(unsigned long) vector:1068 #4 0x1070af381 in tree_sitter::build_tables::ParseTableBuilder::remove_duplicate_parse_states() build_parse_table.cc:378 #5 0x10709d827 in tree_sitter::build_tables::ParseTableBuilder::build() build_parse_table.cc:85 ... SUMMARY: AddressSanitizer: heap-buffer-overflow build_parse_table.cc:398 in tree_sitter::build_tables::ParseTableBuilder::remove_duplicate_parse_states()::'lambda0'(unsigned long*)::operator()(unsigned long*) const Shadow bytes around the buggy address: 0x1c64000030a0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa 0x1c64000030b0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa 0x1c64000030c0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa 0x1c64000030d0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa 0x1c64000030e0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa =>0x1c64000030f0: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa[fa] 0x1c6400003100: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0x1c6400003110: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0x1c6400003120: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0x1c6400003130: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0x1c6400003140: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 --- src/compiler/build_tables/build_parse_table.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/compiler/build_tables/build_parse_table.cc b/src/compiler/build_tables/build_parse_table.cc index 50c84af7..d1d90455 100644 --- a/src/compiler/build_tables/build_parse_table.cc +++ b/src/compiler/build_tables/build_parse_table.cc @@ -394,7 +394,8 @@ class ParseTableBuilder { } else { ParseState &state = *iter; state.each_referenced_state([&new_state_ids](ParseStateId *state_index) { - *state_index = new_state_ids[*state_index]; + if (*state_index != (ParseStateId)(-1)) + *state_index = new_state_ids[*state_index]; }); ++iter; } From 18ba6ebbd73fbcc00dc68e22878f97d7f647236c Mon Sep 17 00:00:00 2001 From: Phil Turnbull Date: Fri, 9 Jun 2017 16:13:17 -0400 Subject: [PATCH 17/19] Move state_id check into each_referenced_state --- src/compiler/build_tables/build_parse_table.cc | 3 +-- src/compiler/parse_table.cc | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/compiler/build_tables/build_parse_table.cc b/src/compiler/build_tables/build_parse_table.cc index d1d90455..50c84af7 100644 --- a/src/compiler/build_tables/build_parse_table.cc +++ b/src/compiler/build_tables/build_parse_table.cc @@ -394,8 +394,7 @@ class ParseTableBuilder { } else { ParseState &state = *iter; state.each_referenced_state([&new_state_ids](ParseStateId *state_index) { - if (*state_index != (ParseStateId)(-1)) - *state_index = new_state_ids[*state_index]; + *state_index = new_state_ids[*state_index]; }); ++iter; } diff --git a/src/compiler/parse_table.cc b/src/compiler/parse_table.cc index cf91154f..5b422d43 100644 --- a/src/compiler/parse_table.cc +++ b/src/compiler/parse_table.cc @@ -154,7 +154,8 @@ void ParseState::each_referenced_state(function fn) { if (action.type == ParseActionTypeShift || action.type == ParseActionTypeRecover) fn(&action.state_index); for (auto &entry : nonterminal_entries) - fn(&entry.second); + if (entry.second != (ParseStateId)(-1)) + fn(&entry.second); } bool ParseState::operator==(const ParseState &other) const { From 577e43f65373de5eac6855c79e136f58950cfd04 Mon Sep 17 00:00:00 2001 From: Phil Turnbull Date: Fri, 9 Jun 2017 16:25:40 -0400 Subject: [PATCH 18/19] shift-extra actions do not have valid state_ids --- src/compiler/parse_table.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/parse_table.cc b/src/compiler/parse_table.cc index 5b422d43..8ea1f8c4 100644 --- a/src/compiler/parse_table.cc +++ b/src/compiler/parse_table.cc @@ -151,7 +151,7 @@ bool ParseState::has_shift_action() const { void ParseState::each_referenced_state(function fn) { for (auto &entry : terminal_entries) for (ParseAction &action : entry.second.actions) - if (action.type == ParseActionTypeShift || action.type == ParseActionTypeRecover) + if ((action.type == ParseActionTypeShift && !action.extra) || action.type == ParseActionTypeRecover) fn(&action.state_index); for (auto &entry : nonterminal_entries) if (entry.second != (ParseStateId)(-1)) From c58f6401d0c3d02b6d0da81908dc07c7c732f88c Mon Sep 17 00:00:00 2001 From: Phil Turnbull Date: Wed, 14 Jun 2017 08:49:38 -0400 Subject: [PATCH 19/19] Non-terminal entries always have valid state-ids --- src/compiler/parse_table.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/compiler/parse_table.cc b/src/compiler/parse_table.cc index 8ea1f8c4..ffa57760 100644 --- a/src/compiler/parse_table.cc +++ b/src/compiler/parse_table.cc @@ -154,8 +154,7 @@ void ParseState::each_referenced_state(function fn) { if ((action.type == ParseActionTypeShift && !action.extra) || action.type == ParseActionTypeRecover) fn(&action.state_index); for (auto &entry : nonterminal_entries) - if (entry.second != (ParseStateId)(-1)) - fn(&entry.second); + fn(&entry.second); } bool ParseState::operator==(const ParseState &other) const {