From e941f8c1758a1cb2c06e11e0d88e724942d824e9 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 1 Sep 2014 21:32:29 -0700 Subject: [PATCH] Fix error in document editing When breaking down the stack in parser.c, the previous code would not account for ubiquitous tokens. This was a problem for a long time, but wasn't noticed until ubiquitous tokens started being used to represent separator characters --- spec/runtime/document_spec.cc | 7 +++-- src/compiler/build_tables/build_lex_table.cc | 6 ++--- src/compiler/grammar.cc | 3 +-- src/runtime/parser.c | 28 +++++++++----------- src/runtime/stack.h | 2 +- 5 files changed, 21 insertions(+), 25 deletions(-) diff --git a/spec/runtime/document_spec.cc b/spec/runtime/document_spec.cc index 98955d17..c9d94bab 100644 --- a/spec/runtime/document_spec.cc +++ b/spec/runtime/document_spec.cc @@ -63,13 +63,16 @@ describe("Document", [&]() { }); }); - describe_skip("edit", [&]() { + describe("edit", [&]() { SpyReader *reader; before_each([&]() { reader = new SpyReader("{ \"key\": [1, 2] }", 5); ts_document_set_language(doc, ts_language_json()); ts_document_set_input(doc, reader->input); + + AssertThat(ts_node_string(ts_document_root_node(doc)), Equals( + "(DOCUMENT (object (string) (array (number) (number))))")); }); after_each([&]() { @@ -87,7 +90,7 @@ describe("Document", [&]() { it("updates the parse tree", [&]() { AssertThat(string(ts_node_string(ts_document_root_node(doc))), Equals( - "(DOCUMENT (object (string) (array (number) (number)) (string) (number)))")); + "(DOCUMENT (object (string) (array (number) (number)) (string) (value (number))))")); }); it("re-reads only the changed portion of the input", [&]() { diff --git a/src/compiler/build_tables/build_lex_table.cc b/src/compiler/build_tables/build_lex_table.cc index 528e6a6b..ce78e2af 100644 --- a/src/compiler/build_tables/build_lex_table.cc +++ b/src/compiler/build_tables/build_lex_table.cc @@ -38,11 +38,9 @@ class LexTableBuilder { if (symbol == rules::ERROR()) continue; else if (symbol == rules::END_OF_INPUT()) - result.insert(LexItem( - symbol, CharacterSet().include(0).copy())); + result.insert(LexItem(symbol, CharacterSet().include(0).copy())); else if (symbol.is_token()) - result.insert( - LexItem(symbol, lex_grammar.rule(symbol))); + result.insert(LexItem(symbol, lex_grammar.rule(symbol))); } return result; } diff --git a/src/compiler/grammar.cc b/src/compiler/grammar.cc index beed8f22..00bb1f32 100644 --- a/src/compiler/grammar.cc +++ b/src/compiler/grammar.cc @@ -12,8 +12,7 @@ using rules::rule_ptr; Grammar::Grammar( const std::vector > &rules) - : rules_(rules), - ubiquitous_tokens_({}) {} + : rules_(rules), ubiquitous_tokens_({}) {} bool Grammar::operator==(const Grammar &other) const { if (other.rules_.size() != rules_.size()) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 360342ac..0bf770f5 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -38,14 +38,22 @@ static size_t breakdown_stack(TSParser *parser, TSInputEdit *edit) { stack->size--; position -= node->size; + DEBUG_PARSE("BREAKDOWN %s %u", parser->language->symbol_names[node->symbol], + ts_stack_top_state(stack)); + for (size_t i = 0; i < child_count && position < edit->position; i++) { TSTree *child = children[i]; TSStateId state = ts_stack_top_state(stack); - TSStateId next_state = - action_for(parser->language, state, child->symbol).data.to_state; + TSParseAction action = action_for(parser->language, state, child->symbol); + TSStateId next_state = (action.type == TSParseActionTypeShift) + ? action.data.to_state + : state; ts_stack_push(stack, next_state, child); ts_tree_retain(child); position += child->size; + + DEBUG_PARSE("PUT_BACK %s %u", + parser->language->symbol_names[child->symbol], next_state); } ts_tree_release(node); @@ -71,9 +79,6 @@ static void reduce(TSParser *parser, TSSymbol symbol, size_t child_count) { TSStack *stack = &parser->stack; parser->next_lookahead = parser->lookahead; - /* size_t trailing_extra_count; */ - /* TSTree **trailing_extras = ts_stack_pop_extras(stack, &trailing_extra_count); */ - /* * Walk down the stack to determine which symbols will be reduced. * The child node count is known ahead of time, but some children @@ -93,15 +98,6 @@ static void reduce(TSParser *parser, TSSymbol symbol, size_t child_count) { int hidden = parser->language->hidden_symbol_flags[symbol]; parser->lookahead = ts_tree_make_node(symbol, child_count, children, hidden); ts_stack_shrink(stack, stack->size - child_count); - - /* TSStateId state = ts_stack_top_state(stack); */ - /* if (trailing_extras) { */ - /* for (size_t i = 0; i < trailing_extra_count; i++) { */ - /* ts_stack_push(&parser->stack, state, trailing_extras[i]); */ - /* ts_tree_release(trailing_extras[i]); */ - /* } */ - /* free(trailing_extras); */ - /* } */ } static int reduce_extra(TSParser *parser, TSSymbol symbol) { @@ -169,8 +165,8 @@ static int handle_error(TSParser *parser) { DEBUG_PARSE("RECOVER %u", state_after_error); ts_stack_shrink(&parser->stack, i + 1); error->size = ts_lexer_position(&parser->lexer) - - parser->lookahead->size - - ts_stack_right_position(&parser->stack); + parser->lookahead->size - + ts_stack_right_position(&parser->stack); ts_stack_push(&parser->stack, state_after_error, error); ts_tree_release(error); return 1; diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 19f4e003..b7156fca 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -29,7 +29,7 @@ TSTree **ts_stack_pop_extras(TSStack *, size_t *); #define TS_STACK_FROM_TOP(stack, entry, index) \ size_t index = stack.size - 1; \ - (void)index; \ + (void) index; \ for (TSStackEntry *entry = stack.entries + stack.size - 1; \ entry >= stack.entries; entry-- && index--)