diff --git a/spec/compiler/build_tables/perform_spec.cpp b/spec/compiler/build_tables/perform_spec.cpp index 8c8e91a5..a0eb7b93 100644 --- a/spec/compiler/build_tables/perform_spec.cpp +++ b/spec/compiler/build_tables/perform_spec.cpp @@ -95,7 +95,7 @@ describe("building parse and lex tables", []() { it("accepts when the start symbol is reduced", [&]() { AssertThat(parse_state(1).actions, Equals(unordered_map({ - { ParseTable::END_OF_INPUT, parse_actions({ ParseAction::Accept() }) } + { "__END__", parse_actions({ ParseAction::Accept() }) } }))); }); diff --git a/spec/compiler/prepare_grammar/extract_tokens_spec.cpp b/spec/compiler/prepare_grammar/extract_tokens_spec.cpp index 9631e0b1..35deb055 100644 --- a/spec/compiler/prepare_grammar/extract_tokens_spec.cpp +++ b/spec/compiler/prepare_grammar/extract_tokens_spec.cpp @@ -33,7 +33,6 @@ describe("preparing a grammar", []() { { "1", rules::seq({ rules::character('a'), rules::character('b') }) }, - { "__END__", character('\0') }, }))); }); @@ -53,7 +52,6 @@ describe("preparing a grammar", []() { { "rule2", seq({ character('a'), character('b') }) }, - { "__END__", character('\0') }, }))); }); }); diff --git a/src/compiler/build_tables/perform.cpp b/src/compiler/build_tables/perform.cpp index 637e4dcb..7557f111 100644 --- a/src/compiler/build_tables/perform.cpp +++ b/src/compiler/build_tables/perform.cpp @@ -7,12 +7,15 @@ #include "grammar.h" using std::pair; +using std::string; using std::vector; using std::unordered_map; namespace tree_sitter { namespace build_tables { static int NOT_FOUND = -1; + static string START = "__START__"; + static string END_OF_INPUT = "__END__"; class TableBuilder { const Grammar grammar; @@ -62,7 +65,7 @@ namespace tree_sitter { void add_reduce_actions(const ParseItemSet &item_set, size_t state_index) { for (ParseItem item : item_set) { if (item.is_done()) { - ParseAction action = (item.rule_name == ParseTable::START) ? + ParseAction action = (item.rule_name == START) ? ParseAction::Accept() : ParseAction::Reduce(item.rule_name, item.consumed_sym_count); parse_table.add_action(state_index, item.lookahead_sym_name, action); @@ -75,6 +78,8 @@ namespace tree_sitter { LexItemSet item_set; for (auto pair : state.actions) { auto symbol = rules::Symbol(pair.first); + if (symbol.name == END_OF_INPUT) + item_set.insert(LexItem(symbol.name, rules::character('\0'))); if (lex_grammar.has_definition(symbol)) item_set.insert(LexItem(symbol.name, lex_grammar.rule(symbol.name))); } @@ -113,7 +118,7 @@ namespace tree_sitter { lex_grammar(lex_grammar) {}; pair build() { - auto item = ParseItem(ParseTable::START, rules::sym(grammar.start_rule_name), 0, ParseTable::END_OF_INPUT); + auto item = ParseItem(START, rules::sym(grammar.start_rule_name), 0, END_OF_INPUT); ParseItemSet item_set = item_set_closure(ParseItemSet({ item }), grammar); add_parse_state(item_set); return pair(parse_table, lex_table); diff --git a/src/compiler/lex_table.h b/src/compiler/lex_table.h index 2b3f052c..67b4c9ef 100644 --- a/src/compiler/lex_table.h +++ b/src/compiler/lex_table.h @@ -56,8 +56,6 @@ namespace tree_sitter { void add_action(size_t state_index, CharMatch match, LexAction action); void add_default_action(size_t state_index, LexAction action); - static const std::string START; - static const std::string END_OF_INPUT; std::vector states; }; } diff --git a/src/compiler/parse_table.cpp b/src/compiler/parse_table.cpp index 3df9675e..f3fb5177 100644 --- a/src/compiler/parse_table.cpp +++ b/src/compiler/parse_table.cpp @@ -73,7 +73,4 @@ namespace tree_sitter { void ParseTable::add_default_action(size_t state_index, ParseAction action) { states[state_index].default_actions.insert(action); } - - const string ParseTable::START = "__START__"; - const string ParseTable::END_OF_INPUT = "__END__"; } diff --git a/src/compiler/parse_table.h b/src/compiler/parse_table.h index 1b597c5f..2bbdc47a 100644 --- a/src/compiler/parse_table.h +++ b/src/compiler/parse_table.h @@ -61,8 +61,6 @@ namespace tree_sitter { void add_action(size_t state_index, std::string symbol_name, ParseAction action); void add_default_action(size_t state_index, ParseAction action); - static const std::string START; - static const std::string END_OF_INPUT; std::vector states; std::unordered_set symbol_names; }; diff --git a/src/compiler/prepare_grammar/extract_tokens.cpp b/src/compiler/prepare_grammar/extract_tokens.cpp index 30391b57..92547f05 100644 --- a/src/compiler/prepare_grammar/extract_tokens.cpp +++ b/src/compiler/prepare_grammar/extract_tokens.cpp @@ -72,7 +72,6 @@ namespace tree_sitter { for (auto pair : extractor.tokens) tokens.insert(pair); - tokens.insert({ "__END__", character('\0') }); return { Grammar(input_grammar.start_rule_name, rules),