From 44745b51797c7002ac05c89c372fbfd92c72463c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 3 Jan 2014 01:02:24 -0800 Subject: [PATCH] Start work on pre-processing grammars --- .../compiler/grammar/prepare_grammar_spec.cpp | 35 +++++++++++ src/compiler/grammar/extract_tokens.cpp | 58 +++++++++++++++++++ src/compiler/grammar/extract_tokens.h | 10 ++++ src/compiler/grammar/grammar.cpp | 31 +++++++++- src/compiler/grammar/grammar.h | 8 ++- src/compiler/grammar/prepare_grammar.cpp | 10 ++++ src/compiler/grammar/prepare_grammar.h | 10 ++++ src/compiler/grammar/search_for_symbols.cpp | 33 +++++++++++ src/compiler/grammar/search_for_symbols.h | 10 ++++ src/compiler/rules/rules.cpp | 4 ++ src/compiler/rules/rules.h | 2 + src/compiler/rules/token.cpp | 32 ++++++++++ src/compiler/rules/token.h | 23 ++++++++ src/compiler/rules/visitor.cpp | 18 +++--- src/compiler/rules/visitor.h | 2 + tree_sitter.xcodeproj/project.pbxproj | 43 +++++++++++++- 16 files changed, 316 insertions(+), 13 deletions(-) create mode 100644 spec/compiler/grammar/prepare_grammar_spec.cpp create mode 100644 src/compiler/grammar/extract_tokens.cpp create mode 100644 src/compiler/grammar/extract_tokens.h create mode 100644 src/compiler/grammar/prepare_grammar.cpp create mode 100644 src/compiler/grammar/prepare_grammar.h create mode 100644 src/compiler/grammar/search_for_symbols.cpp create mode 100644 src/compiler/grammar/search_for_symbols.h create mode 100644 src/compiler/rules/token.cpp create mode 100644 src/compiler/rules/token.h diff --git a/spec/compiler/grammar/prepare_grammar_spec.cpp b/spec/compiler/grammar/prepare_grammar_spec.cpp new file mode 100644 index 00000000..f2d2a89f --- /dev/null +++ b/spec/compiler/grammar/prepare_grammar_spec.cpp @@ -0,0 +1,35 @@ +#include "spec_helper.h" +#include "prepare_grammar.h" + +START_TEST + +using namespace tree_sitter::rules; + +describe("preparing a grammar", []() { + it("extracts character-based subtrees into a separate grammar", [&]() { + pair result = prepare_grammar(Grammar({ + { "rule1", seq({ + character('a'), + character('b'), + seq({ + sym("rule2"), + sym("rule3") }) }) } + })); + + AssertThat(result.first, Equals(Grammar({ + { "rule1", seq({ + token("1"), + seq({ + sym("rule2"), + sym("rule3") }) }) } + }))); + + AssertThat(result.second, Equals(Grammar("", { + { "1", rules::seq({ + rules::character('a'), + rules::character('b') }) } + }))); + }); +}); + +END_TEST \ No newline at end of file diff --git a/src/compiler/grammar/extract_tokens.cpp b/src/compiler/grammar/extract_tokens.cpp new file mode 100644 index 00000000..6c71073a --- /dev/null +++ b/src/compiler/grammar/extract_tokens.cpp @@ -0,0 +1,58 @@ +#include "extract_tokens.h" +#include "search_for_symbols.h" +#include + +using std::pair; +using std::string; +using std::to_string; +using std::unordered_map; + +namespace tree_sitter { + class TokenExtractor : rules::Visitor { + public: + rules::rule_ptr value; + unordered_map tokens; + + rules::rule_ptr apply(const rules::rule_ptr rule) { + if (search_for_symbols(rule)) { + rule->accept(*this); + return value; + } else { + string token_name = add_token(rule); + return rules::token(token_name); + } + } + + string add_token(const rules::rule_ptr &rule) { + string name = to_string(tokens.size() + 1); + tokens.insert({ name, rule }); + return name; + } + + void default_visit(const rules::Rule *rule) { + value = rule->copy(); + } + + void visit(const rules::Choice *choice) { + value = rules::choice({ apply(choice->left), apply(choice->right) }); + } + + void visit(const rules::Seq *seq) { + value = rules::seq({ apply(seq->left), apply(seq->right) }); + } + }; + + pair extract_tokens(const Grammar &input_grammar) { + TokenExtractor extractor; + unordered_map rules; + + for (auto pair : input_grammar.rules) { + rules.insert({ pair.first, extractor.apply(pair.second) }); + } + + return { + Grammar(input_grammar.start_rule_name, rules), + Grammar("", extractor.tokens) + }; + } +} diff --git a/src/compiler/grammar/extract_tokens.h b/src/compiler/grammar/extract_tokens.h new file mode 100644 index 00000000..09139036 --- /dev/null +++ b/src/compiler/grammar/extract_tokens.h @@ -0,0 +1,10 @@ +#ifndef __tree_sitter__extract_tokens__ +#define __tree_sitter__extract_tokens__ + +#include "grammar.h" + +namespace tree_sitter { + std::pair extract_tokens(const Grammar &); +} + +#endif diff --git a/src/compiler/grammar/grammar.cpp b/src/compiler/grammar/grammar.cpp index 4ea2705e..c74ca50a 100644 --- a/src/compiler/grammar/grammar.cpp +++ b/src/compiler/grammar/grammar.cpp @@ -3,9 +3,14 @@ using namespace std; namespace tree_sitter { - Grammar::Grammar(const std::initializer_list> &rules) : + Grammar::Grammar(const std::initializer_list> &rules) : rules(rules), start_rule_name(rules.begin()->first) {} + + Grammar::Grammar(std::string start_rule_name, const std::unordered_map &rules) : + rules(rules), + start_rule_name(start_rule_name) {} + const rules::rule_ptr Grammar::rule(const std::string &name) const { auto iter = rules.find(name); @@ -21,4 +26,28 @@ namespace tree_sitter { } return result; } + + bool Grammar::operator==(const Grammar &other) const { + if (other.start_rule_name != start_rule_name) return false; + if (other.rules.size() != rules.size()) return false; + for (auto pair : rules) { + auto other_pair = other.rules.find(pair.first); + if (other_pair == other.rules.end()) return false; + if (!other_pair->second->operator==(*pair.second)) return false; + } + return true; + } + + std::ostream& operator<<(std::ostream &stream, const Grammar &grammar) { + stream << string("# "); + stream << pair.second; + started = true; + } + return stream << string(">"); + } } diff --git a/src/compiler/grammar/grammar.h b/src/compiler/grammar/grammar.h index cad8eecb..e5ab51eb 100644 --- a/src/compiler/grammar/grammar.h +++ b/src/compiler/grammar/grammar.h @@ -7,15 +7,19 @@ namespace tree_sitter { class Grammar { - typedef std::initializer_list> rule_map_init_list; + typedef std::initializer_list> rule_map_init_list; public: Grammar(const rule_map_init_list &rules); + Grammar(std::string start_rule_name, const std::unordered_map &rules); const rules::rule_ptr rule(const std::string &) const; const std::string start_rule_name; std::vector rule_names() const; + bool operator==(const Grammar &other) const; - const std::unordered_map rules; + const std::unordered_map rules; }; + + std::ostream& operator<<(std::ostream &stream, const Grammar &grammar); } #endif diff --git a/src/compiler/grammar/prepare_grammar.cpp b/src/compiler/grammar/prepare_grammar.cpp new file mode 100644 index 00000000..a1195b35 --- /dev/null +++ b/src/compiler/grammar/prepare_grammar.cpp @@ -0,0 +1,10 @@ +#include "prepare_grammar.h" +#include "extract_tokens.h" + +using std::pair; + +namespace tree_sitter { + pair prepare_grammar(const Grammar &input_grammar) { + return extract_tokens(input_grammar); + } +} diff --git a/src/compiler/grammar/prepare_grammar.h b/src/compiler/grammar/prepare_grammar.h new file mode 100644 index 00000000..5e3fc348 --- /dev/null +++ b/src/compiler/grammar/prepare_grammar.h @@ -0,0 +1,10 @@ +#ifndef __tree_sitter__prepare_grammar__ +#define __tree_sitter__prepare_grammar__ + +#include "grammar.h" + +namespace tree_sitter { + std::pair prepare_grammar(const Grammar &); +} + +#endif diff --git a/src/compiler/grammar/search_for_symbols.cpp b/src/compiler/grammar/search_for_symbols.cpp new file mode 100644 index 00000000..a115d5a3 --- /dev/null +++ b/src/compiler/grammar/search_for_symbols.cpp @@ -0,0 +1,33 @@ +#include "search_for_symbols.h" + +namespace tree_sitter { + class SymbolSearcher : rules::Visitor { + public: + bool value; + + bool apply(const rules::rule_ptr rule) { + rule->accept(*this); + return value; + } + + void default_visit(const rules::Rule *rule) { + value = false; + } + + void visit(const rules::Symbol *symbol) { + value = true; + } + + void visit(const rules::Choice *choice) { + value = apply(choice->left) || apply(choice->right); + } + + void visit(const rules::Seq *seq) { + value = apply(seq->left) || apply(seq->right); + } + }; + + bool search_for_symbols(const rules::rule_ptr &rule) { + return SymbolSearcher().apply(rule); + } +} \ No newline at end of file diff --git a/src/compiler/grammar/search_for_symbols.h b/src/compiler/grammar/search_for_symbols.h new file mode 100644 index 00000000..d6adaf1e --- /dev/null +++ b/src/compiler/grammar/search_for_symbols.h @@ -0,0 +1,10 @@ +#ifndef __tree_sitter__search_for_symbols__ +#define __tree_sitter__search_for_symbols__ + +#include "rules.h" + +namespace tree_sitter { + bool search_for_symbols(const rules::rule_ptr &); +} + +#endif diff --git a/src/compiler/rules/rules.cpp b/src/compiler/rules/rules.cpp index f43caa79..2a511628 100644 --- a/src/compiler/rules/rules.cpp +++ b/src/compiler/rules/rules.cpp @@ -49,5 +49,9 @@ namespace tree_sitter { sym_ptr sym(const string &name) { return make_shared(name); } + + rule_ptr token(const std::string &name) { + return make_shared(name); + } } } diff --git a/src/compiler/rules/rules.h b/src/compiler/rules/rules.h index 41e4611d..173a57a2 100644 --- a/src/compiler/rules/rules.h +++ b/src/compiler/rules/rules.h @@ -4,6 +4,7 @@ #include "rule.h" #include "blank.h" #include "symbol.h" +#include "token.h" #include "choice.h" #include "seq.h" #include "string.h" @@ -24,6 +25,7 @@ namespace tree_sitter { rule_ptr seq(const std::initializer_list &rules); rule_ptr str(const std::string &value); sym_ptr sym(const std::string &name); + rule_ptr token(const std::string &name); } } diff --git a/src/compiler/rules/token.cpp b/src/compiler/rules/token.cpp new file mode 100644 index 00000000..fd13c732 --- /dev/null +++ b/src/compiler/rules/token.cpp @@ -0,0 +1,32 @@ +#include "rules.h" +#include "transition_map.h" + +using std::string; +using std::hash; + +namespace tree_sitter { + namespace rules { + Token::Token(const std::string &name) : name(name) {}; + + bool Token::operator==(const Rule &rule) const { + const Token *other = dynamic_cast(&rule); + return other && (other->name == name); + } + + size_t Token::hash_code() const { + return typeid(this).hash_code() ^ hash()(name); + } + + rule_ptr Token::copy() const { + return std::make_shared(*this); + } + + string Token::to_string() const { + return string("#"; + } + + void Token::accept(Visitor &visitor) const { + visitor.visit(this); + } + } +} \ No newline at end of file diff --git a/src/compiler/rules/token.h b/src/compiler/rules/token.h new file mode 100644 index 00000000..a05931d6 --- /dev/null +++ b/src/compiler/rules/token.h @@ -0,0 +1,23 @@ +#ifndef __tree_sitter__token__ +#define __tree_sitter__token__ + +#include "rule.h" + +namespace tree_sitter { + namespace rules { + class Token : public Rule { + public: + Token(const std::string &name); + + bool operator==(const Rule& other) const; + size_t hash_code() const; + rule_ptr copy() const; + std::string to_string() const; + void accept(Visitor &visitor) const; + + const std::string name; + }; + } +} + +#endif diff --git a/src/compiler/rules/visitor.cpp b/src/compiler/rules/visitor.cpp index 357ff76f..dd786212 100644 --- a/src/compiler/rules/visitor.cpp +++ b/src/compiler/rules/visitor.cpp @@ -2,13 +2,15 @@ namespace tree_sitter { namespace rules { - void Visitor::visit(const Blank *rule) {} - void Visitor::visit(const Symbol *rule) {} - void Visitor::visit(const Character *rule) {} - void Visitor::visit(const Choice *rule) {} - void Visitor::visit(const Repeat *rule) {} - void Visitor::visit(const Seq *rule) {} - void Visitor::visit(const String *rule) {} - void Visitor::visit(const Pattern *rule) {} + void Visitor::default_visit(const Rule *rule) {}; + void Visitor::visit(const Blank *rule) { default_visit(rule); } + void Visitor::visit(const Symbol *rule) { default_visit(rule); } + void Visitor::visit(const Character *rule) { default_visit(rule); } + void Visitor::visit(const Choice *rule) { default_visit(rule); } + void Visitor::visit(const Repeat *rule) { default_visit(rule); } + void Visitor::visit(const Seq *rule) { default_visit(rule); } + void Visitor::visit(const String *rule) { default_visit(rule); } + void Visitor::visit(const Pattern *rule) { default_visit(rule); } + void Visitor::visit(const Token *rule) { default_visit(rule); } } } \ No newline at end of file diff --git a/src/compiler/rules/visitor.h b/src/compiler/rules/visitor.h index e11c9451..63455231 100644 --- a/src/compiler/rules/visitor.h +++ b/src/compiler/rules/visitor.h @@ -7,6 +7,7 @@ namespace tree_sitter { namespace rules { class Visitor { public: + virtual void default_visit(const Rule *rule); virtual void visit(const Blank *rule); virtual void visit(const Symbol *rule); virtual void visit(const Character *rule); @@ -15,6 +16,7 @@ namespace tree_sitter { virtual void visit(const Seq *rule); virtual void visit(const String *rule); virtual void visit(const Pattern *rule); + virtual void visit(const Token *rule); }; } } diff --git a/tree_sitter.xcodeproj/project.pbxproj b/tree_sitter.xcodeproj/project.pbxproj index 9b6c9b1e..cb8b8857 100644 --- a/tree_sitter.xcodeproj/project.pbxproj +++ b/tree_sitter.xcodeproj/project.pbxproj @@ -16,6 +16,12 @@ 1213061B182C84DF00FCF928 /* item.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 12130619182C84DF00FCF928 /* item.cpp */; }; 12130622182C85D300FCF928 /* item_set.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 12130620182C85D300FCF928 /* item_set.cpp */; }; 1214930E181E200B008E9BDA /* main.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 121492E9181E200B008E9BDA /* main.cpp */; }; + 1225CC6418765693000D4723 /* prepare_grammar_spec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1225CC6318765693000D4723 /* prepare_grammar_spec.cpp */; }; + 1225CC6718765737000D4723 /* prepare_grammar.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1225CC6518765737000D4723 /* prepare_grammar.cpp */; }; + 1225CC6A187661C7000D4723 /* extract_tokens.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1225CC68187661C7000D4723 /* extract_tokens.cpp */; }; + 1225CC6D18766969000D4723 /* search_for_symbols.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1225CC6B18766969000D4723 /* search_for_symbols.cpp */; }; + 1225CC701876AFFF000D4723 /* token.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1225CC6E1876AFFF000D4723 /* token.cpp */; }; + 1225CC711876B103000D4723 /* token.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1225CC6E1876AFFF000D4723 /* token.cpp */; }; 1251209B1830145300C9B56A /* rule.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1251209A1830145300C9B56A /* rule.cpp */; }; 125120A018307DEC00C9B56A /* parse_table.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1251209E18307DEC00C9B56A /* parse_table.cpp */; }; 125120A4183083BD00C9B56A /* arithmetic.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 125120A3183083BD00C9B56A /* arithmetic.cpp */; }; @@ -107,6 +113,15 @@ 12130621182C85D300FCF928 /* item_set.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = item_set.h; sourceTree = ""; }; 121492E9181E200B008E9BDA /* main.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = main.cpp; path = spec/main.cpp; sourceTree = SOURCE_ROOT; }; 121492EA181E200B008E9BDA /* rules_spec.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = rules_spec.cpp; path = spec/compiler/rules/rules_spec.cpp; sourceTree = SOURCE_ROOT; }; + 1225CC6318765693000D4723 /* prepare_grammar_spec.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = prepare_grammar_spec.cpp; sourceTree = ""; }; + 1225CC6518765737000D4723 /* prepare_grammar.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = prepare_grammar.cpp; sourceTree = ""; }; + 1225CC6618765737000D4723 /* prepare_grammar.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = prepare_grammar.h; sourceTree = ""; }; + 1225CC68187661C7000D4723 /* extract_tokens.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = extract_tokens.cpp; sourceTree = ""; }; + 1225CC69187661C7000D4723 /* extract_tokens.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = extract_tokens.h; sourceTree = ""; }; + 1225CC6B18766969000D4723 /* search_for_symbols.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = search_for_symbols.cpp; sourceTree = ""; }; + 1225CC6C18766969000D4723 /* search_for_symbols.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = search_for_symbols.h; sourceTree = ""; }; + 1225CC6E1876AFFF000D4723 /* token.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = token.cpp; sourceTree = ""; }; + 1225CC6F1876AFFF000D4723 /* token.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = token.h; sourceTree = ""; }; 12512092182F307C00C9B56A /* table_builder_spec.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = table_builder_spec.cpp; path = spec/compiler/lr/table_builder_spec.cpp; sourceTree = SOURCE_ROOT; }; 1251209A1830145300C9B56A /* rule.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = rule.cpp; sourceTree = ""; }; 1251209E18307DEC00C9B56A /* parse_table.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = parse_table.cpp; sourceTree = ""; }; @@ -183,8 +198,6 @@ 12D136A3183678A2005F3369 /* repeat.h */, 1251209A1830145300C9B56A /* rule.cpp */, 12130607182C374800FCF928 /* rule.h */, - 12FD40E618639B910041A84E /* visitor.cpp */, - 12FD40E41862B3530041A84E /* visitor.h */, 12FD40E818641FB70041A84E /* rules.cpp */, 12E71852181D081C0051A649 /* rules.h */, 12130612182C3A1700FCF928 /* seq.cpp */, @@ -193,6 +206,10 @@ 12130616182C3D2900FCF928 /* string.h */, 12130609182C389100FCF928 /* symbol.cpp */, 1213060A182C389100FCF928 /* symbol.h */, + 1225CC6E1876AFFF000D4723 /* token.cpp */, + 1225CC6F1876AFFF000D4723 /* token.h */, + 12FD40E618639B910041A84E /* visitor.cpp */, + 12FD40E41862B3530041A84E /* visitor.h */, ); path = rules; sourceTree = ""; @@ -228,6 +245,15 @@ path = spec/lr; sourceTree = ""; }; + 1225CC6218765664000D4723 /* grammar */ = { + isa = PBXGroup; + children = ( + 1225CC6318765693000D4723 /* prepare_grammar_spec.cpp */, + ); + name = grammar; + path = compiler/grammar; + sourceTree = ""; + }; 125120A118307FCA00C9B56A /* grammars */ = { isa = PBXGroup; children = ( @@ -290,8 +316,14 @@ 12ED72A5186FC6D90089229B /* grammar */ = { isa = PBXGroup; children = ( + 1225CC68187661C7000D4723 /* extract_tokens.cpp */, + 1225CC69187661C7000D4723 /* extract_tokens.h */, 12F9A64F182DD6BC00FAF50C /* grammar.cpp */, 12F9A650182DD6BC00FAF50C /* grammar.h */, + 1225CC6518765737000D4723 /* prepare_grammar.cpp */, + 1225CC6618765737000D4723 /* prepare_grammar.h */, + 1225CC6B18766969000D4723 /* search_for_symbols.cpp */, + 1225CC6C18766969000D4723 /* search_for_symbols.h */, ); path = grammar; sourceTree = ""; @@ -337,6 +369,7 @@ 12FD40AE185EE6610041A84E /* compiler */ = { isa = PBXGroup; children = ( + 1225CC6218765664000D4723 /* grammar */, 12FD4063185E75290041A84E /* generate_parsers.cpp */, 1213061C182C854F00FCF928 /* lr */, 12D1369F18357066005F3369 /* rules */, @@ -452,6 +485,7 @@ 12FD40D7185FEEDB0041A84E /* item_spec.cpp in Sources */, 12FD40D5185FEEDB0041A84E /* item_set_spec.cpp in Sources */, 12130614182C3A1700FCF928 /* seq.cpp in Sources */, + 1225CC6A187661C7000D4723 /* extract_tokens.cpp in Sources */, 129D242C183EB1EB00FE9F71 /* table_builder.cpp in Sources */, 125120A4183083BD00C9B56A /* arithmetic.cpp in Sources */, 1213061B182C84DF00FCF928 /* item.cpp in Sources */, @@ -462,6 +496,7 @@ 1213060E182C398300FCF928 /* choice.cpp in Sources */, 12F9A64E182DD5FD00FAF50C /* spec_helper.cpp in Sources */, 12FD40F7186A16020041A84E /* lex_table.cpp in Sources */, + 1225CC701876AFFF000D4723 /* token.cpp in Sources */, 12FD40E918641FB70041A84E /* rules.cpp in Sources */, 12FD4061185E68470041A84E /* c_code.cpp in Sources */, 12FD40D8185FEEDF0041A84E /* rules_spec.cpp in Sources */, @@ -471,8 +506,11 @@ 12F9A651182DD6BC00FAF50C /* grammar.cpp in Sources */, 12FD40D6185FEEDB0041A84E /* table_builder_spec.cpp in Sources */, 12D136A4183678A2005F3369 /* repeat.cpp in Sources */, + 1225CC6418765693000D4723 /* prepare_grammar_spec.cpp in Sources */, 12FD40F3186641C00041A84E /* char_match.cpp in Sources */, 12FD40E718639B910041A84E /* visitor.cpp in Sources */, + 1225CC6718765737000D4723 /* prepare_grammar.cpp in Sources */, + 1225CC6D18766969000D4723 /* search_for_symbols.cpp in Sources */, 12ED72A7186FC8220089229B /* transitions_spec.cpp in Sources */, 12130622182C85D300FCF928 /* item_set.cpp in Sources */, 12130605182C348F00FCF928 /* character.cpp in Sources */, @@ -494,6 +532,7 @@ 12FD40B9185EEB5E0041A84E /* string.cpp in Sources */, 12FD40EF186641510041A84E /* transitions.cpp in Sources */, 12FD40BB185EEB5E0041A84E /* blank.cpp in Sources */, + 1225CC711876B103000D4723 /* token.cpp in Sources */, 12FD40F4186641C00041A84E /* char_match.cpp in Sources */, 12FD40BD185EEB5E0041A84E /* choice.cpp in Sources */, 12FD40DF1860064C0041A84E /* tree.c in Sources */,