traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add token helper for building token rules

Browse source 
Now you can specify the structure of tokens using
all of the rule functions, not just `str` and `pattern`
This commit is contained in:
Max Brunsfeld 2014-05-01 12:43:29 -07:00
parent d685edf015
commit 6d40dcf881
8 changed files with 59 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

18
src/compiler/prepare_grammar/extract_tokens.cc
View file

@ -11,6 +11,7 @@
#include "compiler/rules/blank.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/string.h"
#include "compiler/rules/metadata.h"
#include "compiler/rules/pattern.h"
namespace tree_sitter {
@ -28,6 +29,7 @@ namespace tree_sitter {
class IsToken : public rules::RuleFn<bool> {
bool apply_to(const rules::String *rule) { return true; }
bool apply_to(const rules::Pattern *rule) { return true; }
bool apply_to(const rules::Metadata *rule) { return rule->value_for(rules::IS_TOKEN); }
};
class SymbolInliner : public rules::IdentityRuleFn {
@ -69,16 +71,28 @@ namespace tree_sitter {
return index;
}
rule_ptr apply_to_token(const rules::rule_ptr rule) {
size_t index = add_token(rule);
return make_shared<rules::Symbol>(index, rules::SymbolOption(rules::SymbolOptionToken|rules::SymbolOptionAuxiliary));
}
rule_ptr default_apply(const rules::Rule *rule) {
auto result = rule->copy();
if (IsToken().apply(result)) {
size_t index = add_token(result);
return make_shared<rules::Symbol>(index, rules::SymbolOption(rules::SymbolOptionToken|rules::SymbolOptionAuxiliary));
return apply_to_token(result);
} else {
return result;
}
}
rule_ptr apply_to(const rules::Metadata *rule) {
if (rule->value_for(rules::IS_TOKEN)) {
return apply_to_token(rule->copy());
} else {
return make_shared<rules::Metadata>(apply(rule->rule), rule->value);
}
}
public:
vector<pair<string, rule_ptr>> tokens;
};