traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Give extracted string/regex tokens more descriptive names

Browse source
This commit is contained in:
Max Brunsfeld 2014-10-31 08:18:28 -07:00
parent 290cbefe6d
commit 4fc960e4ec
2 changed files with 45 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

45
src/compiler/generate_code/c_code.cc
View file

@ -22,6 +22,38 @@ using std::to_string;
using std::vector;
using util::escape_char;
static const map<char, string> REPLACEMENTS({
{ '=', "EQ" },
{ '\'', "SQUOTE" },
{ '"', "DQUOTE" },
{ '`', "BQUOTE" },
{ '.', "DOT" },
{ ',', "COMMA" },
{ ':', "COLON" },
{ ';', "SEMI" },
{ '(', "LPAREN" },
{ ')', "RPAREN" },
{ '<', "LT" },
{ '>', "GT" },
{ '{', "LBRACE" },
{ '}', "RBRACE" },
{ '[', "LBRACK" },
{ ']', "RBRACK" },
{ '&', "AMP" },
{ '|', "PIPE" },
{ '%', "PERCENT" },
{ '-', "DASH" },
{ '+', "PLUS" },
{ '*', "STAR" },
{ '~', "TILDE" },
{ '!', "BANG" },
{ '^', "CARET" },
{ '$', "DOLLAR" },
{ '@', "AT" },
{ '/', "SLASH" },
{ '\\', "BSLASH" },
});
class CCodeGenerator {
string buffer;
size_t indent_level;
@ -279,9 +311,9 @@ class CCodeGenerator {
} else {
string name = sanitize_name(rule_name(symbol));
if (symbol.is_auxiliary())
return "ts_aux_sym_" + name;
return "aux_sym_" + name;
else
return "ts_sym_" + name;
return "sym_" + name;
}
}
@ -338,6 +370,15 @@ class CCodeGenerator {
if (('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
('0' <= c && c <= '9') || (c == '_')) {
stripped_name += c;
continue;
}
auto replacement = REPLACEMENTS.find(c);
if (replacement != REPLACEMENTS.end()) {
if (stripped_name[stripped_name.size() - 1] != '_')
stripped_name += "_";
stripped_name += replacement->second;
continue;
}
}

4
src/compiler/prepare_grammar/token_description.cc
View file

@ -15,11 +15,11 @@ using std::string;
class TokenDescription : public rules::RuleFn<string> {
string apply_to(const rules::Pattern *rule) {
return "/" + util::escape_string(rule->value) + "/";
return "PAT_" + util::escape_string(rule->value);
}
string apply_to(const rules::String *rule) {
return "'" + util::escape_string(rule->value) + "'";
return "STR_" + util::escape_string(rule->value);
}
string apply_to(const rules::Metadata *rule) { return apply(rule->rule); }