traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Give better symbol names to generated tokens

Browse source 
This should make debugging easier
This commit is contained in:
Max Brunsfeld 2014-03-27 12:54:54 -07:00
parent 2226234924
commit e1ac62edc5
14 changed files with 1017 additions and 842 deletions
Download patch file Download diff file Expand all files Collapse all files

45
src/compiler/generate_code/c_code.cc
View file

@ -49,11 +49,16 @@ namespace tree_sitter {
const string name;
const ParseTable parse_table;
const LexTable lex_table;
const map<rules::Symbol, string> symbol_names;
public:
CCodeGenerator(string name, const ParseTable &parse_table, const LexTable &lex_table) :
CCodeGenerator(string name,
const ParseTable &parse_table,
const LexTable &lex_table,
const map<rules::Symbol, string> &symbol_names) :
name(name),
parse_table(parse_table),
lex_table(lex_table)
lex_table(lex_table),
symbol_names(symbol_names)
{}
string code() {
@ -61,7 +66,7 @@ namespace tree_sitter {
includes(),
state_and_symbol_counts(),
symbol_enum(),
rule_names_list(),
symbol_names_list(),
hidden_symbols_list(),
lex_function(),
lex_states_list(),
@ -83,21 +88,6 @@ namespace tree_sitter {
}
}
string character_code(char character) {
switch (character) {
case '\0':
return "\\0";
case '"':
return "\\\"";
case '\n':
return "\\n";
case '\\':
return "\\\\";
default:
return string() + character;
}
}
string condition_for_character_range(const rules::CharacterRange &range) {
string lookahead("lookahead");
if (range.min == range.max) {
@ -191,14 +181,14 @@ namespace tree_sitter {
return result + "};";
}
string rule_names_list() {
string result = "SYMBOL_NAMES = {\n";
result += indent(string("\"") + "error") + "\",\n";
result += indent(string("\"") + "end") + "\",\n";
string symbol_names_list() {
set<rules::Symbol> symbols(parse_table.symbols);
symbols.insert(rules::Symbol("end", rules::SymbolTypeBuiltIn));
symbols.insert(rules::Symbol("error", rules::SymbolTypeBuiltIn));
string result = "SYMBOL_NAMES = {\n";
for (auto symbol : parse_table.symbols)
if (!symbol.is_built_in())
result += indent(string("\"") + symbol.name) + "\",\n";
result += indent("[" + symbol_id(symbol) + "] = \"" + symbol_names.find(symbol)->second) + "\",\n";
return result + "};";
}
@ -261,8 +251,11 @@ namespace tree_sitter {
}
};
string c_code(string name, const ParseTable &parse_table, const LexTable &lex_table) {
return CCodeGenerator(name, parse_table, lex_table).code();
string c_code(string name,
const ParseTable &parse_table,
const LexTable &lex_table,
const map<rules::Symbol, string> &symbol_names) {
return CCodeGenerator(name, parse_table, lex_table, symbol_names).code();
}
}
}

6
src/compiler/generate_code/c_code.h
View file

@ -2,12 +2,16 @@
#define COMPILER_GENERATE_CODE_C_CODE_H_
#include <string>
#include <map>
#include "compiler/parse_table.h"
#include "compiler/lex_table.h"
namespace tree_sitter {
namespace generate_code {
std::string c_code(std::string name, const ParseTable &parse_table, const LexTable &lex_table);
std::string c_code(std::string name,
const ParseTable &parse_table,
const LexTable &lex_table,
const std::map<rules::Symbol, std::string> &symbol_names);
}
}

35
src/compiler/generate_code/helpers.cc
View file

@ -1,18 +1,11 @@
#include "compiler/generate_code/helpers.h"
#include "compiler/util/string_helpers.h"
namespace tree_sitter {
using std::string;
using std::vector;
namespace generate_code {
static void str_replace(string *input, const string &search, const string &replace) {
size_t pos = 0;
while (1) {
pos = input->find(search, pos);
if (pos == string::npos) break;
input->erase(pos, search.length());
input->insert(pos, replace);
pos += replace.length();
}
}
string join(vector<string> lines, string separator) {
string result;
bool started = false;
@ -30,13 +23,23 @@ namespace tree_sitter {
string indent(string input) {
string tab = " ";
str_replace(&input, "\n", "\n" + tab);
util::str_replace(&input, "\n", "\n" + tab);
return tab + input;
}
string escape_string(string input) {
str_replace(&input, "\"", "\\\"");
return input;
string character_code(char character) {
switch (character) {
case '\0':
return "\\0";
case '"':
return "\\\"";
case '\n':
return "\\n";
case '\\':
return "\\\\";
default:
return string() + character;
}
}
}
}

11
src/compiler/generate_code/helpers.h
View file

@ -5,14 +5,11 @@
#include <vector>
namespace tree_sitter {
using std::string;
using std::vector;
namespace generate_code {
string indent(string input);
string join(vector<string> lines, string separator);
string join(vector<string> lines);
string escape_string(string input);
std::string indent(std::string input);
std::string join(std::vector<std::string> lines, std::string separator);
std::string join(std::vector<std::string> lines);
std::string character_code(char character);
}
}