traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Intern symbols during grammar preparation

Browse source
This commit is contained in:
Max Brunsfeld 2014-04-22 23:38:26 -07:00
parent 33d781f492
commit 68d44fd565
67 changed files with 10829 additions and 10557 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/compiler/rules/built_in_symbols.cc
View file

@ -2,8 +2,8 @@
namespace tree_sitter {
namespace rules {
Symbol ERROR() { return Symbol("error", SymbolTypeBuiltIn); }
Symbol START() { return Symbol("start", SymbolTypeBuiltIn); }
Symbol END_OF_INPUT() { return Symbol("end", SymbolTypeBuiltIn); }
ISymbol END_OF_INPUT() { return ISymbol(-1, SymbolOptionToken); }
ISymbol ERROR() { return ISymbol(-2, SymbolOptionToken); }
ISymbol START() { return ISymbol(-3); }
}
}

8
src/compiler/rules/built_in_symbols.h
View file

@ -1,13 +1,13 @@
#ifndef COMPILER_RULES_BUILT_IN_SYMBOLS_H_
#define COMPILER_RULES_BUILT_IN_SYMBOLS_H_
#include "compiler/rules/symbol.h"
#include "compiler/rules/interned_symbol.h"
namespace tree_sitter {
namespace rules {
Symbol ERROR();
Symbol START();
Symbol END_OF_INPUT();
ISymbol ERROR();
ISymbol START();
ISymbol END_OF_INPUT();
}
}

61
src/compiler/rules/interned_symbol.cc Normal file
View file

@ -0,0 +1,61 @@
#include "compiler/rules/interned_symbol.h"
#include <map>
#include <string>
#include "compiler/rules/visitor.h"
namespace tree_sitter {
using std::string;
using std::to_string;
using std::hash;
namespace rules {
ISymbol::ISymbol(int index) :
index(index),
options(SymbolOption(0)) {}
ISymbol::ISymbol(int index, SymbolOption options) :
index(index),
options(options) {}
bool ISymbol::operator==(const Rule &rule) const {
const ISymbol *other = dynamic_cast<const ISymbol *>(&rule);
return other && (other->index == index) && (other->options == options);
}
size_t ISymbol::hash_code() const {
return hash<size_t>()(index) ^ hash<int16_t>()(options);
}
rule_ptr ISymbol::copy() const {
return std::make_shared<ISymbol>(*this);
}
string ISymbol::to_string() const {
string name = (options & SymbolOptionAuxiliary) ? "aux_" : "";
name += (options & SymbolOptionToken) ? "token" : "sym";
return "#<" + name + std::to_string(index) + ">";
}
bool ISymbol::operator<(const ISymbol &other) const {
if (options < other.options) return true;
if (options > other.options) return false;
return (index < other.index);
}
bool ISymbol::is_token() const {
return options & SymbolOptionToken;
}
bool ISymbol::is_built_in() const {
return index < 0;
}
bool ISymbol::is_auxiliary() const {
return options & SymbolOptionAuxiliary;
}
void ISymbol::accept(Visitor *visitor) const {
visitor->visit(this);
}
}
}

45
src/compiler/rules/interned_symbol.h Normal file
View file

@ -0,0 +1,45 @@
#ifndef COMPILER_RULES_INTERNED_SYMBOL_H_
#define COMPILER_RULES_INTERNED_SYMBOL_H_
#include "compiler/rules/symbol.h"
namespace tree_sitter {
namespace rules {
typedef enum {
SymbolOptionToken = 1 << 0,
SymbolOptionAuxiliary = 1 << 1,
} SymbolOption;
class ISymbol : public Rule {
public:
explicit ISymbol(int index);
ISymbol(int index, SymbolOption options);
bool operator==(const Rule& other) const;
size_t hash_code() const;
rule_ptr copy() const;
std::string to_string() const;
void accept(Visitor *visitor) const;
bool operator<(const ISymbol &other) const;
bool is_token() const;
bool is_built_in() const;
bool is_auxiliary() const;
int index;
SymbolOption options;
};
}
}
namespace std {
template<>
struct hash<tree_sitter::rules::ISymbol> {
size_t operator()(const tree_sitter::rules::ISymbol &rule) const {
return rule.hash_code();
}
};
}
#endif // COMPILER_RULES_INTERNED_SYMBOL_H_

34
src/compiler/rules/symbol.cc
View file

@ -8,8 +8,7 @@ namespace tree_sitter {
using std::hash;
namespace rules {
Symbol::Symbol(const std::string &name) : name(name), type(SymbolTypeNormal) {}
Symbol::Symbol(const std::string &name, SymbolType type) : name(name), type(type) {}
Symbol::Symbol(const std::string &name) : name(name) {}
bool Symbol::operator==(const Rule &rule) const {
const Symbol *other = dynamic_cast<const Symbol *>(&rule);
@ -17,11 +16,11 @@ namespace tree_sitter {
}
bool Symbol::operator==(const Symbol &other) const {
return (other.name == name) && (other.type == type);
return other.name == name;
}
size_t Symbol::hash_code() const {
return hash<string>()(name) ^ hash<int16_t>()(type);
return hash<string>()(name);
}
rule_ptr Symbol::copy() const {
@ -29,34 +28,11 @@ namespace tree_sitter {
}
string Symbol::to_string() const {
switch (type) {
case SymbolTypeNormal:
return string("#<sym '") + name + "'>";
case SymbolTypeAuxiliary:
return string("#<aux_sym '") + name + "'>";
case SymbolTypeBuiltIn:
return string("#<builtin_sym '") + name + "'>";
default:
return "";
}
return string("#<sym '") + name + "'>";
}
bool Symbol::operator<(const Symbol &other) const {
if (type < other.type) return true;
if (type > other.type) return false;
return (name < other.name);
}
bool Symbol::is_built_in() const {
return type == SymbolTypeBuiltIn;
}
bool Symbol::is_auxiliary() const {
return type == SymbolTypeAuxiliary;
}
bool Symbol::is_hidden() const {
return (name.front() == '_' || type == SymbolTypeAuxiliary);
return name < other.name;
}
void Symbol::accept(Visitor *visitor) const {

12
src/compiler/rules/symbol.h
View file

@ -7,16 +7,9 @@
namespace tree_sitter {
namespace rules {
typedef enum {
SymbolTypeNormal,
SymbolTypeAuxiliary,
SymbolTypeBuiltIn
} SymbolType;
class Symbol : public Rule {
public:
explicit Symbol(const std::string &name);
Symbol(const std::string &name, SymbolType type);
bool operator==(const Rule& other) const;
bool operator==(const Symbol &other) const;
@ -27,12 +20,7 @@ namespace tree_sitter {
void accept(Visitor *visitor) const;
bool operator<(const Symbol &other) const;
bool is_built_in() const;
bool is_hidden() const;
bool is_auxiliary() const;
std::string name;
SymbolType type;
};
}
}

5
src/compiler/rules/visitor.h
View file

@ -12,6 +12,7 @@ namespace tree_sitter {
class Repeat;
class Seq;
class String;
class ISymbol;
class Pattern;
class Metadata;
@ -26,6 +27,7 @@ namespace tree_sitter {
virtual void visit(const Seq *rule) = 0;
virtual void visit(const String *rule) = 0;
virtual void visit(const Symbol *rule) = 0;
virtual void visit(const ISymbol *rule) = 0;
};
template<typename T>
@ -48,6 +50,7 @@ namespace tree_sitter {
virtual T apply_to(const Seq *rule) { return default_apply((const Rule *)rule); }
virtual T apply_to(const String *rule) { return default_apply((const Rule *)rule); }
virtual T apply_to(const Symbol *rule) { return default_apply((const Rule *)rule); }
virtual T apply_to(const ISymbol *rule) { return default_apply((const Rule *)rule); }
void visit(const Blank *rule) { value_ = apply_to(rule); }
void visit(const CharacterSet *rule) { value_ = apply_to(rule); }
@ -58,12 +61,14 @@ namespace tree_sitter {
void visit(const Seq *rule) { value_ = apply_to(rule); }
void visit(const String *rule) { value_ = apply_to(rule); }
void visit(const Symbol *rule) { value_ = apply_to(rule); }
void visit(const ISymbol *rule) { value_ = apply_to(rule); }
private:
T value_;
};
class IdentityRuleFn : public RuleFn<rule_ptr> {
protected:
virtual rule_ptr default_apply(const Rule *rule);
virtual rule_ptr apply_to(const Choice *rule);
virtual rule_ptr apply_to(const Metadata *rule);