Add script to trim whitespace
This commit is contained in:
Max Brunsfeld
parent
e681a63552
commit
39aa0ccc91
66 changed files with 350 additions and 347 deletions
|
|
@ -4,23 +4,23 @@
|
|||
namespace tree_sitter {
|
||||
namespace rules {
|
||||
Blank::Blank() {}
|
||||
|
||||
|
||||
bool Blank::operator==(const Rule &rule) const {
|
||||
return dynamic_cast<const Blank *>(&rule) != nullptr;
|
||||
}
|
||||
|
||||
|
||||
size_t Blank::hash_code() const {
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
rule_ptr Blank::copy() const {
|
||||
return std::make_shared<Blank>();
|
||||
}
|
||||
|
||||
|
||||
std::string Blank::to_string() const {
|
||||
return "#<blank>";
|
||||
}
|
||||
|
||||
|
||||
void Blank::accept(Visitor &visitor) const {
|
||||
visitor.visit(this);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ namespace tree_sitter {
|
|||
class Blank : public Rule {
|
||||
public:
|
||||
Blank();
|
||||
|
||||
|
||||
bool operator==(const Rule& other) const;
|
||||
size_t hash_code() const;
|
||||
rule_ptr copy() const;
|
||||
|
|
|
|||
|
|
@ -3,24 +3,24 @@
|
|||
|
||||
namespace tree_sitter {
|
||||
using std::string;
|
||||
|
||||
|
||||
namespace rules {
|
||||
static const char MAX_CHAR = '\xff';
|
||||
|
||||
|
||||
CharacterRange::CharacterRange(char value) : min(value), max(value) {}
|
||||
CharacterRange::CharacterRange(char min, char max) : min(min), max(max) {}
|
||||
|
||||
|
||||
bool CharacterRange::operator==(const CharacterRange &other) const {
|
||||
return min == other.min && max == other.max;
|
||||
}
|
||||
|
||||
|
||||
bool CharacterRange::operator<(const CharacterRange &other) const {
|
||||
if (min < other.min) return true;
|
||||
if (min > other.min) return false;
|
||||
if (max < other.max) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
string escape_character(char input) {
|
||||
switch (input) {
|
||||
case '\0':
|
||||
|
|
@ -31,7 +31,7 @@ namespace tree_sitter {
|
|||
return string() + input;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
string CharacterRange::to_string() const {
|
||||
if (min == 0 && max == MAX_CHAR)
|
||||
return "<ANY>";
|
||||
|
|
|
|||
|
|
@ -14,24 +14,24 @@ namespace tree_sitter {
|
|||
int max_int(const CharacterRange &range) {
|
||||
return range.max == MAX_CHAR ? 255 : (int)range.max;
|
||||
}
|
||||
|
||||
|
||||
int min_int(const CharacterRange &range) {
|
||||
return (int)range.min;
|
||||
}
|
||||
|
||||
|
||||
CharacterSet::CharacterSet() : ranges({}) {}
|
||||
CharacterSet::CharacterSet(const set<CharacterRange> &ranges) : ranges(ranges) {}
|
||||
CharacterSet::CharacterSet(const initializer_list<CharacterRange> &ranges) : ranges(ranges) {}
|
||||
|
||||
|
||||
bool CharacterSet::operator==(const Rule &rule) const {
|
||||
const CharacterSet *other = dynamic_cast<const CharacterSet *>(&rule);
|
||||
return other && (ranges == other->ranges);
|
||||
}
|
||||
|
||||
|
||||
bool CharacterSet::operator<(const CharacterSet &other) const {
|
||||
return ranges < other.ranges;
|
||||
}
|
||||
|
||||
|
||||
size_t CharacterSet::hash_code() const {
|
||||
size_t result = std::hash<size_t>()(ranges.size());
|
||||
for (auto &range : ranges) {
|
||||
|
|
@ -51,13 +51,13 @@ namespace tree_sitter {
|
|||
result += " " + range.to_string();
|
||||
return result + " }>";
|
||||
}
|
||||
|
||||
|
||||
CharacterSet CharacterSet::complement() const {
|
||||
CharacterSet result({ {0, MAX_CHAR} });
|
||||
result.remove_set(*this);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
std::pair<CharacterSet, bool> CharacterSet::most_compact_representation() const {
|
||||
auto first_range = *ranges.begin();
|
||||
if (first_range.min == 0 && first_range.max > 0) {
|
||||
|
|
@ -66,7 +66,7 @@ namespace tree_sitter {
|
|||
return { *this, true };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void add_range(CharacterSet *self, CharacterRange new_range) {
|
||||
set<CharacterRange> new_ranges;
|
||||
|
||||
|
|
@ -87,7 +87,7 @@ namespace tree_sitter {
|
|||
new_range.max = range.max;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (!is_adjacent) {
|
||||
new_ranges.insert(range);
|
||||
}
|
||||
|
|
@ -95,7 +95,7 @@ namespace tree_sitter {
|
|||
new_ranges.insert(new_range);
|
||||
self->ranges = new_ranges;
|
||||
}
|
||||
|
||||
|
||||
CharacterSet remove_range(CharacterSet *self, CharacterRange new_range) {
|
||||
CharacterSet removed_set;
|
||||
set<CharacterRange> new_ranges;
|
||||
|
|
@ -126,17 +126,17 @@ namespace tree_sitter {
|
|||
self->ranges = new_ranges;
|
||||
return removed_set;
|
||||
}
|
||||
|
||||
|
||||
bool CharacterSet::is_empty() const {
|
||||
return ranges.empty();
|
||||
}
|
||||
|
||||
|
||||
void CharacterSet::add_set(const CharacterSet &other) {
|
||||
for (auto &other_range : other.ranges) {
|
||||
add_range(this, other_range);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
CharacterSet CharacterSet::remove_set(const CharacterSet &other) {
|
||||
CharacterSet result;
|
||||
for (auto &other_range : other.ranges) {
|
||||
|
|
@ -145,12 +145,12 @@ namespace tree_sitter {
|
|||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
CharacterSet CharacterSet::intersect(const CharacterSet &set) const {
|
||||
CharacterSet copy = *this;
|
||||
return copy.remove_set(set);
|
||||
}
|
||||
|
||||
|
||||
void CharacterSet::accept(Visitor &visitor) const {
|
||||
visitor.visit(this);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,21 +13,21 @@ namespace tree_sitter {
|
|||
CharacterSet();
|
||||
CharacterSet(const std::set<CharacterRange> &ranges);
|
||||
CharacterSet(const std::initializer_list<CharacterRange> &ranges);
|
||||
|
||||
|
||||
bool operator==(const Rule& other) const;
|
||||
bool operator<(const CharacterSet &) const;
|
||||
size_t hash_code() const;
|
||||
rule_ptr copy() const;
|
||||
std::string to_string() const;
|
||||
void accept(Visitor &visitor) const;
|
||||
|
||||
|
||||
void add_set(const CharacterSet &other);
|
||||
CharacterSet remove_set(const CharacterSet &other);
|
||||
CharacterSet complement() const;
|
||||
CharacterSet intersect(const CharacterSet &) const;
|
||||
std::pair<CharacterSet, bool> most_compact_representation() const;
|
||||
bool is_empty() const;
|
||||
|
||||
|
||||
std::set<CharacterRange> ranges;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,17 +5,17 @@ namespace tree_sitter {
|
|||
using std::string;
|
||||
using std::make_shared;
|
||||
using std::vector;
|
||||
|
||||
|
||||
namespace rules {
|
||||
Choice::Choice(rule_ptr left, rule_ptr right) : left(left), right(right) {};
|
||||
|
||||
|
||||
rule_ptr Choice::Build(const vector<rule_ptr> &rules) {
|
||||
rule_ptr result;
|
||||
for (auto rule : rules)
|
||||
result = result.get() ? make_shared<Choice>(result, rule) : rule;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
bool Choice::operator==(const Rule &rule) const {
|
||||
const Choice *other = dynamic_cast<const Choice *>(&rule);
|
||||
return other && (*other->left == *left) && (*other->right == *right);
|
||||
|
|
@ -24,7 +24,7 @@ namespace tree_sitter {
|
|||
size_t Choice::hash_code() const {
|
||||
return left->hash_code() ^ right->hash_code();
|
||||
}
|
||||
|
||||
|
||||
rule_ptr Choice::copy() const {
|
||||
return std::make_shared<Choice>(*this);
|
||||
}
|
||||
|
|
@ -32,7 +32,7 @@ namespace tree_sitter {
|
|||
string Choice::to_string() const {
|
||||
return string("#<choice ") + left->to_string() + " " + right->to_string() + ">";
|
||||
}
|
||||
|
||||
|
||||
void Choice::accept(Visitor &visitor) const {
|
||||
visitor.visit(this);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,14 +12,14 @@ namespace tree_sitter {
|
|||
using std::hash;
|
||||
using std::make_shared;
|
||||
using std::set;
|
||||
|
||||
|
||||
class PatternParser {
|
||||
public:
|
||||
PatternParser(const string &input) :
|
||||
input(input),
|
||||
length(input.length()),
|
||||
position(0) {}
|
||||
|
||||
|
||||
rule_ptr rule() {
|
||||
auto result = term();
|
||||
while (has_more_input() && peek() == '|') {
|
||||
|
|
@ -28,7 +28,7 @@ namespace tree_sitter {
|
|||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
rule_ptr term() {
|
||||
rule_ptr result = factor();
|
||||
|
|
@ -36,7 +36,7 @@ namespace tree_sitter {
|
|||
result = Seq::Build({ result, factor() });
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
rule_ptr factor() {
|
||||
rule_ptr result = atom();
|
||||
if (has_more_input() && (peek() == '+')) {
|
||||
|
|
@ -45,7 +45,7 @@ namespace tree_sitter {
|
|||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
rule_ptr atom() {
|
||||
rule_ptr result;
|
||||
switch (peek()) {
|
||||
|
|
@ -77,7 +77,7 @@ namespace tree_sitter {
|
|||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
CharacterSet char_set() {
|
||||
bool is_affirmative = true;
|
||||
if (peek() == '^') {
|
||||
|
|
@ -89,7 +89,7 @@ namespace tree_sitter {
|
|||
result.add_set(single_char());
|
||||
return is_affirmative ? result : result.complement();
|
||||
}
|
||||
|
||||
|
||||
CharacterSet single_char() {
|
||||
CharacterSet value;
|
||||
switch (peek()) {
|
||||
|
|
@ -112,7 +112,7 @@ namespace tree_sitter {
|
|||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
|
||||
CharacterSet escaped_char(char value) {
|
||||
switch (value) {
|
||||
case '\\':
|
||||
|
|
@ -128,52 +128,52 @@ namespace tree_sitter {
|
|||
return CharacterSet();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void next() {
|
||||
position++;
|
||||
}
|
||||
|
||||
|
||||
char peek() {
|
||||
return input[position];
|
||||
}
|
||||
|
||||
|
||||
bool has_more_input() {
|
||||
return position < length;
|
||||
}
|
||||
|
||||
|
||||
bool has_error() {
|
||||
return error != "";
|
||||
}
|
||||
|
||||
|
||||
string error;
|
||||
const string input;
|
||||
const size_t length;
|
||||
size_t position;
|
||||
};
|
||||
|
||||
|
||||
Pattern::Pattern(const string &string) : value(string) {};
|
||||
|
||||
|
||||
bool Pattern::operator==(tree_sitter::rules::Rule const &other) const {
|
||||
auto pattern = dynamic_cast<const Pattern *>(&other);
|
||||
return pattern && (pattern->value == value);
|
||||
}
|
||||
|
||||
|
||||
size_t Pattern::hash_code() const {
|
||||
return hash<string>()(value);
|
||||
}
|
||||
|
||||
|
||||
rule_ptr Pattern::copy() const {
|
||||
return std::make_shared<Pattern>(*this);
|
||||
}
|
||||
|
||||
|
||||
string Pattern::to_string() const {
|
||||
return string("#<pattern '") + value + "'>";
|
||||
}
|
||||
|
||||
|
||||
void Pattern::accept(Visitor &visitor) const {
|
||||
visitor.visit(this);
|
||||
}
|
||||
|
||||
|
||||
rule_ptr Pattern::to_rule_tree() const {
|
||||
return PatternParser(value).rule();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ namespace tree_sitter {
|
|||
rule_ptr copy() const;
|
||||
std::string to_string() const;
|
||||
void accept(Visitor &visitor) const;
|
||||
|
||||
|
||||
rule_ptr to_rule_tree() const;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,24 +6,24 @@ using std::string;
|
|||
namespace tree_sitter {
|
||||
namespace rules {
|
||||
Repeat::Repeat(const rule_ptr content) : content(content) {}
|
||||
|
||||
|
||||
bool Repeat::operator==(const Rule &rule) const {
|
||||
const Repeat *other = dynamic_cast<const Repeat *>(&rule);
|
||||
return other && (*other->content == *content);
|
||||
}
|
||||
|
||||
|
||||
size_t Repeat::hash_code() const {
|
||||
return content->hash_code();
|
||||
}
|
||||
|
||||
|
||||
rule_ptr Repeat::copy() const {
|
||||
return std::make_shared<Repeat>(*this);
|
||||
}
|
||||
|
||||
|
||||
string Repeat::to_string() const {
|
||||
return string("#<repeat ") + content->to_string() + ">";
|
||||
}
|
||||
|
||||
|
||||
void Repeat::accept(Visitor &visitor) const {
|
||||
visitor.visit(this);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ namespace tree_sitter {
|
|||
bool Rule::operator!=(const Rule &other) const {
|
||||
return !this->operator==(other);
|
||||
}
|
||||
|
||||
|
||||
ostream& operator<<(ostream& stream, const Rule &rule) {
|
||||
return stream << rule.to_string();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ namespace tree_sitter {
|
|||
virtual std::string to_string() const = 0;
|
||||
virtual void accept(Visitor &visitor) const = 0;
|
||||
};
|
||||
|
||||
|
||||
std::ostream& operator<<(std::ostream& stream, const Rule &rule);
|
||||
std::ostream& operator<<(std::ostream& stream, const rule_ptr &rule);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,20 +15,20 @@ namespace tree_sitter {
|
|||
using std::string;
|
||||
using std::set;
|
||||
using std::vector;
|
||||
|
||||
|
||||
namespace rules {
|
||||
rule_ptr blank() {
|
||||
return make_shared<Blank>();
|
||||
}
|
||||
|
||||
|
||||
rule_ptr choice(const vector<rule_ptr> &rules) {
|
||||
return Choice::Build(rules);
|
||||
}
|
||||
|
||||
|
||||
rule_ptr repeat(const rule_ptr &content) {
|
||||
return std::make_shared<Repeat>(content);
|
||||
}
|
||||
|
||||
|
||||
rule_ptr seq(const vector<rule_ptr> &rules) {
|
||||
return Seq::Build(rules);
|
||||
}
|
||||
|
|
@ -44,11 +44,11 @@ namespace tree_sitter {
|
|||
rule_ptr pattern(const string &value) {
|
||||
return make_shared<Pattern>(value);
|
||||
}
|
||||
|
||||
|
||||
rule_ptr str(const string &value) {
|
||||
return make_shared<String>(value);
|
||||
}
|
||||
|
||||
|
||||
rule_ptr err(const rule_ptr &rule) {
|
||||
return choice({ rule, ERROR.copy() });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,19 +21,19 @@ namespace tree_sitter {
|
|||
const Seq *other = dynamic_cast<const Seq *>(&rule);
|
||||
return other && (*other->left == *left) && (*other->right == *right);
|
||||
}
|
||||
|
||||
|
||||
size_t Seq::hash_code() const {
|
||||
return left->hash_code() ^ right->hash_code();
|
||||
}
|
||||
|
||||
|
||||
rule_ptr Seq::copy() const {
|
||||
return std::make_shared<Seq>(*this);
|
||||
}
|
||||
|
||||
|
||||
string Seq::to_string() const {
|
||||
return string("#<seq ") + left->to_string() + " " + right->to_string() + ">";
|
||||
}
|
||||
|
||||
|
||||
void Seq::accept(Visitor &visitor) const {
|
||||
visitor.visit(this);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,24 +7,24 @@ using std::hash;
|
|||
namespace tree_sitter {
|
||||
namespace rules {
|
||||
String::String(string value) : value(value) {};
|
||||
|
||||
|
||||
bool String::operator==(const Rule &rule) const {
|
||||
const String *other = dynamic_cast<const String *>(&rule);
|
||||
return other && (other->value == value);
|
||||
}
|
||||
|
||||
|
||||
size_t String::hash_code() const {
|
||||
return hash<string>()(value);
|
||||
}
|
||||
|
||||
|
||||
rule_ptr String::copy() const {
|
||||
return std::make_shared<String>(*this);
|
||||
}
|
||||
|
||||
|
||||
string String::to_string() const {
|
||||
return string("#<string '") + value + "'>";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void String::accept(Visitor &visitor) const {
|
||||
visitor.visit(this);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,11 +22,11 @@ namespace tree_sitter {
|
|||
size_t Symbol::hash_code() const {
|
||||
return hash<string>()(name) ^ hash<short int>()(type);
|
||||
}
|
||||
|
||||
|
||||
rule_ptr Symbol::copy() const {
|
||||
return std::make_shared<Symbol>(*this);
|
||||
}
|
||||
|
||||
|
||||
string Symbol::to_string() const {
|
||||
switch (type) {
|
||||
case SymbolTypeNormal:
|
||||
|
|
@ -39,17 +39,17 @@ namespace tree_sitter {
|
|||
return string("#<builtin_sym '") + name + "'>";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
bool Symbol::operator<(const Symbol &other) const {
|
||||
if (type < other.type) return true;
|
||||
if (type > other.type) return false;
|
||||
return (name < other.name);
|
||||
}
|
||||
|
||||
|
||||
bool Symbol::is_built_in() const {
|
||||
return type == SymbolTypeBuiltIn;
|
||||
}
|
||||
|
||||
|
||||
bool Symbol::is_auxiliary() const {
|
||||
return type == SymbolTypeAuxiliary;
|
||||
}
|
||||
|
|
@ -57,7 +57,7 @@ namespace tree_sitter {
|
|||
bool Symbol::is_hidden() const {
|
||||
return (type == SymbolTypeHidden || type == SymbolTypeAuxiliary);
|
||||
}
|
||||
|
||||
|
||||
void Symbol::accept(Visitor &visitor) const {
|
||||
visitor.visit(this);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,8 +11,8 @@ namespace tree_sitter {
|
|||
SymbolTypeHidden,
|
||||
SymbolTypeAuxiliary,
|
||||
SymbolTypeBuiltIn
|
||||
} SymbolType;
|
||||
|
||||
} SymbolType;
|
||||
|
||||
class Symbol : public Rule {
|
||||
public:
|
||||
Symbol(const std::string &name);
|
||||
|
|
@ -20,7 +20,7 @@ namespace tree_sitter {
|
|||
|
||||
bool operator==(const Rule& other) const;
|
||||
bool operator==(const Symbol &other) const;
|
||||
|
||||
|
||||
size_t hash_code() const;
|
||||
rule_ptr copy() const;
|
||||
std::string to_string() const;
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ namespace tree_sitter {
|
|||
class Seq;
|
||||
class String;
|
||||
class Pattern;
|
||||
|
||||
|
||||
class Visitor {
|
||||
public:
|
||||
virtual void default_visit(const Rule *rule);
|
||||
|
|
@ -26,7 +26,7 @@ namespace tree_sitter {
|
|||
virtual void visit(const String *rule);
|
||||
virtual void visit(const Pattern *rule);
|
||||
};
|
||||
|
||||
|
||||
template<typename T>
|
||||
class RuleFn : public Visitor {
|
||||
protected:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue