traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Store rule metadata as a map, not a single number

Browse source 
Need to store more than just boolean values
This commit is contained in:
Max Brunsfeld 2014-04-07 08:50:00 -07:00
parent 610a8e4a29
commit 1da9f1fdfd
12 changed files with 120 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

60
spec/compiler/build_tables/check_metadata_spec.cc
View file

@ -1,60 +0,0 @@
#include "compiler_spec_helper.h"
#include "compiler/rules/metadata.h"
#include "compiler/build_tables/check_metadata.h"
using namespace rules;
using namespace build_tables;
START_TEST
describe("checking if rules have metadata", []() {
MetadataValue value = MetadataValue(1 << 3);
it("returns true for a compatible metadata rule", [&]() {
auto rule = make_shared<Metadata>(sym("x"), MetadataValue(value | 1));
AssertThat(check_metadata(rule, value), IsTrue());
});
it("returns false for an incompatible metadata rule", [&]() {
auto rule = make_shared<Metadata>(sym("x"), MetadataValue(1 << 2));
AssertThat(check_metadata(rule, value), IsFalse());
});
it("returns false for a non-metadata rule", [&]() {
auto rule = sym("x");
AssertThat(check_metadata(rule, value), IsFalse());
});
it("returns true for a compatible metadata rule preceded by rules that can be blank", [&]() {
auto rule = seq({
repeat(sym("x")),
make_shared<Metadata>(sym("x"), MetadataValue(value | 1)),
});
AssertThat(check_metadata(rule, value), IsTrue());
});
it("returns true for a choice including a compatible metadata rule", [&]() {
auto rule = choice({
sym("x"),
make_shared<Metadata>(sym("x"), MetadataValue(value | 1)),
});
AssertThat(check_metadata(rule, value), IsTrue());
});
it("returns true for a repetition containing a compatible metadata rule", [&]() {
auto rule = repeat(make_shared<Metadata>(sym("x"), MetadataValue(value | 1)));
AssertThat(check_metadata(rule, value), IsTrue());
});
it("returns true for a metadata rule preceded by rules that cannot be blank", [&]() {
auto rule = seq({
sym("x"),
make_shared<Metadata>(sym("x"), MetadataValue(value | 1)),
});
AssertThat(check_metadata(rule, value), IsFalse());
});
});
END_TEST

2
spec/compiler/build_tables/first_set_spec.cc
View file

@ -86,7 +86,7 @@ describe("computing FIRST sets", []() {
});
it("ignores metadata rules", [&]() {
auto rule = make_shared<Metadata>(sym("x"), MetadataValue(1));
auto rule = make_shared<Metadata>(sym("x"), map<rules::MetadataKey, int>());
AssertThat(first_set(rule, null_grammar), Equals(set<Symbol>({
Symbol("x"),

81
spec/compiler/build_tables/get_metadata_spec.cc Normal file
View file

@ -0,0 +1,81 @@
#include "compiler_spec_helper.h"
#include "compiler/rules/metadata.h"
#include "compiler/build_tables/get_metadata.h"
using namespace rules;
using namespace build_tables;
START_TEST
describe("getting metadata for rules", []() {
MetadataKey key1 = MetadataKey(100);
MetadataKey key2 = MetadataKey(101);
describe("when given a metadata rule", [&]() {
auto rule = make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 },
{ key2, 2 },
}));
it("returns the value for the given key", [&]() {
AssertThat(get_metadata(rule, key1), Equals(1));
AssertThat(get_metadata(rule, key2), Equals(2));
});
it("returns 0 if the rule does not have the key", [&]() {
AssertThat(get_metadata(rule, MetadataKey(3)), Equals(0));
});
});
describe("when given a non-metadata rule", [&]() {
it("returns 0", [&]() {
auto rule = sym("x");
AssertThat(get_metadata(rule, key1), Equals(0));
});
});
it("works for metadata rules preceded by other rules that can be blank", [&]() {
auto rule = seq({
repeat(sym("x")),
make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 },
{ key2, 2 },
})),
});
AssertThat(get_metadata(rule, key2), Equals(2));
});
it("works for choices containing metadata rule", [&]() {
auto rule = choice({
sym("x"),
make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 },
{ key2, 2 },
})),
});
AssertThat(get_metadata(rule, key2), Equals(1));
});
it("works for repetitions containing metadata rules", [&]() {
auto rule = repeat(make_shared<Metadata>(sym("x"), map<MetadataKey, int>({
{ key1, 1 },
{ key2, 2 },
})));
AssertThat(get_metadata(rule, key2), Equals(2));
});
it("returns 0 for metadata rules preceded by rules that can't be blank", [&]() {
auto rule = seq({
sym("x"),
make_shared<Metadata>(sym("y"), map<MetadataKey, int>({
{ key1, 1 },
{ key2, 2 },
})),
});
AssertThat(get_metadata(rule, key2), Equals(0));
});
});
END_TEST

4
spec/compiler/build_tables/rule_can_be_blank_spec.cc
View file

@ -48,10 +48,10 @@ describe("checking if rules can be blank", [&]() {
});
it("ignores metadata rules", [&]() {
rule = make_shared<rules::Metadata>(blank(), rules::MetadataValue(0));
rule = make_shared<rules::Metadata>(blank(), map<rules::MetadataKey, int>());
AssertThat(rule_can_be_blank(rule), IsTrue());
rule = make_shared<rules::Metadata>(sym("one"), rules::MetadataValue(0));
rule = make_shared<rules::Metadata>(sym("one"), map<rules::MetadataKey, int>());
AssertThat(rule_can_be_blank(rule), IsFalse());
});

5
src/compiler/build_tables/build_tables.cc
View file

@ -15,6 +15,7 @@
namespace tree_sitter {
using std::pair;
using std::string;
using std::map;
using std::unordered_map;
using std::make_shared;
using rules::Symbol;
@ -50,7 +51,7 @@ namespace tree_sitter {
void add_token_start(const LexItemSet &item_set, LexStateId state_id) {
for (auto &item : item_set)
if (item.has_metadata(rules::START_TOKEN))
if (item.get_metadata(rules::START_TOKEN))
lex_table.state(state_id).is_token_start = true;
}
@ -82,7 +83,7 @@ namespace tree_sitter {
rules::rule_ptr after_separators(rules::rule_ptr rule) {
return rules::Seq::Build({
make_shared<rules::Repeat>(CharacterSet({ ' ', '\t', '\n', '\r' }).copy()),
make_shared<rules::Metadata>(rule, rules::START_TOKEN)
make_shared<rules::Metadata>(rule, map<rules::MetadataKey, int>({ {rules::START_TOKEN, 1} }))
});
}

23
src/compiler/build_tables/check_metadata.cc → src/compiler/build_tables/get_metadata.cc
View file

@ -1,4 +1,4 @@
#include "check_metadata.h"
#include "get_metadata.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/choice.h"
#include "compiler/rules/repeat.h"
@ -7,10 +7,11 @@
namespace tree_sitter {
namespace build_tables {
class HasMetadata : public rules::RuleFn<bool> {
rules::MetadataValue metadata_value;
class GetMetadata : public rules::RuleFn<int> {
rules::MetadataKey metadata_key;
public:
HasMetadata(rules::MetadataValue value) : metadata_value(value) {}
GetMetadata(rules::MetadataKey key) :
metadata_key(key) {}
void visit(const rules::Choice *rule) {
value = apply(rule->left) || apply(rule->right);
@ -21,19 +22,21 @@ namespace tree_sitter {
}
void visit(const rules::Seq *rule) {
bool result = apply(rule->left);
if (rule_can_be_blank(rule->left))
result = result || apply(rule->right);
int result = apply(rule->left);
if (rule_can_be_blank(rule->left) && result == 0)
result = apply(rule->right);
value = result;
}
void visit(const rules::Metadata *rule) {
value = rule->value & metadata_value;
auto pair = rule->value.find(metadata_key);
if (pair != rule->value.end())
value = pair->second;
}
};
bool check_metadata(const rules::rule_ptr &rule, rules::MetadataValue value) {
return HasMetadata(value).apply(rule);
int get_metadata(const rules::rule_ptr &rule, rules::MetadataKey key) {
return GetMetadata(key).apply(rule);
}
}
}

2
src/compiler/build_tables/check_metadata.h → src/compiler/build_tables/get_metadata.h
View file

@ -6,7 +6,7 @@
namespace tree_sitter {
namespace build_tables {
bool check_metadata(const rules::rule_ptr &rule, rules::MetadataValue value);
int get_metadata(const rules::rule_ptr &rule, rules::MetadataKey key);
}
}

6
src/compiler/build_tables/item.cc
View file

@ -1,6 +1,6 @@
#include "compiler/build_tables/item.h"
#include "compiler/build_tables/rule_can_be_blank.h"
#include "compiler/build_tables/check_metadata.h"
#include "compiler/build_tables/get_metadata.h"
#include "tree_sitter/compiler.h"
namespace tree_sitter {
@ -20,8 +20,8 @@ namespace tree_sitter {
return rule_can_be_blank(rule);
}
bool Item::has_metadata(rules::MetadataValue value) const {
return check_metadata(rule, value);
int Item::get_metadata(rules::MetadataKey key) const {
return build_tables::get_metadata(rule, key);
}
ostream& operator<<(ostream &stream, const LexItem &item) {

2
src/compiler/build_tables/item.h
View file

@ -15,7 +15,7 @@ namespace tree_sitter {
public:
Item(const rules::Symbol &lhs, rules::rule_ptr rule);
bool is_done() const;
bool has_metadata(rules::MetadataValue) const;
int get_metadata(rules::MetadataKey) const;
rules::Symbol lhs;
rules::rule_ptr rule;

2
src/compiler/build_tables/item_set_closure.cc
View file

@ -25,7 +25,7 @@ namespace tree_sitter {
Symbol non_terminal = pair.first;
set<Symbol> terminals = pair.second;
for (auto &terminal : terminals) {
ParseItem next_item(non_terminal, grammar.rule(non_terminal), {}, terminal);
ParseItem next_item(non_terminal, grammar.rule(non_terminal), 0, terminal);
add_item(item_set, next_item, grammar);
}
}

10
src/compiler/rules/metadata.cc
View file

@ -6,9 +6,10 @@
namespace tree_sitter {
using std::hash;
using std::make_shared;
using std::map;
namespace rules {
Metadata::Metadata(rule_ptr rule, MetadataValue value) : rule(rule), value(value) {}
Metadata::Metadata(rule_ptr rule, map<MetadataKey, int> values) : rule(rule), value(values) {}
bool Metadata::operator==(const Rule &rule) const {
auto other = dynamic_cast<const Metadata *>(&rule);
@ -16,7 +17,12 @@ namespace tree_sitter {
}
size_t Metadata::hash_code() const {
return hash<int>()(value);
size_t result = hash<size_t>()(value.size());
for (auto &pair : value) {
result ^= hash<int>()(pair.first);
result ^= hash<int>()(pair.second);
}
return result;
}
rule_ptr Metadata::copy() const {

11
src/compiler/rules/metadata.h
View file

@ -2,18 +2,19 @@
#define COMPILER_RULES_METADATA_H_
#include <string>
#include <map>
#include "compiler/rules/rule.h"
namespace tree_sitter {
namespace rules {
typedef enum {
NONE = 0,
START_TOKEN = 1,
} MetadataValue;
START_TOKEN,
PRECEDENCE
} MetadataKey;
class Metadata : public Rule {
public:
Metadata(rule_ptr rule, MetadataValue value);
Metadata(rule_ptr rule, std::map<MetadataKey, int> value);
bool operator==(const Rule& other) const;
size_t hash_code() const;
@ -22,7 +23,7 @@ namespace tree_sitter {
void accept(Visitor *visitor) const;
const rule_ptr rule;
const MetadataValue value;
const std::map<MetadataKey, int> value;
};
}
}