traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Reduce allocations when computing rule transitions

Browse source
This commit is contained in:
Max Brunsfeld 2014-04-23 09:01:57 -07:00
parent 3b388d66cd
commit e8f2b788d4
4 changed files with 39 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

19
spec/compiler/build_tables/merge_transitions_spec.cc
View file

@ -9,8 +9,11 @@ START_TEST
describe("merging character set transitions", []() {
typedef map<CharacterSet, int> int_map;
auto bitwise = [](int l, int r) -> int {
return l | r;
auto merge_result = [&](int_map left, int_map right) -> int_map {
merge_char_transitions<int>(left, right, [](int l, int r) -> int {
return l | r;
});
return left;
};
describe("when none of the transitions intersect", [&]() {
@ -26,7 +29,7 @@ describe("merging character set transitions", []() {
{ CharacterSet({ '\t' }), 16 },
});
AssertThat(merge_char_transitions<int>(map1, map2, bitwise), Equals(int_map({
AssertThat(merge_result(map1, map2), Equals(int_map({
{ CharacterSet({ 'a', 'c' }), 1 },
{ CharacterSet({ 'x', 'y' }), 2 },
{ CharacterSet({ '1', '9' }), 4 },
@ -34,7 +37,7 @@ describe("merging character set transitions", []() {
{ CharacterSet({ '\t' }), 16 },
})));
AssertThat(merge_char_transitions<int>(map2, map1, bitwise), Equals(merge_char_transitions<int>(map1, map2, bitwise)));
AssertThat(merge_result(map2, map1), Equals(merge_result(map1, map2)));
});
});
@ -50,14 +53,14 @@ describe("merging character set transitions", []() {
{ CharacterSet({ '3' }), 8 },
});
AssertThat(merge_char_transitions<int>(map1, map2, bitwise), Equals(int_map({
AssertThat(merge_result(map1, map2), Equals(int_map({
{ CharacterSet({ {'a', 'b'}, {'d', 'f'}, {'A', 'F'} }), 1 },
{ CharacterSet({ {'c'} }), 5 },
{ CharacterSet({ {'0', '2'}, {'4', '9'} }), 2 },
{ CharacterSet({ '3' }), 10 },
})));
AssertThat(merge_char_transitions<int>(map2, map1, bitwise), Equals(merge_char_transitions<int>(map1, map2, bitwise)));
AssertThat(merge_result(map2, map1), Equals(merge_result(map1, map2)));
});
});
@ -72,12 +75,12 @@ describe("merging character set transitions", []() {
{ CharacterSet({ 'c' }), 4 },
});
AssertThat(merge_char_transitions<int>(map1, map2, bitwise), Equals(int_map({
AssertThat(merge_result(map1, map2), Equals(int_map({
{ CharacterSet({ 'a' }), 3 },
{ CharacterSet({ 'c' }), 5 },
})));
AssertThat(merge_char_transitions<int>(map2, map1, bitwise), Equals(merge_char_transitions<int>(map1, map2, bitwise)));
AssertThat(merge_result(map2, map1), Equals(merge_result(map1, map2)));
});
});
});

12
src/compiler/build_tables/item_set_transitions.cc
View file

@ -45,9 +45,9 @@ namespace tree_sitter {
map<CharacterSet, LexItemSet> result;
for (const LexItem &item : item_set) {
map<CharacterSet, LexItemSet> item_transitions = char_transitions(item);
result = merge_char_transitions<LexItemSet>(result,
item_transitions,
[](LexItemSet left, LexItemSet right) {
merge_char_transitions<LexItemSet>(result,
item_transitions,
[](LexItemSet left, LexItemSet right) {
return merge_sets(left, right);
});
}
@ -59,9 +59,9 @@ namespace tree_sitter {
map<ISymbol, ParseItemSet> result;
for (const ParseItem &item : item_set) {
map<ISymbol, ParseItemSet> item_transitions = sym_transitions(item, grammar);
result = merge_sym_transitions<ParseItemSet>(result,
item_transitions,
[&](ParseItemSet left, ParseItemSet right) {
merge_sym_transitions<ParseItemSet>(result,
item_transitions,
[&](ParseItemSet left, ParseItemSet right) {
return merge_sets(left, right);
});
}

32
src/compiler/build_tables/merge_transitions.h
View file

@ -15,15 +15,13 @@ namespace tree_sitter {
* using the given function.
*/
template<typename T>
std::map<rules::ISymbol, T>
merge_sym_transitions(const std::map<rules::ISymbol, T> &left,
const std::map<rules::ISymbol, T> &right,
std::function<T(T, T)> merge_fn) {
std::map<rules::ISymbol, T> result(left);
void merge_sym_transitions(std::map<rules::ISymbol, T> &left,
const std::map<rules::ISymbol, T> &right,
std::function<T(T, T)> merge_fn) {
for (auto &pair : right) {
auto rule = pair.first;
bool merged = false;
for (auto &existing_pair : result) {
for (auto &existing_pair : left) {
auto existing_rule = existing_pair.first;
if (existing_rule == rule) {
existing_pair.second = merge_fn(existing_pair.second, pair.second);
@ -32,9 +30,8 @@ namespace tree_sitter {
}
}
if (!merged)
result.insert({ pair.first, pair.second });
left.insert({ pair.first, pair.second });
}
return result;
}
/*
@ -44,19 +41,17 @@ namespace tree_sitter {
* merging the two previous values using the given function.
*/
template<typename T>
std::map<rules::CharacterSet, T>
merge_char_transitions(const std::map<rules::CharacterSet, T> &left,
const std::map<rules::CharacterSet, T> &right,
std::function<T(T, T)> merge_fn) {
std::map<rules::CharacterSet, T> result(left);
void merge_char_transitions(std::map<rules::CharacterSet, T> &left,
const std::map<rules::CharacterSet, T> &right,
std::function<T(T, T)> merge_fn) {
for (auto &new_pair : right) {
rules::CharacterSet new_char_set = new_pair.first;
T new_value = new_pair.second;
std::map<rules::CharacterSet, T> pairs_to_insert;
auto iter = result.begin();
while (iter != result.end()) {
auto iter = left.begin();
while (iter != left.end()) {
rules::CharacterSet char_set = iter->first;
T value = iter->second;
@ -66,18 +61,17 @@ namespace tree_sitter {
if (!char_set.is_empty())
pairs_to_insert.insert({ char_set, value });
pairs_to_insert.insert({ intersection, merge_fn(value, new_value) });
result.erase(iter++);
left.erase(iter++);
} else {
++iter;
}
}
result.insert(pairs_to_insert.begin(), pairs_to_insert.end());
left.insert(pairs_to_insert.begin(), pairs_to_insert.end());
if (!new_char_set.is_empty())
result.insert({ new_char_set, new_pair.second });
left.insert({ new_char_set, new_pair.second });
}
return result;
}
}
}

22
src/compiler/build_tables/rule_transitions.cc
View file

@ -24,21 +24,18 @@ namespace tree_sitter {
namespace build_tables {
template<typename T>
map<T, rule_ptr>
merge_transitions(const map<T, rule_ptr> &left, const map<T, rule_ptr> &right);
void merge_transitions(map<T, rule_ptr> &left, const map<T, rule_ptr> &right);
template<>
map<CharacterSet, rule_ptr>
merge_transitions(const map<CharacterSet, rule_ptr> &left, const map<CharacterSet, rule_ptr> &right) {
return merge_char_transitions<rule_ptr>(left, right, [](rule_ptr left, rule_ptr right) {
void merge_transitions(map<CharacterSet, rule_ptr> &left, const map<CharacterSet, rule_ptr> &right) {
merge_char_transitions<rule_ptr>(left, right, [](rule_ptr left, rule_ptr right) {
return make_shared<rules::Choice>(left, right);
});
}
template<>
map<ISymbol, rule_ptr>
merge_transitions(const map<ISymbol, rule_ptr> &left, const map<ISymbol, rule_ptr> &right) {
return merge_sym_transitions<rule_ptr>(left, right, [](rule_ptr left, rule_ptr right) {
void merge_transitions(map<ISymbol, rule_ptr> &left, const map<ISymbol, rule_ptr> &right) {
merge_sym_transitions<rule_ptr>(left, right, [](rule_ptr left, rule_ptr right) {
return make_shared<rules::Choice>(left, right);
});
}
@ -71,9 +68,9 @@ namespace tree_sitter {
}
map<T, rule_ptr> apply_to(const rules::Choice *rule) {
auto left_transitions = this->apply(rule->left);
auto right_transitions = this->apply(rule->right);
return merge_transitions<T>(left_transitions, right_transitions);
auto result = this->apply(rule->left);
merge_transitions<T>(result, this->apply(rule->right));
return result;
}
map<T, rule_ptr> apply_to(const rules::Seq *rule) {
@ -81,8 +78,7 @@ namespace tree_sitter {
return rules::Seq::Build({ left_rule, rule->right });
});
if (rule_can_be_blank(rule->left)) {
auto right_transitions = this->apply(rule->right);
result = merge_transitions<T>(result, right_transitions);
merge_transitions<T>(result, this->apply(rule->right));
}
return result;
}