traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge equivalent stacks in a separate stage of parsing

Browse source 
* No more automatic merging every time a state is pushed to the stack
* When popping from the stack, the current version is always preserved
This commit is contained in:
Max Brunsfeld 2016-04-10 14:12:24 -07:00
parent 827573f1c7
commit 695be5bc79
7 changed files with 582 additions and 755 deletions
Download patch file Download diff file Expand all files Collapse all files

742
spec/runtime/stack_spec.cc
View file

@ -93,7 +93,6 @@ describe("Stack", [&]() {
TSTree *trees[tree_count];
TreeSelectionSpy tree_selection_spy;
TSLength tree_len = {2, 3, 0, 3};
TSSymbolMetadata metadata = {true, true, true, true};
before_each([&]() {
record_alloc::start();
@ -120,9 +119,9 @@ describe("Stack", [&]() {
AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty());
});
describe("pushing entries to the stack", [&]() {
it("adds entries to the stack", [&]() {
AssertThat(ts_stack_version_count(stack), Equals(1));
describe("push(version, tree, is_pending, state)", [&]() {
it("adds entries to the given version of the stack", [&]() {
AssertThat(ts_stack_version_count(stack), Equals<size_t>(1));
AssertThat(ts_stack_top_state(stack, 0), Equals(0));
AssertThat(ts_stack_top_position(stack, 0), Equals(ts_length_zero()));
@ -150,473 +149,382 @@ describe("Stack", [&]() {
});
});
describe("popping nodes from the stack", [&]() {
describe("merge()", [&]() {
before_each([&]() {
// . <──0── A <──1── B <──2── C*
// . <──0── A <──1── B*
// ↑
// └───2─── C*
ts_stack_push(stack, 0, trees[0], false, stateA);
ts_stack_pop_count(stack, 0, 0);
ts_stack_push(stack, 0, trees[1], false, stateB);
ts_stack_push(stack, 0, trees[2], false, stateC);
ts_stack_push(stack, 1, trees[2], false, stateC);
});
it("removes the given number of nodes from the stack", [&]() {
// . <──0── A*
StackPopResult pop_result = ts_stack_pop_count(stack, 0, 2);
AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop_result.slices.size, Equals<size_t>(1));
it("combines versions that have the same top states and positions", [&]() {
// . <──0── A <──1── B <──3── D*
// ↑
// └───2─── C <──4── D*
ts_stack_push(stack, 0, trees[3], false, stateD);
ts_stack_push(stack, 1, trees[4], false, stateD);
StackSlice slice = pop_result.slices.contents[0];
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[1], trees[2] })));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateA));
free_slice_array(&pop_result.slices);
// .*
pop_result = ts_stack_pop_count(stack, 0, 1);
AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop_result.slices.size, Equals<size_t>(1));
slice = pop_result.slices.contents[0];
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[0] })));
AssertThat(ts_stack_top_state(stack, 0), Equals(0));
free_slice_array(&pop_result.slices);
// . <──0── A <──1── B <──3── D*
// ↑ |
// └───2─── C <──4───┘
ts_stack_merge(stack);
AssertThat(ts_stack_version_count(stack), Equals<size_t>(1));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateD, 0},
{stateB, 1},
{stateC, 1},
{stateA, 2},
{0, 3},
})));
});
it("does not count 'extra' trees toward the count", [&]() {
trees[1]->extra = true;
// .*
StackPopResult pop_result = ts_stack_pop_count(stack, 0, 2);
AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice = pop_result.slices.contents[0];
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[0], trees[1], trees[2] })));
AssertThat(ts_stack_top_state(stack, 0), Equals(0));
free_slice_array(&pop_result.slices);
it("does not combine versions that have different states", [&]() {
ts_stack_merge(stack);
AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
});
it("pops the entire stack when given a negative count", [&]() {
// .*
StackPopResult pop_result = ts_stack_pop_count(stack, 0, -1);
AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop_result.slices.size, Equals<size_t>(1));
it("does not combine versions that have different positions", [&]() {
// . <──0── A <──1── B <────3──── D*
// ↑
// └───2─── C <──4── D*
trees[3]->size = tree_len * 3;
ts_stack_push(stack, 0, trees[3], false, stateD);
ts_stack_push(stack, 1, trees[4], false, stateD);
StackSlice slice = pop_result.slices.contents[0];
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[0], trees[1], trees[2] })));
free_slice_array(&pop_result.slices);
ts_stack_merge(stack);
AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
});
describe("when an error state exists above the given depth", [&]() {
it("stops popping nodes at the error", [&]() {
// . <──0── A <──1── B <──2── C <──3── ERROR <──4── D*
ts_stack_push(stack, 0, trees[3], false, ts_parse_state_error);
ts_stack_push(stack, 0, trees[4], false, stateD);
StackPopResult pop_result = ts_stack_pop_count(stack, 0, 3);
AssertThat(pop_result.status, Equals(StackPopResult::StackPopStoppedAtError));
AssertThat(ts_stack_version_count(stack), Equals(1));
AssertThat(ts_stack_top_state(stack, 0), Equals(ts_parse_state_error));
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice = pop_result.slices.contents[0];
AssertThat(slice.version, Equals(0));
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[4] })));
free_slice_array(&pop_result.slices);
});
});
describe("popping pending nodes from the stack", [&]() {
it("removes the top node from the stack if it was pushed in pending mode", [&]() {
ts_stack_push(stack, 0, trees[3], true, stateD);
StackPopResult pop = ts_stack_pop_pending(stack, 0);
AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop.slices.size, Equals<size_t>(1));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateC, 0},
{stateB, 1},
{stateA, 2},
{0, 3},
})));
free_slice_array(&pop.slices);
});
it("does nothing if the top node was not pushed in pending mode", [&]() {
describe("when the merged versions have more than one common entry", [&]() {
it("combines all of the top common entries", [&]() {
// . <──0── A <──1── B <──3── D <──5── E*
// ↑
// └───2─── C <──4── D <──5── E*
ts_stack_push(stack, 0, trees[3], false, stateD);
ts_stack_push(stack, 0, trees[5], false, stateE);
ts_stack_push(stack, 1, trees[4], false, stateD);
ts_stack_push(stack, 1, trees[5], false, stateE);
StackPopResult pop = ts_stack_pop_pending(stack, 0);
AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop.slices.size, Equals<size_t>(0));
// . <──0── A <──1── B <──3── D <──5── E*
// ↑ |
// └───2─── C <──4───┘
ts_stack_merge(stack);
AssertThat(ts_stack_version_count(stack), Equals<size_t>(1));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateD, 0},
{stateC, 1},
{stateE, 0},
{stateD, 1},
{stateB, 2},
{stateC, 2},
{stateA, 3},
{0, 4},
})));
});
});
});
describe("pop_count(version, count)", [&]() {
before_each([&]() {
// . <──0── A <──1── B <──2── C*
ts_stack_push(stack, 0, trees[0], false, stateA);
ts_stack_push(stack, 0, trees[1], false, stateB);
ts_stack_push(stack, 0, trees[2], false, stateC);
});
it("creates a new version with the given number of entries removed", [&]() {
// . <──0── A <──1── B <──2── C*
// ↑
// └─*
StackPopResult pop = ts_stack_pop_count(stack, 0, 2);
AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop.slices.size, Equals<size_t>(1));
AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
StackSlice slice = pop.slices.contents[0];
AssertThat(slice.version, Equals<StackVersion>(1));
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[1], trees[2] })));
AssertThat(ts_stack_top_state(stack, 1), Equals(stateA));
free_slice_array(&pop.slices);
});
it("does not count 'extra' trees toward the given count", [&]() {
trees[1]->extra = true;
// . <──0── A <──1── B <──2── C*
// ↑
// └─*
StackPopResult pop = ts_stack_pop_count(stack, 0, 2);
AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop.slices.size, Equals<size_t>(1));
StackSlice slice = pop.slices.contents[0];
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[0], trees[1], trees[2] })));
AssertThat(ts_stack_top_state(stack, 1), Equals(0));
free_slice_array(&pop.slices);
});
it("stops popping entries early if it reaches an error tree", [&]() {
// . <──0── A <──1── B <──2── C <──3── ERROR <──4── D*
ts_stack_push(stack, 0, trees[3], false, ts_parse_state_error);
ts_stack_push(stack, 0, trees[4], false, stateD);
// . <──0── A <──1── B <──2── C <──3── ERROR <──4── D*
// ↑
// └─*
StackPopResult pop = ts_stack_pop_count(stack, 0, 3);
AssertThat(pop.status, Equals(StackPopResult::StackPopStoppedAtError));
AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
AssertThat(ts_stack_top_state(stack, 1), Equals(ts_parse_state_error));
AssertThat(pop.slices.size, Equals<size_t>(1));
StackSlice slice = pop.slices.contents[0];
AssertThat(slice.version, Equals<StackVersion>(1));
AssertThat(slice.trees, Equals(vector<TSTree *>({ trees[4] })));
free_slice_array(&pop.slices);
});
describe("when the version has been merged", [&]() {
before_each([&]() {
// . <──0── A <──1── B <──2── C <──3── D <──10── I*
// ↑ |
// └───4─── E <──5── F <──6───┘
ts_stack_push(stack, 0, trees[3], false, stateD);
StackPopResult pop = ts_stack_pop_count(stack, 0, 3);
free_slice_array(&pop.slices);
});
});
});
ts_stack_push(stack, 1, trees[4], false, stateE);
ts_stack_push(stack, 1, trees[5], false, stateF);
ts_stack_push(stack, 1, trees[6], false, stateD);
ts_stack_merge(stack);
ts_stack_push(stack, 0, trees[10], false, stateI);
describe("splitting the stack", [&]() {
it("creates a new independent version with the same entries", [&]() {
// . <──0── A <──1── B <──2── C*
ts_stack_push(stack, 0, trees[0], false, stateA);
ts_stack_push(stack, 0, trees[1], false, stateB);
ts_stack_push(stack, 0, trees[2], false, stateC);
// . <──0── A <──1── B <──2── C*
// ↑
// └─*
int new_index = ts_stack_split(stack, 0);
AssertThat(ts_stack_version_count(stack), Equals(2));
AssertThat(new_index, Equals(1));
AssertThat(ts_stack_top_state(stack, 1), Equals(stateC));
// . <──0── A <──1── B <──2── C <──3── D*
// ↑
// └─*
ts_stack_push(stack, 0, trees[3], false, stateD);
StackPopResult pop_result = ts_stack_pop_count(stack, 1, 1);
AssertThat(ts_stack_version_count(stack), Equals(2));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateD));
AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len * 4));
AssertThat(ts_stack_top_state(stack, 1), Equals(stateB));
AssertThat(ts_stack_top_position(stack, 1), Equals(tree_len * 2));
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice = pop_result.slices.contents[0];
AssertThat(slice.trees.size, Equals<size_t>(1));
free_slice_array(&pop_result.slices);
// . <──0── A <──1── B <──2── C <──3── D*
// ↑
// └───4─── E <──5── F*
ts_stack_push(stack, 1, trees[4], false, stateE);
ts_stack_push(stack, 1, trees[5], false, stateF);
AssertThat(ts_stack_version_count(stack), Equals(2));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateD));
AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len * 4));
AssertThat(ts_stack_top_state(stack, 1), Equals(stateF));
AssertThat(ts_stack_top_position(stack, 1), Equals(tree_len * 4));
});
});
describe("pushing the same state onto two different versions of the stack", [&]() {
before_each([&]() {
// . <──0── A <──1── B <──2── C <──3── D*
// ↑
// └───4─── E <──5── F*
ts_stack_push(stack, 0, trees[0], false, stateA);
ts_stack_push(stack, 0, trees[1], false, stateB);
ts_stack_split(stack, 0);
ts_stack_push(stack, 0, trees[2], false, stateC);
ts_stack_push(stack, 0, trees[3], false, stateD);
ts_stack_push(stack, 1, trees[4], false, stateE);
ts_stack_push(stack, 1, trees[5], false, stateF);
AssertThat(ts_stack_version_count(stack), Equals(2));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateD, 0},
{stateC, 1},
{stateB, 2},
{stateA, 3},
{0, 4},
})));
AssertThat(get_stack_entries(stack, 1), Equals(vector<StackEntry>({
{stateF, 0},
{stateE, 1},
{stateB, 2},
{stateA, 3},
{0, 4},
})));
});
it("merges the versions", [&]() {
// . <──0── A <──1── B <──2── C <──3── D <──6── G*
// ↑ |
// └───4─── E <──5── F <──7───┘
AssertThat(ts_stack_push(stack, 0, trees[6], false, stateG), Equals(StackPushContinued));
AssertThat(ts_stack_push(stack, 1, trees[7], false, stateG), Equals(StackPushMerged));
AssertThat(ts_stack_version_count(stack), Equals(1));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateG, 0},
{stateD, 1},
{stateF, 1},
{stateC, 2},
{stateE, 2},
{stateB, 3},
{stateA, 4},
{0, 5},
})));
});
describe("when the merged nodes share a successor", [&]() {
it("recursively merges the successor nodes", [&]() {
// . <──0── A <──1── B <──2── C <──3── D <──6── G <──7──H*
// ↑
// └───4─── E <──5── F <──8── G*
AssertThat(ts_stack_push(stack, 0, trees[6], false, stateG), Equals(StackPushContinued));
AssertThat(ts_stack_push(stack, 0, trees[7], false, stateH), Equals(StackPushContinued));
AssertThat(ts_stack_push(stack, 1, trees[6], false, stateG), Equals(StackPushContinued));
// . <──0── A <──1── B <──2── C <──3── D <──6── G <──7──H*
// ↑ |
// └───4─── E <──5── F <──8───┘
AssertThat(ts_stack_push(stack, 1, trees[7], false, stateH), Equals(StackPushMerged));
AssertThat(ts_stack_version_count(stack), Equals(1));
AssertThat(ts_stack_version_count(stack), Equals<size_t>(1));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateH, 0},
{stateG, 1},
{stateD, 2},
{stateI, 0},
{stateD, 1},
{stateC, 2},
{stateF, 2},
{stateC, 3},
{stateB, 3},
{stateE, 3},
{stateB, 4},
{stateA, 5},
{0, 6},
{stateA, 4},
{0, 5},
})));
});
});
describe("when the first version is only one node deep", [&]() {
it("creates a node with one null successor and one non-null successor", [&]() {
ts_tree_retain(trees[2]);
ts_tree_retain(trees[3]);
TSTree *parent = ts_tree_make_node(5, 2, tree_array({ trees[2], trees[3] }), metadata);
describe("when there are two paths that reveal different versions", [&]() {
it("returns an entry for each revealed version", [&]() {
// . <──0── A <──1── B <──2── C <──3── D <──10── I*
// ↑ ↑
// | └*
// |
// └───4─── E*
StackPopResult pop = ts_stack_pop_count(stack, 0, 3);
AssertThat(pop.slices.size, Equals<size_t>(2));
// . <──────5─────── C*
// ↑ |
// └───2─── B ───3───┘
ts_stack_clear(stack);
ts_stack_split(stack, 0);
AssertThat(ts_stack_push(stack, 0, parent, false, stateC), Equals(StackPushContinued));
AssertThat(ts_stack_push(stack, 1, trees[2], false, stateB), Equals(StackPushContinued));
AssertThat(ts_stack_push(stack, 1, trees[3], false, stateC), Equals(StackPushMerged));
StackSlice slice1 = pop.slices.contents[0];
AssertThat(slice1.version, Equals<StackVersion>(1));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[2], trees[3], trees[10] })));
AssertThat(ts_stack_version_count(stack), Equals(1));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateC));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateC, 0},
{0, 1},
{stateB, 1},
{0, 2},
})));
StackSlice slice2 = pop.slices.contents[1];
AssertThat(slice2.version, Equals<StackVersion>(2));
AssertThat(slice2.trees, Equals(vector<TSTree *>({ trees[5], trees[6], trees[10] })));
ts_tree_release(parent);
});
});
});
AssertThat(ts_stack_version_count(stack), Equals<size_t>(3));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateI, 0},
{stateD, 1},
{stateC, 2},
{stateF, 2},
{stateB, 3},
{stateE, 3},
{stateA, 4},
{0, 5},
})));
AssertThat(get_stack_entries(stack, 1), Equals(vector<StackEntry>({
{stateB, 0},
{stateA, 1},
{0, 2},
})));
AssertThat(get_stack_entries(stack, 2), Equals(vector<StackEntry>({
{stateE, 0},
{stateA, 1},
{0, 2},
})));
describe("popping from a stack version that has been merged", [&]() {
before_each([&]() {
// . <──0── A <──1── B <──2── C <──3── D <──4── E*
// ↑ |
// └───5─── F <──6── G <──7───┘
ts_stack_push(stack, 0, trees[0], false, stateA);
ts_stack_push(stack, 0, trees[1], false, stateB);
ts_stack_split(stack, 0);
ts_stack_push(stack, 0, trees[2], false, stateC);
ts_stack_push(stack, 0, trees[3], false, stateD);
ts_stack_push(stack, 0, trees[4], false, stateE);
ts_stack_push(stack, 1, trees[5], false, stateF);
ts_stack_push(stack, 1, trees[6], false, stateG);
ts_stack_push(stack, 1, trees[7], false, stateE);
AssertThat(ts_stack_version_count(stack), Equals(1));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateE, 0},
{stateD, 1},
{stateG, 1},
{stateC, 2},
{stateF, 2},
{stateB, 3},
{stateA, 4},
{0, 5},
})));
});
describe("when there are two paths that lead to two different versions", [&]() {
it("returns an entry for each revealed version", [&]() {
// . <──0── A <──1── B <──2── C*
// ↑
// └───5─── F*
StackPopResult pop_result = ts_stack_pop_count(stack, 0, 2);
AssertThat(pop_result.slices.size, Equals<size_t>(2));
StackSlice slice1 = pop_result.slices.contents[0];
AssertThat(slice1.version, Equals(0));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateC));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[3], trees[4] })));
StackSlice slice2 = pop_result.slices.contents[1];
AssertThat(slice2.version, Equals(1));
AssertThat(ts_stack_top_state(stack, 1), Equals(stateF));
AssertThat(slice2.trees, Equals(vector<TSTree *>({ trees[6], trees[7] })));
AssertThat(ts_stack_version_count(stack), Equals(2));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateC, 0},
{stateB, 1},
{stateA, 2},
{0, 3},
})));
AssertThat(get_stack_entries(stack, 1), Equals(vector<StackEntry>({
{stateF, 0},
{stateB, 1},
{stateA, 2},
{0, 3},
})));
free_slice_array(&pop_result.slices);
});
});
describe("when there is one path, leading to one version", [&]() {
it("returns a single entry", [&]() {
// . <──0── A <──1── B <──2── C <──3── D <──4── E <──8──H*
// ↑ |
// └───5─── F <──6── G <──7───┘
AssertThat(ts_stack_push(stack, 0, trees[8], false, stateH), Equals(StackPushContinued));
AssertThat(ts_stack_version_count(stack), Equals(1));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateH));
// . <──0── A <──1── B <──2── C <──3── D <──4── E*
// ↑ |
// └───5─── F <──6── G <──7───┘
StackPopResult pop_result = ts_stack_pop_count(stack, 0, 1);
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice1 = pop_result.slices.contents[0];
AssertThat(slice1.version, Equals(0));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[8] })));
AssertThat(ts_stack_version_count(stack), Equals(1));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateE));
free_slice_array(&pop_result.slices);
});
});
describe("when there are two paths that converge at the same version", [&]() {
describe("when the first path is preferred by the callback", [&]() {
it("returns one entry for that version, with the first path of trees", [&]() {
tree_selection_spy.tree_to_return = trees[2];
// . <──0── A <──1── B*
StackPopResult pop_result = ts_stack_pop_count(stack, 0, 3);
AssertThat(ts_stack_version_count(stack), Equals(1));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateB));
AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len * 2));
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice1 = pop_result.slices.contents[0];
AssertThat(slice1.version, Equals(0));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[2], trees[3], trees[4] })));
free_slice_array(&pop_result.slices);
free_slice_array(&pop.slices);
});
});
describe("when the second path is preferred by the callback", [&]() {
it("returns one entry for that version, with the second path of trees", [&]() {
describe("when there is one path that ends at a merged version", [&]() {
it("returns a single entry", [&]() {
// . <──0── A <──1── B <──2── C <──3── D <──10── I*
// | |
// └───5─── F <──6── G <──7───┘
// |
// └*
StackPopResult pop = ts_stack_pop_count(stack, 0, 1);
AssertThat(pop.slices.size, Equals<size_t>(1));
StackSlice slice1 = pop.slices.contents[0];
AssertThat(slice1.version, Equals<StackVersion>(1));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[10] })));
AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateI));
AssertThat(ts_stack_top_state(stack, 1), Equals(stateD));
free_slice_array(&pop.slices);
});
});
describe("when there are two paths that converge on one version", [&]() {
it("returns the first path of trees if they are selected by the selection callback", [&]() {
tree_selection_spy.tree_to_return = trees[1];
// . <──0── A <──1── B <──2── C <──3── D <──10── I*
// ↑ |
// ├───4─── E <──5── F <──6───┘
// |
// └*
StackPopResult pop = ts_stack_pop_count(stack, 0, 4);
AssertThat(pop.slices.size, Equals<size_t>(1));
StackSlice slice1 = pop.slices.contents[0];
AssertThat(slice1.version, Equals<StackVersion>(1));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[1], trees[2], trees[3], trees[10] })));
AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateI));
AssertThat(ts_stack_top_state(stack, 1), Equals(stateA));
free_slice_array(&pop.slices);
});
it("returns the second path of trees if they are selected by the selection callback", [&]() {
tree_selection_spy.tree_to_return = trees[4];
// . <──0── A <──1── B*
StackPopResult pop_result = ts_stack_pop_count(stack, 0, 3);
AssertThat(ts_stack_version_count(stack), Equals(1));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateB));
AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len * 2));
// . <──0── A <──1── B <──2── C <──3── D <──10── I*
// ↑ |
// ├───4─── E <──5── F <──6───┘
// |
// └*
StackPopResult pop = ts_stack_pop_count(stack, 0, 4);
AssertThat(pop.slices.size, Equals<size_t>(1));
AssertThat(pop_result.slices.size, Equals<size_t>(1));
StackSlice slice1 = pop_result.slices.contents[0];
AssertThat(slice1.version, Equals(0));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[5], trees[6], trees[7] })))
StackSlice slice1 = pop.slices.contents[0];
AssertThat(slice1.version, Equals<StackVersion>(1));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[4], trees[5], trees[6], trees[10] })))
free_slice_array(&pop_result.slices);
AssertThat(ts_stack_version_count(stack), Equals<size_t>(2));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateI));
AssertThat(ts_stack_top_state(stack, 1), Equals(stateA));
free_slice_array(&pop.slices);
});
});
describe("when there are three paths that lead to three different versions", [&]() {
it("returns three entries with different arrays of trees", [&]() {
// . <──0── A <──1── B <──2── C <──3── D <──10── I*
// ↑ |
// ├───4─── E <──5── F <──6───┘
// | |
// └───7─── G <──8── H <──9───┘
StackPopResult pop = ts_stack_pop_count(stack, 0, 4);
free_slice_array(&pop.slices);
ts_stack_push(stack, 1, trees[7], false, stateG);
ts_stack_push(stack, 1, trees[8], false, stateH);
ts_stack_push(stack, 1, trees[9], false, stateD);
ts_stack_push(stack, 1, trees[10], false, stateI);
ts_stack_merge(stack);
AssertThat(ts_stack_version_count(stack), Equals<size_t>(1));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateI, 0},
{stateD, 1},
{stateC, 2},
{stateF, 2},
{stateH, 2},
{stateB, 3},
{stateE, 3},
{stateG, 3},
{stateA, 4},
{0, 5},
})));
// . <──0── A <──1── B <──2── C <──3── D <──10── I*
// ↑ ↑
// | └*
// |
// ├───4─── E <──5── F*
// |
// └───7─── G <──8── H*
pop = ts_stack_pop_count(stack, 0, 2);
AssertThat(pop.slices.size, Equals<size_t>(3));
StackSlice slice1 = pop.slices.contents[0];
AssertThat(slice1.version, Equals<StackVersion>(1));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[3], trees[10] })))
StackSlice slice2 = pop.slices.contents[1];
AssertThat(slice2.version, Equals<StackVersion>(2));
AssertThat(slice2.trees, Equals(vector<TSTree *>({ trees[6], trees[10] })))
StackSlice slice3 = pop.slices.contents[2];
AssertThat(slice3.version, Equals<StackVersion>(3));
AssertThat(slice3.trees, Equals(vector<TSTree *>({ trees[9], trees[10] })))
AssertThat(ts_stack_version_count(stack), Equals<size_t>(4));
AssertThat(ts_stack_top_state(stack, 0), Equals(stateI));
AssertThat(ts_stack_top_state(stack, 1), Equals(stateC));
AssertThat(ts_stack_top_state(stack, 2), Equals(stateF));
AssertThat(ts_stack_top_state(stack, 3), Equals(stateH));
free_slice_array(&pop.slices);
});
});
});
});
describe("popping from a stack version that has been 3-way merged", [&]() {
describe("pop_pending(version)", [&]() {
before_each([&]() {
// . <──0── A <──1── B <──2── C <──3── D <──10── I
// ↑ |
// ├───4─── E <──5── F <──6───┤
// | |
// └───7─── G <──8── H <──9───┘
ts_stack_clear(stack);
ts_stack_push(stack, 0, trees[0], false, stateA);
ts_stack_split(stack, 0);
ts_stack_split(stack, 1);
ts_stack_push(stack, 0, trees[1], false, stateB);
ts_stack_push(stack, 0, trees[2], false, stateC);
ts_stack_push(stack, 0, trees[3], false, stateD);
ts_stack_push(stack, 1, trees[4], false, stateE);
ts_stack_push(stack, 1, trees[5], false, stateF);
ts_stack_push(stack, 1, trees[6], false, stateD);
ts_stack_push(stack, 1, trees[7], false, stateG);
ts_stack_push(stack, 1, trees[8], false, stateH);
ts_stack_push(stack, 1, trees[9], false, stateD);
ts_stack_push(stack, 0, trees[10], false, stateI);
AssertThat(ts_stack_version_count(stack), Equals(1));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateI, 0},
{stateD, 1},
{stateC, 2},
{stateF, 2},
{stateH, 2},
{stateB, 3},
{stateE, 3},
{stateG, 3},
{stateA, 4},
{0, 5},
})));
});
describe("when there are three different paths that lead to three different versions", [&]() {
it("returns three entries with different arrays of trees", [&]() {
// . <──0── A <──1── B <──2── C*
// ↑
// ├───4─── E <──5── F*
// |
// └───7─── G <──8── H*
StackPopResult pop_result = ts_stack_pop_count(stack, 0, 2);
AssertThat(ts_stack_version_count(stack), Equals(3));
it("removes the top node from the stack if it was pushed in pending mode", [&]() {
ts_stack_push(stack, 0, trees[1], true, stateB);
AssertThat(pop_result.slices.size, Equals<size_t>(3));
StackPopResult pop = ts_stack_pop_pending(stack, 0);
AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop.slices.size, Equals<size_t>(1));
StackSlice slice1 = pop_result.slices.contents[0];
AssertThat(ts_stack_top_state(stack, 0), Equals(stateC));
AssertThat(slice1.version, Equals(0));
AssertThat(slice1.trees, Equals(vector<TSTree *>({ trees[3], trees[10] })))
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateA, 0},
{0, 1},
})));
StackSlice slice2 = pop_result.slices.contents[1];
AssertThat(ts_stack_top_state(stack, 1), Equals(stateF));
AssertThat(slice2.version, Equals(1));
AssertThat(slice2.trees, Equals(vector<TSTree *>({ trees[6], trees[10] })))
free_slice_array(&pop.slices);
});
StackSlice slice3 = pop_result.slices.contents[2];
AssertThat(ts_stack_top_state(stack, 2), Equals(stateH));
AssertThat(slice3.version, Equals(2));
AssertThat(slice3.trees, Equals(vector<TSTree *>({ trees[9], trees[10] })))
it("does nothing if the top node was not pushed in pending mode", [&]() {
ts_stack_push(stack, 0, trees[1], false, stateB);
free_slice_array(&pop_result.slices);
});
StackPopResult pop = ts_stack_pop_pending(stack, 0);
AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded));
AssertThat(pop.slices.size, Equals<size_t>(0));
AssertThat(get_stack_entries(stack, 0), Equals(vector<StackEntry>({
{stateB, 0},
{stateA, 1},
{0, 2},
})));
free_slice_array(&pop.slices);
});
});
});

2
src/runtime/array.h
View file

@ -43,7 +43,7 @@ extern "C" {
#define array_push(self, element) \
(((self)->size < (self)->capacity || \
array_grow((self), (self)->capacity ? (self)->capacity * 2 : 4)) && \
array_grow((self), (self)->capacity ? (self)->capacity * 2 : 8)) && \
((self)->contents[(self)->size++] = (element), true))
#define array_splice(self, index, old_count, new_count, new_elements) \

396
src/runtime/parser.c
View file

@ -37,6 +37,15 @@
#define BOOL_STRING(value) (value ? "true" : "false")
#define CHECK(expr) \
if (!(expr)) { \
goto error; \
}
static const TSParseAction ERROR_ACTION = {.type = TSParseActionTypeError };
static const size_t NO_ERROR_DEPTH = (size_t)(-1);
typedef struct {
TSTree *tree;
size_t char_index;
@ -67,17 +76,15 @@ typedef struct {
enum {
ReduceFailed,
ReduceSucceeded,
ReduceMerged,
ReduceStoppedAtError,
} status;
StackSlice partial_slice;
StackSlice slice;
} ReduceResult;
typedef enum {
RepairFailed,
RepairSucceeded,
RepairMerged,
RepairNoneFound,
} RepairResult;
@ -109,7 +116,6 @@ static BreakdownResult ts_parser__breakdown_top_of_stack(TSParser *self,
LOG("breakdown_pop sym:%s, size:%lu", SYM_NAME(parent->symbol),
ts_tree_total_size(parent).chars);
StackPushResult last_push = StackPushContinued;
TSStateId state = ts_stack_top_state(self->stack, slice.version);
for (size_t j = 0; j < parent->child_count; j++) {
last_child = parent->children[j];
@ -124,27 +130,19 @@ static BreakdownResult ts_parser__breakdown_top_of_stack(TSParser *self,
state = action.data.to_state;
}
LOG("breakdown_push sym:%s, size:%lu", SYM_NAME(last_child->symbol),
ts_tree_total_size(last_child).chars);
last_push = ts_stack_push(self->stack, slice.version, last_child,
is_still_pending, state);
if (last_push == StackPushFailed)
if (!ts_stack_push(self->stack, slice.version, last_child,
is_still_pending, state))
goto error;
}
for (size_t j = 1, count = slice.trees.size; j < count; j++) {
TSTree *tree = slice.trees.contents[j];
last_push = ts_stack_push(self->stack, slice.version, tree, false, state);
if (last_push == StackPushFailed)
if (!ts_stack_push(self->stack, slice.version, tree, false, state))
goto error;
}
if (i == 0)
assert(last_push != StackPushMerged);
else
assert(last_push == StackPushMerged);
for (size_t j = 0, count = removed_trees.size; j < count; j++)
ts_tree_release(removed_trees.contents[j]);
array_delete(&removed_trees);
@ -212,9 +210,8 @@ static bool ts_parser__can_reuse(TSParser *self, StackVersion version,
return true;
}
static TSTree *ts_parser__get_next_lookahead(TSParser *self,
StackVersion version,
ReusableNode *reusable_node) {
static TSTree *ts_parser__get_lookahead(TSParser *self, StackVersion version,
ReusableNode *reusable_node) {
TSLength position = ts_stack_top_position(self->stack, version);
while (reusable_node->tree) {
@ -281,31 +278,28 @@ static int ts_parser__select_tree(void *data, TSTree *left, TSTree *right) {
return comparison;
}
static ParseActionResult ts_parser__shift(TSParser *self, StackVersion version,
TSStateId parse_state,
TSTree *lookahead) {
bool is_pending = lookahead->child_count > 0;
switch (
ts_stack_push(self->stack, version, lookahead, is_pending, parse_state)) {
case StackPushFailed:
return ParseActionFailed;
case StackPushMerged:
LOG("merge version:%d", version);
return ParseActionRemoved;
default:
return ParseActionUpdated;
}
static bool ts_parser__push(TSParser *self, StackVersion version, TSTree *tree,
TSStateId state) {
bool result = ts_stack_push(self->stack, version, tree, false, state);
ts_tree_release(tree);
return result;
}
static ParseActionResult ts_parser__shift_extra(TSParser *self,
StackVersion version,
TSStateId state,
TSTree *lookahead) {
static bool ts_parser__shift(TSParser *self, StackVersion version,
TSStateId state, TSTree *lookahead) {
bool is_pending = lookahead->child_count > 0;
if (!ts_stack_push(self->stack, version, lookahead, is_pending, state))
return false;
return true;
}
static bool ts_parser__shift_extra(TSParser *self, StackVersion version,
TSStateId state, TSTree *lookahead) {
TSSymbolMetadata metadata = self->language->symbol_metadata[lookahead->symbol];
if (metadata.structural && ts_stack_version_count(self->stack) > 1) {
TSTree *copy = ts_tree_make_copy(lookahead);
if (!copy)
return ParseActionFailed;
return false;
copy->extra = true;
ParseActionResult result = ts_parser__shift(self, version, state, copy);
ts_tree_release(copy);
@ -319,14 +313,14 @@ static ParseActionResult ts_parser__shift_extra(TSParser *self,
static ReduceResult ts_parser__reduce(TSParser *self, StackVersion version,
TSSymbol symbol, int child_count,
bool extra, bool fragile) {
ReduceResult result = {.status = ReduceSucceeded};
ReduceResult result = {.status = ReduceSucceeded };
StackPopResult pop = ts_stack_pop_count(self->stack, version, child_count);
switch (pop.status) {
case StackPopFailed:
goto error;
case StackPopStoppedAtError:
result.status = ReduceStoppedAtError;
result.partial_slice = pop.slices.contents[0];
result.slice = pop.slices.contents[0];
return result;
default:
break;
@ -335,17 +329,15 @@ static ReduceResult ts_parser__reduce(TSParser *self, StackVersion version,
TSSymbolMetadata metadata =
ts_language_symbol_metadata(self->language, symbol);
size_t removed_versions = 0;
for (size_t i = 0; i < pop.slices.size; i++) {
StackSlice slice = pop.slices.contents[i];
slice.version -= removed_versions;
size_t child_count = slice.trees.size;
while (child_count > 0 && slice.trees.contents[child_count - 1]->extra)
child_count--;
TSTree *parent = ts_tree_make_node(symbol, child_count,
slice.trees.contents, metadata);
TSTree *parent =
ts_tree_make_node(symbol, child_count, slice.trees.contents, metadata);
if (!parent) {
ts_tree_array_delete(&slice.trees);
goto error;
@ -371,43 +363,16 @@ static ReduceResult ts_parser__reduce(TSParser *self, StackVersion version,
new_state = action.data.to_state;
}
bool did_merge = false;
StackPushResult push =
ts_stack_push(self->stack, slice.version, parent, false, new_state);
ts_tree_release(parent);
switch (push) {
case StackPushFailed:
goto error;
case StackPushMerged:
did_merge = true;
removed_versions++;
break;
case StackPushContinued:
break;
}
for (size_t j = child_count; !did_merge && j < slice.trees.size; j++) {
TSTree *extra_tree = slice.trees.contents[j];
StackPushResult push =
ts_stack_push(self->stack, slice.version, extra_tree, false, new_state);
ts_tree_release(extra_tree);
switch (push) {
case StackPushFailed:
goto error;
case StackPushMerged:
did_merge = true;
removed_versions++;
break;
case StackPushContinued:
break;
}
CHECK(ts_parser__push(self, slice.version, parent, new_state));
for (size_t j = child_count; j < slice.trees.size; j++) {
TSTree *tree = slice.trees.contents[j];
CHECK(ts_parser__push(self, slice.version, tree, new_state));
}
}
if (removed_versions == pop.slices.size)
result.status = ReduceMerged;
result.slice = pop.slices.contents[0];
return result;
error:
return (ReduceResult){.status = ReduceFailed };
}
@ -454,44 +419,41 @@ static StackIterateAction ts_parser__error_repair_callback(void *payload,
return StackIterateContinue;
}
static RepairResult ts_parser__repair_error(TSParser *self, StackVersion version,
StackSlice partial_slice,
static RepairResult ts_parser__repair_error(TSParser *self, StackSlice slice,
TSTree *lookahead,
const TSParseAction *actions,
size_t action_count) {
ErrorRepairSession session = {
.count_above_error = 0,
.count_above_error = ts_tree_array_essential_count(&slice.trees),
.best_repair_index = -1,
.repairs = &self->error_repairs,
.lookahead_symbol = lookahead->symbol,
.parser = self,
};
for (size_t i = 0; i < partial_slice.trees.size; i++)
if (!partial_slice.trees.contents[i]->extra)
session.count_above_error++;
array_clear(&self->error_repairs);
for (size_t i = 0; i < action_count; i++)
if (actions[i].type == TSParseActionTypeReduce &&
actions[i].data.child_count > session.count_above_error)
array_push(&self->error_repairs, ((ErrorRepair){
.action = actions[i],
.in_progress_state_count = 0,
.essential_tree_count = 0,
}));
CHECK(array_push(&self->error_repairs, ((ErrorRepair){
.action = actions[i],
.in_progress_state_count = 0,
.essential_tree_count = 0,
})));
StackPopResult result = ts_stack_pop_until(
self->stack, version, ts_parser__error_repair_callback, &session);
if (!result.slices.size) {
ts_tree_array_delete(&partial_slice.trees);
StackPopResult pop = ts_stack_pop_until(
self->stack, slice.version, ts_parser__error_repair_callback, &session);
if (!pop.slices.size) {
ts_tree_array_delete(&slice.trees);
return RepairNoneFound;
}
ts_stack_renumber_version(self->stack, pop.slices.contents[0].version,
slice.version);
ErrorRepair repair = self->error_repairs.contents[session.best_repair_index];
size_t count_needed_below =
repair.action.data.child_count - session.count_above_error;
TreeArray children_below = result.slices.contents[0].trees;
TreeArray children_below = pop.slices.contents[0].trees;
size_t count_skipped_below = repair.essential_tree_count - count_needed_below;
TSSymbol symbol = repair.action.data.symbol;
@ -502,42 +464,35 @@ static RepairResult ts_parser__repair_error(TSParser *self, StackVersion version
if (count_skipped_below > 0) {
TreeArray skipped_children = array_new();
if (!array_grow(&skipped_children, count_skipped_below))
goto error;
CHECK(array_grow(&skipped_children, count_skipped_below));
for (size_t i = count_needed_below; i < children_below.size; i++)
array_push(&skipped_children, children_below.contents[i]);
TSTree *error = ts_tree_make_error_node(&skipped_children);
if (!error)
goto error;
CHECK(error);
children_below.size = count_needed_below;
array_push(&children_below, error);
}
for (size_t i = 0; i < partial_slice.trees.size; i++)
array_push(&children_below, partial_slice.trees.contents[i]);
array_delete(&partial_slice);
for (size_t i = 0; i < slice.trees.size; i++)
array_push(&children_below, slice.trees.contents[i]);
array_delete(&slice);
TSTree *parent =
ts_tree_make_node(symbol, children_below.size, children_below.contents,
ts_language_symbol_metadata(self->language, symbol));
StackPushResult push_result =
ts_stack_push(self->stack, version, parent, false, repair.next_state);
CHECK(parent);
CHECK(ts_stack_push(self->stack, slice.version, parent, false,
repair.next_state));
ts_tree_release(parent);
switch (push_result) {
case StackPushFailed:
return RepairFailed;
case StackPushMerged:
return RepairMerged;
default:
return RepairSucceeded;
}
return RepairSucceeded;
error:
return RepairFailed;
}
static ParseActionResult ts_parser__start(TSParser *self, TSInput input,
TSTree *previous_tree) {
static void ts_parser__start(TSParser *self, TSInput input,
TSTree *previous_tree) {
if (previous_tree) {
LOG("parse_after_edit");
} else {
@ -550,19 +505,17 @@ static ParseActionResult ts_parser__start(TSParser *self, TSInput input,
ts_parser__select_tree);
self->finished_tree = NULL;
return ParseActionUpdated;
}
static ParseActionResult ts_parser__accept(TSParser *self, StackVersion version) {
static bool ts_parser__accept(TSParser *self, StackVersion version) {
TreeArray trees = ts_stack_pop_all(self->stack, version);
if (!trees.contents)
goto error;
CHECK(trees.contents);
for (size_t i = trees.size - 1; i + 1 > 0; i--) {
if (!trees.contents[i]->extra) {
TSTree *root = trees.contents[i];
if (!array_splice(&trees, i, 1, root->child_count, root->children))
goto error;
CHECK(array_splice(&trees, i, 1, root->child_count, root->children));
ts_tree_set_children(root, trees.size, trees.contents);
if (!trees.size)
array_delete(&trees);
@ -580,11 +533,11 @@ static ParseActionResult ts_parser__accept(TSParser *self, StackVersion version)
}
}
return ParseActionRemoved;
return true;
error:
ts_tree_array_delete(&trees);
return ParseActionFailed;
return false;
}
static ParseActionResult ts_parser__handle_error(TSParser *self,
@ -593,70 +546,59 @@ static ParseActionResult ts_parser__handle_error(TSParser *self,
TreeArray invalid_trees = array_new();
TSTree *next_token = self->language->lex_fn(&self->lexer, 0, true);
ts_tree_retain(invalid_tree);
if (!array_push(&invalid_trees, invalid_tree))
goto error;
CHECK(array_push(&invalid_trees, invalid_tree));
for (;;) {
if (next_token->symbol == ts_builtin_sym_end) {
LOG_ACTION("fail_to_recover");
ts_tree_release(next_token);
TSTree *error = ts_tree_make_error_node(&invalid_trees);
if (!ts_stack_push(self->stack, version, error, false, 0))
goto error;
CHECK(error);
CHECK(ts_parser__push(self, version, error, 0));
TreeArray trees = ts_stack_pop_all(self->stack, version);
if (!trees.contents)
goto error;
CHECK(trees.contents);
TSTree *parent = ts_tree_make_node(
ts_builtin_sym_start, trees.size, trees.contents,
ts_language_symbol_metadata(self->language, ts_builtin_sym_start));
if (!ts_stack_push(self->stack, version, parent, false, 0))
goto error;
ts_tree_release(parent);
ts_tree_release(error);
ts_tree_release(next_token);
return ts_parser__accept(self, version);
CHECK(parent);
CHECK(ts_parser__push(self, version, parent, 0));
CHECK(ts_parser__accept(self, version));
return ParseActionRemoved;
}
TSLength position = self->lexer.current_position;
TSTree *following_token = self->language->lex_fn(&self->lexer, 0, true);
CHECK(following_token);
if (!ts_language_symbol_metadata(self->language, next_token->symbol).extra) {
TSParseAction action = ts_language_last_action(
self->language, ts_parse_state_error, next_token->symbol);
assert(action.type == TSParseActionTypeShift);
TSStateId next_state = action.data.to_state;
if (ts_language_has_action(self->language, next_state,
following_token->symbol)) {
LOG_ACTION("resume_without_context state:%d", next_state);
ts_tree_release(following_token);
ts_lexer_reset(&self->lexer, position);
ts_tree_steal_padding(*array_back(&invalid_trees), next_token);
TSTree *error = ts_tree_make_error_node(&invalid_trees);
if (!error)
goto error;
if (!ts_stack_push(self->stack, version, error, false,
ts_parse_state_error))
goto error;
if (!ts_stack_push(self->stack, version, next_token, false, next_state))
goto error;
ts_tree_release(error);
ts_tree_release(next_token);
ts_tree_release(following_token);
CHECK(error);
CHECK(ts_parser__push(self, version, error, ts_parse_state_error));
CHECK(ts_parser__push(self, version, next_token, next_state));
return ParseActionUpdated;
}
}
if (!array_push(&invalid_trees, next_token))
goto error;
CHECK(array_push(&invalid_trees, next_token));
next_token = following_token;
}
error:
return ParseActionFailed;
return false;
}
static ParseActionResult ts_parser__consume_lookahead(TSParser *self,
@ -664,138 +606,125 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self,
TSTree *lookahead) {
for (;;) {
TSStateId state = ts_stack_top_state(self->stack, version);
bool error_repair_failed = false;
size_t error_repair_depth = NO_ERROR_DEPTH;
StackVersion last_reduction_version = STACK_VERSION_NONE;
size_t action_count;
const TSParseAction *actions = ts_language_actions(
self->language, state, lookahead->symbol, &action_count);
/*
* If there are multiple actions for the current state and lookahead symbol,
* split the stack so that each one can be performed. If there is a `SHIFT`
* action, it will always appear *last* in the list of actions. Perform it
* on the original stack version and return.
*/
bool repaired_error = false;
size_t child_count_above_error = -1;
for (size_t i = 0; i < action_count; i++) {
TSParseAction action = actions[i];
for (size_t i = 0; i < action_count || error_repair_failed; i++) {
TSParseAction action;
if (i < action_count)
action = actions[i];
else if (error_repair_failed)
action = ERROR_ACTION;
else
break;
bool should_skip = repaired_error &&
action.type == TSParseActionTypeReduce &&
action.data.child_count > child_count_above_error;
StackVersion current_version;
if (i == action_count - 1) {
if (should_skip)
action.type = TSParseActionTypeError;
current_version = version;
} else {
if (should_skip)
continue;
current_version = ts_stack_split(self->stack, version);
LOG_ACTION("split_action from_version:%d, new_version:%d", version,
current_version);
}
if (error_repair_depth != NO_ERROR_DEPTH &&
action.type == TSParseActionTypeReduce &&
action.data.child_count > error_repair_depth)
continue;
LOG_STACK();
switch (action.type) {
case TSParseActionTypeError: {
switch (ts_parser__breakdown_top_of_stack(self, current_version)) {
switch (ts_parser__breakdown_top_of_stack(self, version)) {
case BreakdownFailed:
return ParseActionFailed;
goto error;
case BreakdownPerformed:
return ParseActionRemoved;
default:
continue;
case BreakdownAborted:
break;
}
if (ts_stack_version_count(self->stack) == 1) {
LOG_ACTION("handle_error %s", SYM_NAME(lookahead->symbol));
return ts_parser__handle_error(self, current_version, lookahead);
return ts_parser__handle_error(self, version, lookahead);
} else {
LOG_ACTION("bail current_version:%d", current_version);
ts_stack_remove_version(self->stack, current_version);
LOG_ACTION("bail version:%d", version);
ts_stack_remove_version(self->stack, version);
return ParseActionRemoved;
}
}
case TSParseActionTypeShift:
case TSParseActionTypeShift: {
if (action.extra) {
LOG_ACTION("shift_extra");
return ts_parser__shift_extra(self, current_version, state,
lookahead);
CHECK(ts_parser__shift_extra(self, version, state, lookahead));
return ParseActionUpdated;
} else {
LOG_ACTION("shift state:%u", action.data.to_state);
TSStateId state = action.data.to_state;
return ts_parser__shift(self, current_version, state, lookahead);
TSStateId new_state = action.data.to_state;
LOG_ACTION("shift state:%u", new_state);
CHECK(ts_parser__shift(self, version, new_state, lookahead));
return ParseActionUpdated;
}
}
case TSParseActionTypeReduce:
case TSParseActionTypeReduce: {
if (action.extra) {
LOG_ACTION("reduce_extra sym:%s", SYM_NAME(action.data.symbol));
ts_parser__reduce(self, current_version, action.data.symbol, 1,
true, false);
ReduceResult reduction = ts_parser__reduce(
self, version, action.data.symbol, 1, true, false);
CHECK(reduction.status);
ts_stack_renumber_version(self->stack, reduction.slice.version,
version);
} else {
LOG_ACTION("reduce sym:%s, child_count:%u, fragile:%s",
SYM_NAME(action.data.symbol), action.data.child_count,
BOOL_STRING(action.fragile));
ReduceResult result =
ts_parser__reduce(self, current_version, action.data.symbol,
ReduceResult reduction =
ts_parser__reduce(self, version, action.data.symbol,
action.data.child_count, false, action.fragile);
switch (result.status) {
switch (reduction.status) {
case ReduceFailed:
return ParseActionFailed;
case ReduceMerged:
if (current_version == version)
return ParseActionRemoved;
break;
goto error;
case ReduceSucceeded:
last_reduction_version = reduction.slice.version;
break;
case ReduceStoppedAtError:
repaired_error = true;
child_count_above_error = 0;
for (size_t j = 0; j < result.partial_slice.trees.size; j++)
if (!result.partial_slice.trees.contents[j]->extra)
child_count_above_error++;
error_repair_depth =
ts_tree_array_essential_count(&reduction.slice.trees);
LOG_ACTION("repair version:%d", current_version);
switch (ts_parser__repair_error(self, current_version,
result.partial_slice, lookahead,
actions, action_count)) {
LOG_ACTION("repair version:%d", version);
switch (ts_parser__repair_error(
self, reduction.slice, lookahead, actions, action_count)) {
case RepairFailed:
return ParseActionFailed;
goto error;
case RepairNoneFound:
LOG_ACTION("repair_failed");
if (ts_stack_version_count(self->stack) == 1) {
LOG_ACTION("handle_error %s", SYM_NAME(lookahead->symbol));
return ts_parser__handle_error(self, current_version,
lookahead);
} else {
LOG_ACTION("bail version:%d, count:%d", current_version,
ts_stack_version_count(self->stack));
ts_stack_remove_version(self->stack, current_version);
if (current_version == version)
return ParseActionRemoved;
break;
}
case RepairMerged:
if (current_version == version)
return ParseActionRemoved;
LOG_ACTION("no_repair_found");
ts_stack_remove_version(self->stack,
reduction.slice.version);
error_repair_failed = true;
break;
default:
case RepairSucceeded:
last_reduction_version = reduction.slice.version;
break;
}
break;
}
}
break;
}
case TSParseActionTypeAccept:
case TSParseActionTypeAccept: {
LOG_ACTION("accept");
return ts_parser__accept(self, current_version);
CHECK(ts_parser__accept(self, version));
return ParseActionRemoved;
}
}
}
if (last_reduction_version != STACK_VERSION_NONE)
ts_stack_renumber_version(self->stack, last_reduction_version, version);
}
error:
return ParseActionFailed;
}
bool ts_parser_init(TSParser *self) {
@ -857,7 +786,6 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input, TSTree *previous_tree) {
for (bool removed = false; !removed;) {
last_position = position;
size_t new_position = ts_stack_top_position(self->stack, version).chars;
if (new_position > max_position) {
max_position = new_position;
next_reusable_node = reusable_node;
@ -870,15 +798,14 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input, TSTree *previous_tree) {
position = new_position;
LOG_ACTION("process version:%d, head_count:%d, state:%d, pos:%lu",
LOG_ACTION("process version:%d, version_count:%lu, state:%d, pos:%lu",
version, ts_stack_version_count(self->stack),
ts_stack_top_state(self->stack, version), position);
if (!lookahead || (position != last_position) ||
!ts_parser__can_reuse(self, version, lookahead)) {
ts_tree_release(lookahead);
lookahead =
ts_parser__get_next_lookahead(self, version, &reusable_node);
lookahead = ts_parser__get_lookahead(self, version, &reusable_node);
if (!lookahead)
return NULL;
}
@ -899,6 +826,7 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input, TSTree *previous_tree) {
}
}
ts_stack_merge(self->stack);
ts_tree_release(lookahead);
if (ts_stack_version_count(self->stack) == 0) {

163
src/runtime/stack.c
View file

@ -114,29 +114,25 @@ static StackNode *stack_node_new(StackNode *next, TSTree *tree, bool is_pending,
return node;
}
static void stack_node_add_successor(StackNode *self, TSTree *new_tree,
bool is_pending, StackNode *new_node) {
static void stack_node_add_link(StackNode *self, StackLink link) {
for (int i = 0; i < self->successor_count; i++) {
StackLink successor = self->successors[i];
if (successor.tree == new_tree) {
if (successor.node == new_node)
StackLink existing_link = self->successors[i];
if (existing_link.tree == link.tree) {
if (existing_link.node == link.node)
return;
if (successor.node && new_node &&
successor.node->state == new_node->state) {
for (int j = 0; j < new_node->successor_count; j++) {
stack_node_add_successor(successor.node, new_node->successors[j].tree,
is_pending, new_node->successors[j].node);
}
if (existing_link.node->state == link.node->state) {
for (int j = 0; j < link.node->successor_count; j++)
stack_node_add_link(existing_link.node, link.node->successors[j]);
return;
}
}
}
if (self->successor_count < MAX_SUCCESSOR_COUNT) {
stack_node_retain(new_node);
ts_tree_retain(new_tree);
stack_node_retain(link.node);
ts_tree_retain(link.tree);
self->successors[self->successor_count++] = (StackLink){
new_node, new_tree, is_pending,
link.node, link.tree, link.is_pending,
};
}
}
@ -210,7 +206,7 @@ TSLength ts_stack_top_position(const Stack *self, StackVersion version) {
return (*array_get(&self->heads, version))->position;
}
int ts_stack_version_count(const Stack *self) {
size_t ts_stack_version_count(const Stack *self) {
return self->heads.size;
}
@ -248,19 +244,10 @@ static void ts_stack__merge_slice(Stack *self, StackSlice *slice,
*/
static StackVersion ts_stack__add_version(Stack *self, StackNode *node) {
if (array_push(&self->heads, node)) {
stack_node_retain(node);
return (StackVersion)(self->heads.size - 1);
} else {
return -1;
}
}
static StackVersion ts_stack__find_version(Stack *self, StackNode *node) {
for (size_t i = 0; i < self->heads.size; i++)
if (self->heads.contents[i] == node)
return (StackVersion)i;
return -1;
if (!array_push(&self->heads, node))
return STACK_VERSION_NONE;
stack_node_retain(node);
return (StackVersion)(self->heads.size - 1);
}
void ts_stack_remove_version(Stack *self, StackVersion version) {
@ -273,35 +260,43 @@ void ts_stack_remove_version(Stack *self, StackVersion version) {
* Section: Mutating the stack (Public)
*/
StackPushResult ts_stack_push(Stack *self, StackVersion version, TSTree *tree,
bool is_pending, TSStateId state) {
StackNode *current_version = *array_get(&self->heads, version);
TSLength position =
ts_length_add(current_version->position, ts_tree_total_size(tree));
for (int i = 0; i < version; i++) {
StackNode *prior_node = self->heads.contents[i];
if (prior_node->state == state &&
prior_node->position.chars == position.chars) {
stack_node_add_successor(prior_node, tree, is_pending, current_version);
ts_stack_remove_version(self, version);
return StackPushMerged;
}
}
StackNode *new_version = stack_node_new(current_version, tree, is_pending,
state, position, &self->node_pool);
if (!new_version)
return StackPushFailed;
stack_node_release(current_version, &self->node_pool);
self->heads.contents[version] = new_version;
return StackPushContinued;
bool ts_stack_push(Stack *self, StackVersion version, TSTree *tree,
bool is_pending, TSStateId state) {
StackNode *node = *array_get(&self->heads, version);
TSLength position = ts_length_add(node->position, ts_tree_total_size(tree));
StackNode *new_node =
stack_node_new(node, tree, is_pending, state, position, &self->node_pool);
if (!new_node)
return false;
stack_node_release(node, &self->node_pool);
self->heads.contents[version] = new_node;
return true;
}
StackVersion ts_stack_split(Stack *self, StackVersion version) {
StackNode *head = self->heads.contents[version];
return ts_stack__add_version(self, head);
void ts_stack_merge(Stack *self) {
for (size_t i = 0; i < self->heads.size; i++) {
StackNode *node = self->heads.contents[i];
for (size_t j = 0; j < i; j++) {
StackNode *prior_node = self->heads.contents[j];
if (prior_node->state == node->state &&
prior_node->position.chars == node->position.chars) {
for (size_t k = 0; k < node->successor_count; k++) {
StackLink successor = node->successors[k];
stack_node_add_link(prior_node, successor);
}
ts_stack_remove_version(self, i--);
break;
}
}
}
}
void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) {
assert(v2 < v1);
assert((size_t)v1 < self->heads.size);
stack_node_release(self->heads.contents[v2], &self->node_pool);
self->heads.contents[v2] = self->heads.contents[v1];
array_erase(&self->heads, v1);
}
static inline ALWAYS_INLINE StackPopResult
@ -309,23 +304,20 @@ static inline ALWAYS_INLINE StackPopResult
void *payload) {
array_clear(&self->slices);
array_clear(&self->pop_paths);
StackNode *initial_version = *array_get(&self->heads, version);
PopPath pop_path = {
.node = initial_version,
.node = *array_get(&self->heads, version),
.trees = array_new(),
.essential_tree_count = 0,
.is_done = false,
.is_pending = true,
};
if (!array_grow(&pop_path.trees, STARTING_TREE_CAPACITY))
goto error;
if (!array_push(&self->pop_paths, pop_path))
goto error;
bool all_paths_done = false;
for (size_t depth = 0; !all_paths_done; depth++) {
all_paths_done = true;
for (size_t i = 0, size = self->pop_paths.size; i < size; i++) {
PopPath *path = &self->pop_paths.contents[i];
if (path->is_done)
@ -386,39 +378,29 @@ static inline ALWAYS_INLINE StackPopResult
if (!path->is_done)
continue;
StackSlice slice = {.trees = path->trees, .version = -1 };
StackSlice slice = {.trees = path->trees, .version = STACK_VERSION_NONE };
array_reverse(&slice.trees);
if (i == 0) {
stack_node_retain(path->node);
self->heads.contents[version] = path->node;
slice.version = version;
} else {
slice.version = ts_stack__find_version(self, path->node);
if (slice.version == -1) {
if ((slice.version = ts_stack__add_version(self, path->node)) == -1)
goto error;
} else {
bool merged = false;
for (size_t j = 0; j < self->slices.size; j++) {
StackSlice *prior_result = &self->slices.contents[j];
if (prior_result->version == slice.version) {
ts_stack__merge_slice(self, prior_result, &slice);
merged = true;
break;
}
}
if (merged)
continue;
bool merged = false;
for (size_t j = 0; j < self->slices.size; j++) {
StackSlice *prior_slice = &self->slices.contents[j];
StackNode *prior_node = self->heads.contents[prior_slice->version];
if (prior_node == path->node) {
ts_stack__merge_slice(self, prior_slice, &slice);
merged = true;
break;
}
}
if (!array_push(&self->slices, slice))
goto error;
if (!merged) {
slice.version = ts_stack__add_version(self, path->node);
if (slice.version == STACK_VERSION_NONE)
goto error;
if (!array_push(&self->slices, slice))
goto error;
}
}
if (self->slices.size)
stack_node_release(initial_version, &self->node_pool);
return (StackPopResult){.status = StackPopSucceeded, .slices = self->slices };
error:
@ -478,7 +460,12 @@ static inline ALWAYS_INLINE StackIterateAction
}
StackPopResult ts_stack_pop_pending(Stack *self, StackVersion version) {
return stack__pop(self, version, stack__pop_pending_callback, NULL);
StackPopResult pop = stack__pop(self, version, stack__pop_pending_callback, NULL);
if (pop.slices.size > 0) {
ts_stack_renumber_version(self, pop.slices.contents[0].version, version);
pop.slices.contents[0].version = version;
}
return pop;
}
static inline ALWAYS_INLINE StackIterateAction
@ -492,7 +479,7 @@ TreeArray ts_stack_pop_all(Stack *self, StackVersion version) {
if (pop.status != StackPopSucceeded)
return (TreeArray)array_new();
assert(pop.slices.size == 1);
assert(pop.slices.contents[0].version == version);
ts_stack_renumber_version(self, pop.slices.contents[0].version, version);
return pop.slices.contents[0].trees;
}

23
src/runtime/stack.h
View file

@ -12,7 +12,9 @@ extern "C" {
typedef struct Stack Stack;
typedef int StackVersion;
typedef unsigned int StackVersion;
#define STACK_VERSION_NONE ((StackVersion)-1)
typedef struct {
TreeArray trees;
@ -21,12 +23,6 @@ typedef struct {
typedef Array(StackSlice) StackSliceArray;
typedef enum {
StackPushFailed,
StackPushMerged,
StackPushContinued,
} StackPushResult;
typedef struct {
enum {
StackPopFailed,
@ -62,7 +58,7 @@ void ts_stack_delete(Stack *);
/*
* Get the stack's current number of versions.
*/
int ts_stack_version_count(const Stack *);
size_t ts_stack_version_count(const Stack *);
/*
* Get the state at the top of the given version of the stack. If the stack is
@ -80,7 +76,7 @@ TSLength ts_stack_top_position(const Stack *, StackVersion);
* Push a tree and state onto the given head of the stack. This could cause
* the version to merge with an existing version.
*/
StackPushResult ts_stack_push(Stack *, StackVersion, TSTree *, bool, TSStateId);
bool ts_stack_push(Stack *, StackVersion, TSTree *, bool, TSStateId);
/*
* Pop the given number of entries from the given version of the stack. This
@ -98,12 +94,9 @@ StackPopResult ts_stack_pop_pending(Stack *, StackVersion);
TreeArray ts_stack_pop_all(Stack *, StackVersion);
/*
* Split the given stack head into two versions, so that the stack can be
* transformed from its current state in multiple alternative ways. Returns
* the ID of the newly-created version.
*/
StackVersion ts_stack_split(Stack *, StackVersion);
void ts_stack_merge(Stack *);
void ts_stack_renumber_version(Stack *, StackVersion, StackVersion);
/*
* Remove the given version from the stack.

10
src/runtime/tree.c
View file

@ -51,6 +51,16 @@ void ts_tree_array_delete(TreeArray *self) {
array_delete(self);
}
size_t ts_tree_array_essential_count(const TreeArray *self) {
size_t result = 0;
for (size_t i = 0; i < self->size; i++) {
TSTree *tree = self->contents[i];
if (!tree->extra && tree->symbol != ts_builtin_sym_error)
result++;
}
return result;
}
TSTree *ts_tree_make_error(TSLength size, TSLength padding, char lookahead_char) {
TSTree *result = ts_tree_make_leaf(ts_builtin_sym_error, padding, size,
(TSSymbolMetadata){

1
src/runtime/tree.h
View file

@ -46,6 +46,7 @@ struct TSTree {
typedef Array(TSTree *) TreeArray;
TreeArray ts_tree_array_copy(TreeArray *);
void ts_tree_array_delete(TreeArray *);
size_t ts_tree_array_essential_count(const TreeArray *);
TSTree *ts_tree_make_leaf(TSSymbol, TSLength, TSLength, TSSymbolMetadata);
TSTree *ts_tree_make_node(TSSymbol, size_t, TSTree **, TSSymbolMetadata);