From 695be5bc79958bc2d659f3bfe5cb02269fa84ccb Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Sun, 10 Apr 2016 14:12:24 -0700 Subject: [PATCH] Merge equivalent stacks in a separate stage of parsing * No more automatic merging every time a state is pushed to the stack * When popping from the stack, the current version is always preserved --- spec/runtime/stack_spec.cc | 742 ++++++++++++++++--------------------- src/runtime/array.h | 2 +- src/runtime/parser.c | 396 ++++++++------------ src/runtime/stack.c | 163 ++++---- src/runtime/stack.h | 23 +- src/runtime/tree.c | 10 + src/runtime/tree.h | 1 + 7 files changed, 582 insertions(+), 755 deletions(-) diff --git a/spec/runtime/stack_spec.cc b/spec/runtime/stack_spec.cc index 19cc7254..88041e95 100644 --- a/spec/runtime/stack_spec.cc +++ b/spec/runtime/stack_spec.cc @@ -93,7 +93,6 @@ describe("Stack", [&]() { TSTree *trees[tree_count]; TreeSelectionSpy tree_selection_spy; TSLength tree_len = {2, 3, 0, 3}; - TSSymbolMetadata metadata = {true, true, true, true}; before_each([&]() { record_alloc::start(); @@ -120,9 +119,9 @@ describe("Stack", [&]() { AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty()); }); - describe("pushing entries to the stack", [&]() { - it("adds entries to the stack", [&]() { - AssertThat(ts_stack_version_count(stack), Equals(1)); + describe("push(version, tree, is_pending, state)", [&]() { + it("adds entries to the given version of the stack", [&]() { + AssertThat(ts_stack_version_count(stack), Equals(1)); AssertThat(ts_stack_top_state(stack, 0), Equals(0)); AssertThat(ts_stack_top_position(stack, 0), Equals(ts_length_zero())); @@ -150,473 +149,382 @@ describe("Stack", [&]() { }); }); - describe("popping nodes from the stack", [&]() { + describe("merge()", [&]() { before_each([&]() { - // . <──0── A <──1── B <──2── C* + // . <──0── A <──1── B* + // ↑ + // └───2─── C* ts_stack_push(stack, 0, trees[0], false, stateA); + ts_stack_pop_count(stack, 0, 0); ts_stack_push(stack, 0, trees[1], false, stateB); - ts_stack_push(stack, 0, trees[2], false, stateC); + ts_stack_push(stack, 1, trees[2], false, stateC); }); - it("removes the given number of nodes from the stack", [&]() { - // . <──0── A* - StackPopResult pop_result = ts_stack_pop_count(stack, 0, 2); - AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded)); - AssertThat(pop_result.slices.size, Equals(1)); + it("combines versions that have the same top states and positions", [&]() { + // . <──0── A <──1── B <──3── D* + // ↑ + // └───2─── C <──4── D* + ts_stack_push(stack, 0, trees[3], false, stateD); + ts_stack_push(stack, 1, trees[4], false, stateD); - StackSlice slice = pop_result.slices.contents[0]; - AssertThat(slice.trees, Equals(vector({ trees[1], trees[2] }))); - AssertThat(ts_stack_top_state(stack, 0), Equals(stateA)); - free_slice_array(&pop_result.slices); - - // .* - pop_result = ts_stack_pop_count(stack, 0, 1); - AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded)); - AssertThat(pop_result.slices.size, Equals(1)); - - slice = pop_result.slices.contents[0]; - AssertThat(slice.trees, Equals(vector({ trees[0] }))); - AssertThat(ts_stack_top_state(stack, 0), Equals(0)); - - free_slice_array(&pop_result.slices); + // . <──0── A <──1── B <──3── D* + // ↑ | + // └───2─── C <──4───┘ + ts_stack_merge(stack); + AssertThat(ts_stack_version_count(stack), Equals(1)); + AssertThat(get_stack_entries(stack, 0), Equals(vector({ + {stateD, 0}, + {stateB, 1}, + {stateC, 1}, + {stateA, 2}, + {0, 3}, + }))); }); - it("does not count 'extra' trees toward the count", [&]() { - trees[1]->extra = true; - - // .* - StackPopResult pop_result = ts_stack_pop_count(stack, 0, 2); - AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded)); - AssertThat(pop_result.slices.size, Equals(1)); - - StackSlice slice = pop_result.slices.contents[0]; - AssertThat(slice.trees, Equals(vector({ trees[0], trees[1], trees[2] }))); - AssertThat(ts_stack_top_state(stack, 0), Equals(0)); - - free_slice_array(&pop_result.slices); + it("does not combine versions that have different states", [&]() { + ts_stack_merge(stack); + AssertThat(ts_stack_version_count(stack), Equals(2)); }); - it("pops the entire stack when given a negative count", [&]() { - // .* - StackPopResult pop_result = ts_stack_pop_count(stack, 0, -1); - AssertThat(pop_result.status, Equals(StackPopResult::StackPopSucceeded)); - AssertThat(pop_result.slices.size, Equals(1)); + it("does not combine versions that have different positions", [&]() { + // . <──0── A <──1── B <────3──── D* + // ↑ + // └───2─── C <──4── D* + trees[3]->size = tree_len * 3; + ts_stack_push(stack, 0, trees[3], false, stateD); + ts_stack_push(stack, 1, trees[4], false, stateD); - StackSlice slice = pop_result.slices.contents[0]; - AssertThat(slice.trees, Equals(vector({ trees[0], trees[1], trees[2] }))); - - free_slice_array(&pop_result.slices); + ts_stack_merge(stack); + AssertThat(ts_stack_version_count(stack), Equals(2)); }); - describe("when an error state exists above the given depth", [&]() { - it("stops popping nodes at the error", [&]() { - // . <──0── A <──1── B <──2── C <──3── ERROR <──4── D* - ts_stack_push(stack, 0, trees[3], false, ts_parse_state_error); - ts_stack_push(stack, 0, trees[4], false, stateD); - - StackPopResult pop_result = ts_stack_pop_count(stack, 0, 3); - AssertThat(pop_result.status, Equals(StackPopResult::StackPopStoppedAtError)); - - AssertThat(ts_stack_version_count(stack), Equals(1)); - AssertThat(ts_stack_top_state(stack, 0), Equals(ts_parse_state_error)); - - AssertThat(pop_result.slices.size, Equals(1)); - StackSlice slice = pop_result.slices.contents[0]; - AssertThat(slice.version, Equals(0)); - AssertThat(slice.trees, Equals(vector({ trees[4] }))); - - free_slice_array(&pop_result.slices); - }); - }); - - describe("popping pending nodes from the stack", [&]() { - it("removes the top node from the stack if it was pushed in pending mode", [&]() { - ts_stack_push(stack, 0, trees[3], true, stateD); - - StackPopResult pop = ts_stack_pop_pending(stack, 0); - AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded)); - AssertThat(pop.slices.size, Equals(1)); - - AssertThat(get_stack_entries(stack, 0), Equals(vector({ - {stateC, 0}, - {stateB, 1}, - {stateA, 2}, - {0, 3}, - }))); - - free_slice_array(&pop.slices); - }); - - it("does nothing if the top node was not pushed in pending mode", [&]() { + describe("when the merged versions have more than one common entry", [&]() { + it("combines all of the top common entries", [&]() { + // . <──0── A <──1── B <──3── D <──5── E* + // ↑ + // └───2─── C <──4── D <──5── E* ts_stack_push(stack, 0, trees[3], false, stateD); + ts_stack_push(stack, 0, trees[5], false, stateE); + ts_stack_push(stack, 1, trees[4], false, stateD); + ts_stack_push(stack, 1, trees[5], false, stateE); - StackPopResult pop = ts_stack_pop_pending(stack, 0); - AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded)); - AssertThat(pop.slices.size, Equals(0)); - + // . <──0── A <──1── B <──3── D <──5── E* + // ↑ | + // └───2─── C <──4───┘ + ts_stack_merge(stack); + AssertThat(ts_stack_version_count(stack), Equals(1)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ - {stateD, 0}, - {stateC, 1}, + {stateE, 0}, + {stateD, 1}, {stateB, 2}, + {stateC, 2}, {stateA, 3}, {0, 4}, }))); + }); + }); + }); + describe("pop_count(version, count)", [&]() { + before_each([&]() { + // . <──0── A <──1── B <──2── C* + ts_stack_push(stack, 0, trees[0], false, stateA); + ts_stack_push(stack, 0, trees[1], false, stateB); + ts_stack_push(stack, 0, trees[2], false, stateC); + }); + + it("creates a new version with the given number of entries removed", [&]() { + // . <──0── A <──1── B <──2── C* + // ↑ + // └─* + StackPopResult pop = ts_stack_pop_count(stack, 0, 2); + AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded)); + AssertThat(pop.slices.size, Equals(1)); + AssertThat(ts_stack_version_count(stack), Equals(2)); + + StackSlice slice = pop.slices.contents[0]; + AssertThat(slice.version, Equals(1)); + AssertThat(slice.trees, Equals(vector({ trees[1], trees[2] }))); + AssertThat(ts_stack_top_state(stack, 1), Equals(stateA)); + + free_slice_array(&pop.slices); + }); + + it("does not count 'extra' trees toward the given count", [&]() { + trees[1]->extra = true; + + // . <──0── A <──1── B <──2── C* + // ↑ + // └─* + StackPopResult pop = ts_stack_pop_count(stack, 0, 2); + AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded)); + AssertThat(pop.slices.size, Equals(1)); + + StackSlice slice = pop.slices.contents[0]; + AssertThat(slice.trees, Equals(vector({ trees[0], trees[1], trees[2] }))); + AssertThat(ts_stack_top_state(stack, 1), Equals(0)); + + free_slice_array(&pop.slices); + }); + + it("stops popping entries early if it reaches an error tree", [&]() { + // . <──0── A <──1── B <──2── C <──3── ERROR <──4── D* + ts_stack_push(stack, 0, trees[3], false, ts_parse_state_error); + ts_stack_push(stack, 0, trees[4], false, stateD); + + // . <──0── A <──1── B <──2── C <──3── ERROR <──4── D* + // ↑ + // └─* + StackPopResult pop = ts_stack_pop_count(stack, 0, 3); + AssertThat(pop.status, Equals(StackPopResult::StackPopStoppedAtError)); + + AssertThat(ts_stack_version_count(stack), Equals(2)); + AssertThat(ts_stack_top_state(stack, 1), Equals(ts_parse_state_error)); + + AssertThat(pop.slices.size, Equals(1)); + StackSlice slice = pop.slices.contents[0]; + AssertThat(slice.version, Equals(1)); + AssertThat(slice.trees, Equals(vector({ trees[4] }))); + + free_slice_array(&pop.slices); + }); + + describe("when the version has been merged", [&]() { + before_each([&]() { + // . <──0── A <──1── B <──2── C <──3── D <──10── I* + // ↑ | + // └───4─── E <──5── F <──6───┘ + ts_stack_push(stack, 0, trees[3], false, stateD); + StackPopResult pop = ts_stack_pop_count(stack, 0, 3); free_slice_array(&pop.slices); - }); - }); - }); + ts_stack_push(stack, 1, trees[4], false, stateE); + ts_stack_push(stack, 1, trees[5], false, stateF); + ts_stack_push(stack, 1, trees[6], false, stateD); + ts_stack_merge(stack); + ts_stack_push(stack, 0, trees[10], false, stateI); - describe("splitting the stack", [&]() { - it("creates a new independent version with the same entries", [&]() { - // . <──0── A <──1── B <──2── C* - ts_stack_push(stack, 0, trees[0], false, stateA); - ts_stack_push(stack, 0, trees[1], false, stateB); - ts_stack_push(stack, 0, trees[2], false, stateC); - - // . <──0── A <──1── B <──2── C* - // ↑ - // └─* - int new_index = ts_stack_split(stack, 0); - AssertThat(ts_stack_version_count(stack), Equals(2)); - AssertThat(new_index, Equals(1)); - AssertThat(ts_stack_top_state(stack, 1), Equals(stateC)); - - // . <──0── A <──1── B <──2── C <──3── D* - // ↑ - // └─* - ts_stack_push(stack, 0, trees[3], false, stateD); - StackPopResult pop_result = ts_stack_pop_count(stack, 1, 1); - - AssertThat(ts_stack_version_count(stack), Equals(2)); - AssertThat(ts_stack_top_state(stack, 0), Equals(stateD)); - AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len * 4)); - AssertThat(ts_stack_top_state(stack, 1), Equals(stateB)); - AssertThat(ts_stack_top_position(stack, 1), Equals(tree_len * 2)); - - AssertThat(pop_result.slices.size, Equals(1)); - StackSlice slice = pop_result.slices.contents[0]; - AssertThat(slice.trees.size, Equals(1)); - free_slice_array(&pop_result.slices); - - // . <──0── A <──1── B <──2── C <──3── D* - // ↑ - // └───4─── E <──5── F* - ts_stack_push(stack, 1, trees[4], false, stateE); - ts_stack_push(stack, 1, trees[5], false, stateF); - - AssertThat(ts_stack_version_count(stack), Equals(2)); - AssertThat(ts_stack_top_state(stack, 0), Equals(stateD)); - AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len * 4)); - AssertThat(ts_stack_top_state(stack, 1), Equals(stateF)); - AssertThat(ts_stack_top_position(stack, 1), Equals(tree_len * 4)); - }); - }); - - describe("pushing the same state onto two different versions of the stack", [&]() { - before_each([&]() { - // . <──0── A <──1── B <──2── C <──3── D* - // ↑ - // └───4─── E <──5── F* - ts_stack_push(stack, 0, trees[0], false, stateA); - ts_stack_push(stack, 0, trees[1], false, stateB); - ts_stack_split(stack, 0); - ts_stack_push(stack, 0, trees[2], false, stateC); - ts_stack_push(stack, 0, trees[3], false, stateD); - ts_stack_push(stack, 1, trees[4], false, stateE); - ts_stack_push(stack, 1, trees[5], false, stateF); - - AssertThat(ts_stack_version_count(stack), Equals(2)); - AssertThat(get_stack_entries(stack, 0), Equals(vector({ - {stateD, 0}, - {stateC, 1}, - {stateB, 2}, - {stateA, 3}, - {0, 4}, - }))); - AssertThat(get_stack_entries(stack, 1), Equals(vector({ - {stateF, 0}, - {stateE, 1}, - {stateB, 2}, - {stateA, 3}, - {0, 4}, - }))); - }); - - it("merges the versions", [&]() { - // . <──0── A <──1── B <──2── C <──3── D <──6── G* - // ↑ | - // └───4─── E <──5── F <──7───┘ - AssertThat(ts_stack_push(stack, 0, trees[6], false, stateG), Equals(StackPushContinued)); - AssertThat(ts_stack_push(stack, 1, trees[7], false, stateG), Equals(StackPushMerged)); - - AssertThat(ts_stack_version_count(stack), Equals(1)); - AssertThat(get_stack_entries(stack, 0), Equals(vector({ - {stateG, 0}, - {stateD, 1}, - {stateF, 1}, - {stateC, 2}, - {stateE, 2}, - {stateB, 3}, - {stateA, 4}, - {0, 5}, - }))); - }); - - describe("when the merged nodes share a successor", [&]() { - it("recursively merges the successor nodes", [&]() { - // . <──0── A <──1── B <──2── C <──3── D <──6── G <──7──H* - // ↑ - // └───4─── E <──5── F <──8── G* - AssertThat(ts_stack_push(stack, 0, trees[6], false, stateG), Equals(StackPushContinued)); - AssertThat(ts_stack_push(stack, 0, trees[7], false, stateH), Equals(StackPushContinued)); - AssertThat(ts_stack_push(stack, 1, trees[6], false, stateG), Equals(StackPushContinued)); - - // . <──0── A <──1── B <──2── C <──3── D <──6── G <──7──H* - // ↑ | - // └───4─── E <──5── F <──8───┘ - AssertThat(ts_stack_push(stack, 1, trees[7], false, stateH), Equals(StackPushMerged)); - - AssertThat(ts_stack_version_count(stack), Equals(1)); + AssertThat(ts_stack_version_count(stack), Equals(1)); AssertThat(get_stack_entries(stack, 0), Equals(vector({ - {stateH, 0}, - {stateG, 1}, - {stateD, 2}, + {stateI, 0}, + {stateD, 1}, + {stateC, 2}, {stateF, 2}, - {stateC, 3}, + {stateB, 3}, {stateE, 3}, - {stateB, 4}, - {stateA, 5}, - {0, 6}, + {stateA, 4}, + {0, 5}, }))); }); - }); - describe("when the first version is only one node deep", [&]() { - it("creates a node with one null successor and one non-null successor", [&]() { - ts_tree_retain(trees[2]); - ts_tree_retain(trees[3]); - TSTree *parent = ts_tree_make_node(5, 2, tree_array({ trees[2], trees[3] }), metadata); + describe("when there are two paths that reveal different versions", [&]() { + it("returns an entry for each revealed version", [&]() { + // . <──0── A <──1── B <──2── C <──3── D <──10── I* + // ↑ ↑ + // | └* + // | + // └───4─── E* + StackPopResult pop = ts_stack_pop_count(stack, 0, 3); + AssertThat(pop.slices.size, Equals(2)); - // . <──────5─────── C* - // ↑ | - // └───2─── B ───3───┘ - ts_stack_clear(stack); - ts_stack_split(stack, 0); - AssertThat(ts_stack_push(stack, 0, parent, false, stateC), Equals(StackPushContinued)); - AssertThat(ts_stack_push(stack, 1, trees[2], false, stateB), Equals(StackPushContinued)); - AssertThat(ts_stack_push(stack, 1, trees[3], false, stateC), Equals(StackPushMerged)); + StackSlice slice1 = pop.slices.contents[0]; + AssertThat(slice1.version, Equals(1)); + AssertThat(slice1.trees, Equals(vector({ trees[2], trees[3], trees[10] }))); - AssertThat(ts_stack_version_count(stack), Equals(1)); - AssertThat(ts_stack_top_state(stack, 0), Equals(stateC)); - AssertThat(get_stack_entries(stack, 0), Equals(vector({ - {stateC, 0}, - {0, 1}, - {stateB, 1}, - {0, 2}, - }))); + StackSlice slice2 = pop.slices.contents[1]; + AssertThat(slice2.version, Equals(2)); + AssertThat(slice2.trees, Equals(vector({ trees[5], trees[6], trees[10] }))); - ts_tree_release(parent); - }); - }); - }); + AssertThat(ts_stack_version_count(stack), Equals(3)); + AssertThat(get_stack_entries(stack, 0), Equals(vector({ + {stateI, 0}, + {stateD, 1}, + {stateC, 2}, + {stateF, 2}, + {stateB, 3}, + {stateE, 3}, + {stateA, 4}, + {0, 5}, + }))); + AssertThat(get_stack_entries(stack, 1), Equals(vector({ + {stateB, 0}, + {stateA, 1}, + {0, 2}, + }))); + AssertThat(get_stack_entries(stack, 2), Equals(vector({ + {stateE, 0}, + {stateA, 1}, + {0, 2}, + }))); - describe("popping from a stack version that has been merged", [&]() { - before_each([&]() { - // . <──0── A <──1── B <──2── C <──3── D <──4── E* - // ↑ | - // └───5─── F <──6── G <──7───┘ - ts_stack_push(stack, 0, trees[0], false, stateA); - ts_stack_push(stack, 0, trees[1], false, stateB); - ts_stack_split(stack, 0); - ts_stack_push(stack, 0, trees[2], false, stateC); - ts_stack_push(stack, 0, trees[3], false, stateD); - ts_stack_push(stack, 0, trees[4], false, stateE); - ts_stack_push(stack, 1, trees[5], false, stateF); - ts_stack_push(stack, 1, trees[6], false, stateG); - ts_stack_push(stack, 1, trees[7], false, stateE); - - AssertThat(ts_stack_version_count(stack), Equals(1)); - AssertThat(get_stack_entries(stack, 0), Equals(vector({ - {stateE, 0}, - {stateD, 1}, - {stateG, 1}, - {stateC, 2}, - {stateF, 2}, - {stateB, 3}, - {stateA, 4}, - {0, 5}, - }))); - }); - - describe("when there are two paths that lead to two different versions", [&]() { - it("returns an entry for each revealed version", [&]() { - // . <──0── A <──1── B <──2── C* - // ↑ - // └───5─── F* - StackPopResult pop_result = ts_stack_pop_count(stack, 0, 2); - - AssertThat(pop_result.slices.size, Equals(2)); - StackSlice slice1 = pop_result.slices.contents[0]; - AssertThat(slice1.version, Equals(0)); - AssertThat(ts_stack_top_state(stack, 0), Equals(stateC)); - AssertThat(slice1.trees, Equals(vector({ trees[3], trees[4] }))); - - StackSlice slice2 = pop_result.slices.contents[1]; - AssertThat(slice2.version, Equals(1)); - AssertThat(ts_stack_top_state(stack, 1), Equals(stateF)); - AssertThat(slice2.trees, Equals(vector({ trees[6], trees[7] }))); - - AssertThat(ts_stack_version_count(stack), Equals(2)); - AssertThat(get_stack_entries(stack, 0), Equals(vector({ - {stateC, 0}, - {stateB, 1}, - {stateA, 2}, - {0, 3}, - }))); - AssertThat(get_stack_entries(stack, 1), Equals(vector({ - {stateF, 0}, - {stateB, 1}, - {stateA, 2}, - {0, 3}, - }))); - - free_slice_array(&pop_result.slices); - }); - }); - - describe("when there is one path, leading to one version", [&]() { - it("returns a single entry", [&]() { - // . <──0── A <──1── B <──2── C <──3── D <──4── E <──8──H* - // ↑ | - // └───5─── F <──6── G <──7───┘ - AssertThat(ts_stack_push(stack, 0, trees[8], false, stateH), Equals(StackPushContinued)); - AssertThat(ts_stack_version_count(stack), Equals(1)); - AssertThat(ts_stack_top_state(stack, 0), Equals(stateH)); - - // . <──0── A <──1── B <──2── C <──3── D <──4── E* - // ↑ | - // └───5─── F <──6── G <──7───┘ - StackPopResult pop_result = ts_stack_pop_count(stack, 0, 1); - AssertThat(pop_result.slices.size, Equals(1)); - StackSlice slice1 = pop_result.slices.contents[0]; - AssertThat(slice1.version, Equals(0)); - AssertThat(slice1.trees, Equals(vector({ trees[8] }))); - - AssertThat(ts_stack_version_count(stack), Equals(1)); - AssertThat(ts_stack_top_state(stack, 0), Equals(stateE)); - - free_slice_array(&pop_result.slices); - }); - }); - - describe("when there are two paths that converge at the same version", [&]() { - describe("when the first path is preferred by the callback", [&]() { - it("returns one entry for that version, with the first path of trees", [&]() { - tree_selection_spy.tree_to_return = trees[2]; - - // . <──0── A <──1── B* - StackPopResult pop_result = ts_stack_pop_count(stack, 0, 3); - AssertThat(ts_stack_version_count(stack), Equals(1)); - AssertThat(ts_stack_top_state(stack, 0), Equals(stateB)); - AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len * 2)); - - AssertThat(pop_result.slices.size, Equals(1)); - StackSlice slice1 = pop_result.slices.contents[0]; - AssertThat(slice1.version, Equals(0)); - AssertThat(slice1.trees, Equals(vector({ trees[2], trees[3], trees[4] }))); - - free_slice_array(&pop_result.slices); + free_slice_array(&pop.slices); }); }); - describe("when the second path is preferred by the callback", [&]() { - it("returns one entry for that version, with the second path of trees", [&]() { + describe("when there is one path that ends at a merged version", [&]() { + it("returns a single entry", [&]() { + // . <──0── A <──1── B <──2── C <──3── D <──10── I* + // | | + // └───5─── F <──6── G <──7───┘ + // | + // └* + StackPopResult pop = ts_stack_pop_count(stack, 0, 1); + AssertThat(pop.slices.size, Equals(1)); + + StackSlice slice1 = pop.slices.contents[0]; + AssertThat(slice1.version, Equals(1)); + AssertThat(slice1.trees, Equals(vector({ trees[10] }))); + + AssertThat(ts_stack_version_count(stack), Equals(2)); + AssertThat(ts_stack_top_state(stack, 0), Equals(stateI)); + AssertThat(ts_stack_top_state(stack, 1), Equals(stateD)); + + free_slice_array(&pop.slices); + }); + }); + + describe("when there are two paths that converge on one version", [&]() { + it("returns the first path of trees if they are selected by the selection callback", [&]() { + tree_selection_spy.tree_to_return = trees[1]; + + // . <──0── A <──1── B <──2── C <──3── D <──10── I* + // ↑ | + // ├───4─── E <──5── F <──6───┘ + // | + // └* + StackPopResult pop = ts_stack_pop_count(stack, 0, 4); + AssertThat(pop.slices.size, Equals(1)); + + StackSlice slice1 = pop.slices.contents[0]; + AssertThat(slice1.version, Equals(1)); + AssertThat(slice1.trees, Equals(vector({ trees[1], trees[2], trees[3], trees[10] }))); + + AssertThat(ts_stack_version_count(stack), Equals(2)); + AssertThat(ts_stack_top_state(stack, 0), Equals(stateI)); + AssertThat(ts_stack_top_state(stack, 1), Equals(stateA)); + + free_slice_array(&pop.slices); + }); + + it("returns the second path of trees if they are selected by the selection callback", [&]() { tree_selection_spy.tree_to_return = trees[4]; - // . <──0── A <──1── B* - StackPopResult pop_result = ts_stack_pop_count(stack, 0, 3); - AssertThat(ts_stack_version_count(stack), Equals(1)); - AssertThat(ts_stack_top_state(stack, 0), Equals(stateB)); - AssertThat(ts_stack_top_position(stack, 0), Equals(tree_len * 2)); + // . <──0── A <──1── B <──2── C <──3── D <──10── I* + // ↑ | + // ├───4─── E <──5── F <──6───┘ + // | + // └* + StackPopResult pop = ts_stack_pop_count(stack, 0, 4); + AssertThat(pop.slices.size, Equals(1)); - AssertThat(pop_result.slices.size, Equals(1)); - StackSlice slice1 = pop_result.slices.contents[0]; - AssertThat(slice1.version, Equals(0)); - AssertThat(slice1.trees, Equals(vector({ trees[5], trees[6], trees[7] }))) + StackSlice slice1 = pop.slices.contents[0]; + AssertThat(slice1.version, Equals(1)); + AssertThat(slice1.trees, Equals(vector({ trees[4], trees[5], trees[6], trees[10] }))) - free_slice_array(&pop_result.slices); + AssertThat(ts_stack_version_count(stack), Equals(2)); + AssertThat(ts_stack_top_state(stack, 0), Equals(stateI)); + AssertThat(ts_stack_top_state(stack, 1), Equals(stateA)); + + free_slice_array(&pop.slices); + }); + }); + + describe("when there are three paths that lead to three different versions", [&]() { + it("returns three entries with different arrays of trees", [&]() { + // . <──0── A <──1── B <──2── C <──3── D <──10── I* + // ↑ | + // ├───4─── E <──5── F <──6───┘ + // | | + // └───7─── G <──8── H <──9───┘ + StackPopResult pop = ts_stack_pop_count(stack, 0, 4); + free_slice_array(&pop.slices); + ts_stack_push(stack, 1, trees[7], false, stateG); + ts_stack_push(stack, 1, trees[8], false, stateH); + ts_stack_push(stack, 1, trees[9], false, stateD); + ts_stack_push(stack, 1, trees[10], false, stateI); + ts_stack_merge(stack); + + AssertThat(ts_stack_version_count(stack), Equals(1)); + AssertThat(get_stack_entries(stack, 0), Equals(vector({ + {stateI, 0}, + {stateD, 1}, + {stateC, 2}, + {stateF, 2}, + {stateH, 2}, + {stateB, 3}, + {stateE, 3}, + {stateG, 3}, + {stateA, 4}, + {0, 5}, + }))); + + // . <──0── A <──1── B <──2── C <──3── D <──10── I* + // ↑ ↑ + // | └* + // | + // ├───4─── E <──5── F* + // | + // └───7─── G <──8── H* + pop = ts_stack_pop_count(stack, 0, 2); + AssertThat(pop.slices.size, Equals(3)); + + StackSlice slice1 = pop.slices.contents[0]; + AssertThat(slice1.version, Equals(1)); + AssertThat(slice1.trees, Equals(vector({ trees[3], trees[10] }))) + + StackSlice slice2 = pop.slices.contents[1]; + AssertThat(slice2.version, Equals(2)); + AssertThat(slice2.trees, Equals(vector({ trees[6], trees[10] }))) + + StackSlice slice3 = pop.slices.contents[2]; + AssertThat(slice3.version, Equals(3)); + AssertThat(slice3.trees, Equals(vector({ trees[9], trees[10] }))) + + AssertThat(ts_stack_version_count(stack), Equals(4)); + AssertThat(ts_stack_top_state(stack, 0), Equals(stateI)); + AssertThat(ts_stack_top_state(stack, 1), Equals(stateC)); + AssertThat(ts_stack_top_state(stack, 2), Equals(stateF)); + AssertThat(ts_stack_top_state(stack, 3), Equals(stateH)); + + free_slice_array(&pop.slices); }); }); }); }); - describe("popping from a stack version that has been 3-way merged", [&]() { + describe("pop_pending(version)", [&]() { before_each([&]() { - // . <──0── A <──1── B <──2── C <──3── D <──10── I - // ↑ | - // ├───4─── E <──5── F <──6───┤ - // | | - // └───7─── G <──8── H <──9───┘ - ts_stack_clear(stack); ts_stack_push(stack, 0, trees[0], false, stateA); - ts_stack_split(stack, 0); - ts_stack_split(stack, 1); - ts_stack_push(stack, 0, trees[1], false, stateB); - ts_stack_push(stack, 0, trees[2], false, stateC); - ts_stack_push(stack, 0, trees[3], false, stateD); - ts_stack_push(stack, 1, trees[4], false, stateE); - ts_stack_push(stack, 1, trees[5], false, stateF); - ts_stack_push(stack, 1, trees[6], false, stateD); - ts_stack_push(stack, 1, trees[7], false, stateG); - ts_stack_push(stack, 1, trees[8], false, stateH); - ts_stack_push(stack, 1, trees[9], false, stateD); - ts_stack_push(stack, 0, trees[10], false, stateI); - - AssertThat(ts_stack_version_count(stack), Equals(1)); - AssertThat(get_stack_entries(stack, 0), Equals(vector({ - {stateI, 0}, - {stateD, 1}, - {stateC, 2}, - {stateF, 2}, - {stateH, 2}, - {stateB, 3}, - {stateE, 3}, - {stateG, 3}, - {stateA, 4}, - {0, 5}, - }))); }); - describe("when there are three different paths that lead to three different versions", [&]() { - it("returns three entries with different arrays of trees", [&]() { - // . <──0── A <──1── B <──2── C* - // ↑ - // ├───4─── E <──5── F* - // | - // └───7─── G <──8── H* - StackPopResult pop_result = ts_stack_pop_count(stack, 0, 2); - AssertThat(ts_stack_version_count(stack), Equals(3)); + it("removes the top node from the stack if it was pushed in pending mode", [&]() { + ts_stack_push(stack, 0, trees[1], true, stateB); - AssertThat(pop_result.slices.size, Equals(3)); + StackPopResult pop = ts_stack_pop_pending(stack, 0); + AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded)); + AssertThat(pop.slices.size, Equals(1)); - StackSlice slice1 = pop_result.slices.contents[0]; - AssertThat(ts_stack_top_state(stack, 0), Equals(stateC)); - AssertThat(slice1.version, Equals(0)); - AssertThat(slice1.trees, Equals(vector({ trees[3], trees[10] }))) + AssertThat(get_stack_entries(stack, 0), Equals(vector({ + {stateA, 0}, + {0, 1}, + }))); - StackSlice slice2 = pop_result.slices.contents[1]; - AssertThat(ts_stack_top_state(stack, 1), Equals(stateF)); - AssertThat(slice2.version, Equals(1)); - AssertThat(slice2.trees, Equals(vector({ trees[6], trees[10] }))) + free_slice_array(&pop.slices); + }); - StackSlice slice3 = pop_result.slices.contents[2]; - AssertThat(ts_stack_top_state(stack, 2), Equals(stateH)); - AssertThat(slice3.version, Equals(2)); - AssertThat(slice3.trees, Equals(vector({ trees[9], trees[10] }))) + it("does nothing if the top node was not pushed in pending mode", [&]() { + ts_stack_push(stack, 0, trees[1], false, stateB); - free_slice_array(&pop_result.slices); - }); + StackPopResult pop = ts_stack_pop_pending(stack, 0); + AssertThat(pop.status, Equals(StackPopResult::StackPopSucceeded)); + AssertThat(pop.slices.size, Equals(0)); + + AssertThat(get_stack_entries(stack, 0), Equals(vector({ + {stateB, 0}, + {stateA, 1}, + {0, 2}, + }))); + + free_slice_array(&pop.slices); }); }); }); diff --git a/src/runtime/array.h b/src/runtime/array.h index e6934b10..239de3ae 100644 --- a/src/runtime/array.h +++ b/src/runtime/array.h @@ -43,7 +43,7 @@ extern "C" { #define array_push(self, element) \ (((self)->size < (self)->capacity || \ - array_grow((self), (self)->capacity ? (self)->capacity * 2 : 4)) && \ + array_grow((self), (self)->capacity ? (self)->capacity * 2 : 8)) && \ ((self)->contents[(self)->size++] = (element), true)) #define array_splice(self, index, old_count, new_count, new_elements) \ diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 18b12491..0de94c11 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -37,6 +37,15 @@ #define BOOL_STRING(value) (value ? "true" : "false") +#define CHECK(expr) \ + if (!(expr)) { \ + goto error; \ + } + +static const TSParseAction ERROR_ACTION = {.type = TSParseActionTypeError }; + +static const size_t NO_ERROR_DEPTH = (size_t)(-1); + typedef struct { TSTree *tree; size_t char_index; @@ -67,17 +76,15 @@ typedef struct { enum { ReduceFailed, ReduceSucceeded, - ReduceMerged, ReduceStoppedAtError, } status; - StackSlice partial_slice; + StackSlice slice; } ReduceResult; typedef enum { RepairFailed, RepairSucceeded, - RepairMerged, RepairNoneFound, } RepairResult; @@ -109,7 +116,6 @@ static BreakdownResult ts_parser__breakdown_top_of_stack(TSParser *self, LOG("breakdown_pop sym:%s, size:%lu", SYM_NAME(parent->symbol), ts_tree_total_size(parent).chars); - StackPushResult last_push = StackPushContinued; TSStateId state = ts_stack_top_state(self->stack, slice.version); for (size_t j = 0; j < parent->child_count; j++) { last_child = parent->children[j]; @@ -124,27 +130,19 @@ static BreakdownResult ts_parser__breakdown_top_of_stack(TSParser *self, state = action.data.to_state; } - LOG("breakdown_push sym:%s, size:%lu", SYM_NAME(last_child->symbol), ts_tree_total_size(last_child).chars); - last_push = ts_stack_push(self->stack, slice.version, last_child, - is_still_pending, state); - if (last_push == StackPushFailed) + if (!ts_stack_push(self->stack, slice.version, last_child, + is_still_pending, state)) goto error; } for (size_t j = 1, count = slice.trees.size; j < count; j++) { TSTree *tree = slice.trees.contents[j]; - last_push = ts_stack_push(self->stack, slice.version, tree, false, state); - if (last_push == StackPushFailed) + if (!ts_stack_push(self->stack, slice.version, tree, false, state)) goto error; } - if (i == 0) - assert(last_push != StackPushMerged); - else - assert(last_push == StackPushMerged); - for (size_t j = 0, count = removed_trees.size; j < count; j++) ts_tree_release(removed_trees.contents[j]); array_delete(&removed_trees); @@ -212,9 +210,8 @@ static bool ts_parser__can_reuse(TSParser *self, StackVersion version, return true; } -static TSTree *ts_parser__get_next_lookahead(TSParser *self, - StackVersion version, - ReusableNode *reusable_node) { +static TSTree *ts_parser__get_lookahead(TSParser *self, StackVersion version, + ReusableNode *reusable_node) { TSLength position = ts_stack_top_position(self->stack, version); while (reusable_node->tree) { @@ -281,31 +278,28 @@ static int ts_parser__select_tree(void *data, TSTree *left, TSTree *right) { return comparison; } -static ParseActionResult ts_parser__shift(TSParser *self, StackVersion version, - TSStateId parse_state, - TSTree *lookahead) { - bool is_pending = lookahead->child_count > 0; - switch ( - ts_stack_push(self->stack, version, lookahead, is_pending, parse_state)) { - case StackPushFailed: - return ParseActionFailed; - case StackPushMerged: - LOG("merge version:%d", version); - return ParseActionRemoved; - default: - return ParseActionUpdated; - } +static bool ts_parser__push(TSParser *self, StackVersion version, TSTree *tree, + TSStateId state) { + bool result = ts_stack_push(self->stack, version, tree, false, state); + ts_tree_release(tree); + return result; } -static ParseActionResult ts_parser__shift_extra(TSParser *self, - StackVersion version, - TSStateId state, - TSTree *lookahead) { +static bool ts_parser__shift(TSParser *self, StackVersion version, + TSStateId state, TSTree *lookahead) { + bool is_pending = lookahead->child_count > 0; + if (!ts_stack_push(self->stack, version, lookahead, is_pending, state)) + return false; + return true; +} + +static bool ts_parser__shift_extra(TSParser *self, StackVersion version, + TSStateId state, TSTree *lookahead) { TSSymbolMetadata metadata = self->language->symbol_metadata[lookahead->symbol]; if (metadata.structural && ts_stack_version_count(self->stack) > 1) { TSTree *copy = ts_tree_make_copy(lookahead); if (!copy) - return ParseActionFailed; + return false; copy->extra = true; ParseActionResult result = ts_parser__shift(self, version, state, copy); ts_tree_release(copy); @@ -319,14 +313,14 @@ static ParseActionResult ts_parser__shift_extra(TSParser *self, static ReduceResult ts_parser__reduce(TSParser *self, StackVersion version, TSSymbol symbol, int child_count, bool extra, bool fragile) { - ReduceResult result = {.status = ReduceSucceeded}; + ReduceResult result = {.status = ReduceSucceeded }; StackPopResult pop = ts_stack_pop_count(self->stack, version, child_count); switch (pop.status) { case StackPopFailed: goto error; case StackPopStoppedAtError: result.status = ReduceStoppedAtError; - result.partial_slice = pop.slices.contents[0]; + result.slice = pop.slices.contents[0]; return result; default: break; @@ -335,17 +329,15 @@ static ReduceResult ts_parser__reduce(TSParser *self, StackVersion version, TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, symbol); - size_t removed_versions = 0; for (size_t i = 0; i < pop.slices.size; i++) { StackSlice slice = pop.slices.contents[i]; - slice.version -= removed_versions; size_t child_count = slice.trees.size; while (child_count > 0 && slice.trees.contents[child_count - 1]->extra) child_count--; - TSTree *parent = ts_tree_make_node(symbol, child_count, - slice.trees.contents, metadata); + TSTree *parent = + ts_tree_make_node(symbol, child_count, slice.trees.contents, metadata); if (!parent) { ts_tree_array_delete(&slice.trees); goto error; @@ -371,43 +363,16 @@ static ReduceResult ts_parser__reduce(TSParser *self, StackVersion version, new_state = action.data.to_state; } - bool did_merge = false; - - StackPushResult push = - ts_stack_push(self->stack, slice.version, parent, false, new_state); - ts_tree_release(parent); - switch (push) { - case StackPushFailed: - goto error; - case StackPushMerged: - did_merge = true; - removed_versions++; - break; - case StackPushContinued: - break; - } - - for (size_t j = child_count; !did_merge && j < slice.trees.size; j++) { - TSTree *extra_tree = slice.trees.contents[j]; - StackPushResult push = - ts_stack_push(self->stack, slice.version, extra_tree, false, new_state); - ts_tree_release(extra_tree); - switch (push) { - case StackPushFailed: - goto error; - case StackPushMerged: - did_merge = true; - removed_versions++; - break; - case StackPushContinued: - break; - } + CHECK(ts_parser__push(self, slice.version, parent, new_state)); + for (size_t j = child_count; j < slice.trees.size; j++) { + TSTree *tree = slice.trees.contents[j]; + CHECK(ts_parser__push(self, slice.version, tree, new_state)); } } - if (removed_versions == pop.slices.size) - result.status = ReduceMerged; + result.slice = pop.slices.contents[0]; return result; + error: return (ReduceResult){.status = ReduceFailed }; } @@ -454,44 +419,41 @@ static StackIterateAction ts_parser__error_repair_callback(void *payload, return StackIterateContinue; } -static RepairResult ts_parser__repair_error(TSParser *self, StackVersion version, - StackSlice partial_slice, +static RepairResult ts_parser__repair_error(TSParser *self, StackSlice slice, TSTree *lookahead, const TSParseAction *actions, size_t action_count) { ErrorRepairSession session = { - .count_above_error = 0, + .count_above_error = ts_tree_array_essential_count(&slice.trees), .best_repair_index = -1, .repairs = &self->error_repairs, .lookahead_symbol = lookahead->symbol, .parser = self, }; - for (size_t i = 0; i < partial_slice.trees.size; i++) - if (!partial_slice.trees.contents[i]->extra) - session.count_above_error++; - array_clear(&self->error_repairs); for (size_t i = 0; i < action_count; i++) if (actions[i].type == TSParseActionTypeReduce && actions[i].data.child_count > session.count_above_error) - array_push(&self->error_repairs, ((ErrorRepair){ - .action = actions[i], - .in_progress_state_count = 0, - .essential_tree_count = 0, - })); + CHECK(array_push(&self->error_repairs, ((ErrorRepair){ + .action = actions[i], + .in_progress_state_count = 0, + .essential_tree_count = 0, + }))); - StackPopResult result = ts_stack_pop_until( - self->stack, version, ts_parser__error_repair_callback, &session); - if (!result.slices.size) { - ts_tree_array_delete(&partial_slice.trees); + StackPopResult pop = ts_stack_pop_until( + self->stack, slice.version, ts_parser__error_repair_callback, &session); + if (!pop.slices.size) { + ts_tree_array_delete(&slice.trees); return RepairNoneFound; } + ts_stack_renumber_version(self->stack, pop.slices.contents[0].version, + slice.version); ErrorRepair repair = self->error_repairs.contents[session.best_repair_index]; size_t count_needed_below = repair.action.data.child_count - session.count_above_error; - TreeArray children_below = result.slices.contents[0].trees; + TreeArray children_below = pop.slices.contents[0].trees; size_t count_skipped_below = repair.essential_tree_count - count_needed_below; TSSymbol symbol = repair.action.data.symbol; @@ -502,42 +464,35 @@ static RepairResult ts_parser__repair_error(TSParser *self, StackVersion version if (count_skipped_below > 0) { TreeArray skipped_children = array_new(); - if (!array_grow(&skipped_children, count_skipped_below)) - goto error; + CHECK(array_grow(&skipped_children, count_skipped_below)); for (size_t i = count_needed_below; i < children_below.size; i++) array_push(&skipped_children, children_below.contents[i]); + TSTree *error = ts_tree_make_error_node(&skipped_children); - if (!error) - goto error; + CHECK(error); children_below.size = count_needed_below; array_push(&children_below, error); } - for (size_t i = 0; i < partial_slice.trees.size; i++) - array_push(&children_below, partial_slice.trees.contents[i]); - array_delete(&partial_slice); + for (size_t i = 0; i < slice.trees.size; i++) + array_push(&children_below, slice.trees.contents[i]); + array_delete(&slice); + TSTree *parent = ts_tree_make_node(symbol, children_below.size, children_below.contents, ts_language_symbol_metadata(self->language, symbol)); - - StackPushResult push_result = - ts_stack_push(self->stack, version, parent, false, repair.next_state); + CHECK(parent); + CHECK(ts_stack_push(self->stack, slice.version, parent, false, + repair.next_state)); ts_tree_release(parent); - switch (push_result) { - case StackPushFailed: - return RepairFailed; - case StackPushMerged: - return RepairMerged; - default: - return RepairSucceeded; - } + return RepairSucceeded; error: return RepairFailed; } -static ParseActionResult ts_parser__start(TSParser *self, TSInput input, - TSTree *previous_tree) { +static void ts_parser__start(TSParser *self, TSInput input, + TSTree *previous_tree) { if (previous_tree) { LOG("parse_after_edit"); } else { @@ -550,19 +505,17 @@ static ParseActionResult ts_parser__start(TSParser *self, TSInput input, ts_parser__select_tree); self->finished_tree = NULL; - return ParseActionUpdated; } -static ParseActionResult ts_parser__accept(TSParser *self, StackVersion version) { +static bool ts_parser__accept(TSParser *self, StackVersion version) { TreeArray trees = ts_stack_pop_all(self->stack, version); - if (!trees.contents) - goto error; + CHECK(trees.contents); for (size_t i = trees.size - 1; i + 1 > 0; i--) { if (!trees.contents[i]->extra) { TSTree *root = trees.contents[i]; - if (!array_splice(&trees, i, 1, root->child_count, root->children)) - goto error; + CHECK(array_splice(&trees, i, 1, root->child_count, root->children)); + ts_tree_set_children(root, trees.size, trees.contents); if (!trees.size) array_delete(&trees); @@ -580,11 +533,11 @@ static ParseActionResult ts_parser__accept(TSParser *self, StackVersion version) } } - return ParseActionRemoved; + return true; error: ts_tree_array_delete(&trees); - return ParseActionFailed; + return false; } static ParseActionResult ts_parser__handle_error(TSParser *self, @@ -593,70 +546,59 @@ static ParseActionResult ts_parser__handle_error(TSParser *self, TreeArray invalid_trees = array_new(); TSTree *next_token = self->language->lex_fn(&self->lexer, 0, true); ts_tree_retain(invalid_tree); - if (!array_push(&invalid_trees, invalid_tree)) - goto error; + CHECK(array_push(&invalid_trees, invalid_tree)); for (;;) { if (next_token->symbol == ts_builtin_sym_end) { LOG_ACTION("fail_to_recover"); + + ts_tree_release(next_token); TSTree *error = ts_tree_make_error_node(&invalid_trees); - if (!ts_stack_push(self->stack, version, error, false, 0)) - goto error; + CHECK(error); + CHECK(ts_parser__push(self, version, error, 0)); TreeArray trees = ts_stack_pop_all(self->stack, version); - if (!trees.contents) - goto error; + CHECK(trees.contents); TSTree *parent = ts_tree_make_node( ts_builtin_sym_start, trees.size, trees.contents, ts_language_symbol_metadata(self->language, ts_builtin_sym_start)); - - if (!ts_stack_push(self->stack, version, parent, false, 0)) - goto error; - - ts_tree_release(parent); - ts_tree_release(error); - ts_tree_release(next_token); - return ts_parser__accept(self, version); + CHECK(parent); + CHECK(ts_parser__push(self, version, parent, 0)); + CHECK(ts_parser__accept(self, version)); + return ParseActionRemoved; } TSLength position = self->lexer.current_position; TSTree *following_token = self->language->lex_fn(&self->lexer, 0, true); + CHECK(following_token); if (!ts_language_symbol_metadata(self->language, next_token->symbol).extra) { TSParseAction action = ts_language_last_action( self->language, ts_parse_state_error, next_token->symbol); assert(action.type == TSParseActionTypeShift); - TSStateId next_state = action.data.to_state; + if (ts_language_has_action(self->language, next_state, following_token->symbol)) { LOG_ACTION("resume_without_context state:%d", next_state); + + ts_tree_release(following_token); ts_lexer_reset(&self->lexer, position); ts_tree_steal_padding(*array_back(&invalid_trees), next_token); TSTree *error = ts_tree_make_error_node(&invalid_trees); - if (!error) - goto error; - if (!ts_stack_push(self->stack, version, error, false, - ts_parse_state_error)) - goto error; - if (!ts_stack_push(self->stack, version, next_token, false, next_state)) - goto error; - - ts_tree_release(error); - ts_tree_release(next_token); - ts_tree_release(following_token); - + CHECK(error); + CHECK(ts_parser__push(self, version, error, ts_parse_state_error)); + CHECK(ts_parser__push(self, version, next_token, next_state)); return ParseActionUpdated; } } - if (!array_push(&invalid_trees, next_token)) - goto error; + CHECK(array_push(&invalid_trees, next_token)); next_token = following_token; } error: - return ParseActionFailed; + return false; } static ParseActionResult ts_parser__consume_lookahead(TSParser *self, @@ -664,138 +606,125 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self, TSTree *lookahead) { for (;;) { TSStateId state = ts_stack_top_state(self->stack, version); + + bool error_repair_failed = false; + size_t error_repair_depth = NO_ERROR_DEPTH; + StackVersion last_reduction_version = STACK_VERSION_NONE; + size_t action_count; const TSParseAction *actions = ts_language_actions( self->language, state, lookahead->symbol, &action_count); - /* - * If there are multiple actions for the current state and lookahead symbol, - * split the stack so that each one can be performed. If there is a `SHIFT` - * action, it will always appear *last* in the list of actions. Perform it - * on the original stack version and return. - */ - bool repaired_error = false; - size_t child_count_above_error = -1; - for (size_t i = 0; i < action_count; i++) { - TSParseAction action = actions[i]; + for (size_t i = 0; i < action_count || error_repair_failed; i++) { + TSParseAction action; + if (i < action_count) + action = actions[i]; + else if (error_repair_failed) + action = ERROR_ACTION; + else + break; - bool should_skip = repaired_error && - action.type == TSParseActionTypeReduce && - action.data.child_count > child_count_above_error; - - StackVersion current_version; - if (i == action_count - 1) { - if (should_skip) - action.type = TSParseActionTypeError; - current_version = version; - } else { - if (should_skip) - continue; - current_version = ts_stack_split(self->stack, version); - LOG_ACTION("split_action from_version:%d, new_version:%d", version, - current_version); - } + if (error_repair_depth != NO_ERROR_DEPTH && + action.type == TSParseActionTypeReduce && + action.data.child_count > error_repair_depth) + continue; LOG_STACK(); switch (action.type) { case TSParseActionTypeError: { - switch (ts_parser__breakdown_top_of_stack(self, current_version)) { + switch (ts_parser__breakdown_top_of_stack(self, version)) { case BreakdownFailed: - return ParseActionFailed; + goto error; case BreakdownPerformed: - return ParseActionRemoved; - default: + continue; + case BreakdownAborted: break; } if (ts_stack_version_count(self->stack) == 1) { LOG_ACTION("handle_error %s", SYM_NAME(lookahead->symbol)); - return ts_parser__handle_error(self, current_version, lookahead); + return ts_parser__handle_error(self, version, lookahead); } else { - LOG_ACTION("bail current_version:%d", current_version); - ts_stack_remove_version(self->stack, current_version); + LOG_ACTION("bail version:%d", version); + ts_stack_remove_version(self->stack, version); return ParseActionRemoved; } } - case TSParseActionTypeShift: + case TSParseActionTypeShift: { if (action.extra) { LOG_ACTION("shift_extra"); - return ts_parser__shift_extra(self, current_version, state, - lookahead); + CHECK(ts_parser__shift_extra(self, version, state, lookahead)); + return ParseActionUpdated; } else { - LOG_ACTION("shift state:%u", action.data.to_state); - TSStateId state = action.data.to_state; - return ts_parser__shift(self, current_version, state, lookahead); + TSStateId new_state = action.data.to_state; + LOG_ACTION("shift state:%u", new_state); + CHECK(ts_parser__shift(self, version, new_state, lookahead)); + return ParseActionUpdated; } + } - case TSParseActionTypeReduce: + case TSParseActionTypeReduce: { if (action.extra) { LOG_ACTION("reduce_extra sym:%s", SYM_NAME(action.data.symbol)); - ts_parser__reduce(self, current_version, action.data.symbol, 1, - true, false); + ReduceResult reduction = ts_parser__reduce( + self, version, action.data.symbol, 1, true, false); + CHECK(reduction.status); + ts_stack_renumber_version(self->stack, reduction.slice.version, + version); } else { LOG_ACTION("reduce sym:%s, child_count:%u, fragile:%s", SYM_NAME(action.data.symbol), action.data.child_count, BOOL_STRING(action.fragile)); - ReduceResult result = - ts_parser__reduce(self, current_version, action.data.symbol, + ReduceResult reduction = + ts_parser__reduce(self, version, action.data.symbol, action.data.child_count, false, action.fragile); - switch (result.status) { + switch (reduction.status) { case ReduceFailed: - return ParseActionFailed; - case ReduceMerged: - if (current_version == version) - return ParseActionRemoved; - break; + goto error; case ReduceSucceeded: + last_reduction_version = reduction.slice.version; break; case ReduceStoppedAtError: - repaired_error = true; - child_count_above_error = 0; - for (size_t j = 0; j < result.partial_slice.trees.size; j++) - if (!result.partial_slice.trees.contents[j]->extra) - child_count_above_error++; + error_repair_depth = + ts_tree_array_essential_count(&reduction.slice.trees); - LOG_ACTION("repair version:%d", current_version); - switch (ts_parser__repair_error(self, current_version, - result.partial_slice, lookahead, - actions, action_count)) { + LOG_ACTION("repair version:%d", version); + switch (ts_parser__repair_error( + self, reduction.slice, lookahead, actions, action_count)) { case RepairFailed: - return ParseActionFailed; + goto error; case RepairNoneFound: - LOG_ACTION("repair_failed"); - if (ts_stack_version_count(self->stack) == 1) { - LOG_ACTION("handle_error %s", SYM_NAME(lookahead->symbol)); - return ts_parser__handle_error(self, current_version, - lookahead); - } else { - LOG_ACTION("bail version:%d, count:%d", current_version, - ts_stack_version_count(self->stack)); - ts_stack_remove_version(self->stack, current_version); - if (current_version == version) - return ParseActionRemoved; - break; - } - case RepairMerged: - if (current_version == version) - return ParseActionRemoved; + LOG_ACTION("no_repair_found"); + ts_stack_remove_version(self->stack, + reduction.slice.version); + error_repair_failed = true; break; - default: + case RepairSucceeded: + last_reduction_version = reduction.slice.version; break; } break; } } break; + } - case TSParseActionTypeAccept: + case TSParseActionTypeAccept: { LOG_ACTION("accept"); - return ts_parser__accept(self, current_version); + CHECK(ts_parser__accept(self, version)); + return ParseActionRemoved; + } } } + + if (last_reduction_version != STACK_VERSION_NONE) + ts_stack_renumber_version(self->stack, last_reduction_version, version); } + +error: + return ParseActionFailed; } bool ts_parser_init(TSParser *self) { @@ -857,7 +786,6 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input, TSTree *previous_tree) { for (bool removed = false; !removed;) { last_position = position; size_t new_position = ts_stack_top_position(self->stack, version).chars; - if (new_position > max_position) { max_position = new_position; next_reusable_node = reusable_node; @@ -870,15 +798,14 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input, TSTree *previous_tree) { position = new_position; - LOG_ACTION("process version:%d, head_count:%d, state:%d, pos:%lu", + LOG_ACTION("process version:%d, version_count:%lu, state:%d, pos:%lu", version, ts_stack_version_count(self->stack), ts_stack_top_state(self->stack, version), position); if (!lookahead || (position != last_position) || !ts_parser__can_reuse(self, version, lookahead)) { ts_tree_release(lookahead); - lookahead = - ts_parser__get_next_lookahead(self, version, &reusable_node); + lookahead = ts_parser__get_lookahead(self, version, &reusable_node); if (!lookahead) return NULL; } @@ -899,6 +826,7 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input, TSTree *previous_tree) { } } + ts_stack_merge(self->stack); ts_tree_release(lookahead); if (ts_stack_version_count(self->stack) == 0) { diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 76079cbf..ecdce2c5 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -114,29 +114,25 @@ static StackNode *stack_node_new(StackNode *next, TSTree *tree, bool is_pending, return node; } -static void stack_node_add_successor(StackNode *self, TSTree *new_tree, - bool is_pending, StackNode *new_node) { +static void stack_node_add_link(StackNode *self, StackLink link) { for (int i = 0; i < self->successor_count; i++) { - StackLink successor = self->successors[i]; - if (successor.tree == new_tree) { - if (successor.node == new_node) + StackLink existing_link = self->successors[i]; + if (existing_link.tree == link.tree) { + if (existing_link.node == link.node) return; - if (successor.node && new_node && - successor.node->state == new_node->state) { - for (int j = 0; j < new_node->successor_count; j++) { - stack_node_add_successor(successor.node, new_node->successors[j].tree, - is_pending, new_node->successors[j].node); - } + if (existing_link.node->state == link.node->state) { + for (int j = 0; j < link.node->successor_count; j++) + stack_node_add_link(existing_link.node, link.node->successors[j]); return; } } } if (self->successor_count < MAX_SUCCESSOR_COUNT) { - stack_node_retain(new_node); - ts_tree_retain(new_tree); + stack_node_retain(link.node); + ts_tree_retain(link.tree); self->successors[self->successor_count++] = (StackLink){ - new_node, new_tree, is_pending, + link.node, link.tree, link.is_pending, }; } } @@ -210,7 +206,7 @@ TSLength ts_stack_top_position(const Stack *self, StackVersion version) { return (*array_get(&self->heads, version))->position; } -int ts_stack_version_count(const Stack *self) { +size_t ts_stack_version_count(const Stack *self) { return self->heads.size; } @@ -248,19 +244,10 @@ static void ts_stack__merge_slice(Stack *self, StackSlice *slice, */ static StackVersion ts_stack__add_version(Stack *self, StackNode *node) { - if (array_push(&self->heads, node)) { - stack_node_retain(node); - return (StackVersion)(self->heads.size - 1); - } else { - return -1; - } -} - -static StackVersion ts_stack__find_version(Stack *self, StackNode *node) { - for (size_t i = 0; i < self->heads.size; i++) - if (self->heads.contents[i] == node) - return (StackVersion)i; - return -1; + if (!array_push(&self->heads, node)) + return STACK_VERSION_NONE; + stack_node_retain(node); + return (StackVersion)(self->heads.size - 1); } void ts_stack_remove_version(Stack *self, StackVersion version) { @@ -273,35 +260,43 @@ void ts_stack_remove_version(Stack *self, StackVersion version) { * Section: Mutating the stack (Public) */ -StackPushResult ts_stack_push(Stack *self, StackVersion version, TSTree *tree, - bool is_pending, TSStateId state) { - StackNode *current_version = *array_get(&self->heads, version); - TSLength position = - ts_length_add(current_version->position, ts_tree_total_size(tree)); - - for (int i = 0; i < version; i++) { - StackNode *prior_node = self->heads.contents[i]; - if (prior_node->state == state && - prior_node->position.chars == position.chars) { - stack_node_add_successor(prior_node, tree, is_pending, current_version); - ts_stack_remove_version(self, version); - return StackPushMerged; - } - } - - StackNode *new_version = stack_node_new(current_version, tree, is_pending, - state, position, &self->node_pool); - if (!new_version) - return StackPushFailed; - - stack_node_release(current_version, &self->node_pool); - self->heads.contents[version] = new_version; - return StackPushContinued; +bool ts_stack_push(Stack *self, StackVersion version, TSTree *tree, + bool is_pending, TSStateId state) { + StackNode *node = *array_get(&self->heads, version); + TSLength position = ts_length_add(node->position, ts_tree_total_size(tree)); + StackNode *new_node = + stack_node_new(node, tree, is_pending, state, position, &self->node_pool); + if (!new_node) + return false; + stack_node_release(node, &self->node_pool); + self->heads.contents[version] = new_node; + return true; } -StackVersion ts_stack_split(Stack *self, StackVersion version) { - StackNode *head = self->heads.contents[version]; - return ts_stack__add_version(self, head); +void ts_stack_merge(Stack *self) { + for (size_t i = 0; i < self->heads.size; i++) { + StackNode *node = self->heads.contents[i]; + for (size_t j = 0; j < i; j++) { + StackNode *prior_node = self->heads.contents[j]; + if (prior_node->state == node->state && + prior_node->position.chars == node->position.chars) { + for (size_t k = 0; k < node->successor_count; k++) { + StackLink successor = node->successors[k]; + stack_node_add_link(prior_node, successor); + } + ts_stack_remove_version(self, i--); + break; + } + } + } +} + +void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) { + assert(v2 < v1); + assert((size_t)v1 < self->heads.size); + stack_node_release(self->heads.contents[v2], &self->node_pool); + self->heads.contents[v2] = self->heads.contents[v1]; + array_erase(&self->heads, v1); } static inline ALWAYS_INLINE StackPopResult @@ -309,23 +304,20 @@ static inline ALWAYS_INLINE StackPopResult void *payload) { array_clear(&self->slices); array_clear(&self->pop_paths); - StackNode *initial_version = *array_get(&self->heads, version); - PopPath pop_path = { - .node = initial_version, + .node = *array_get(&self->heads, version), .trees = array_new(), .essential_tree_count = 0, .is_done = false, .is_pending = true, }; - if (!array_grow(&pop_path.trees, STARTING_TREE_CAPACITY)) - goto error; if (!array_push(&self->pop_paths, pop_path)) goto error; bool all_paths_done = false; for (size_t depth = 0; !all_paths_done; depth++) { all_paths_done = true; + for (size_t i = 0, size = self->pop_paths.size; i < size; i++) { PopPath *path = &self->pop_paths.contents[i]; if (path->is_done) @@ -386,39 +378,29 @@ static inline ALWAYS_INLINE StackPopResult if (!path->is_done) continue; - StackSlice slice = {.trees = path->trees, .version = -1 }; + StackSlice slice = {.trees = path->trees, .version = STACK_VERSION_NONE }; array_reverse(&slice.trees); - if (i == 0) { - stack_node_retain(path->node); - self->heads.contents[version] = path->node; - slice.version = version; - } else { - slice.version = ts_stack__find_version(self, path->node); - if (slice.version == -1) { - if ((slice.version = ts_stack__add_version(self, path->node)) == -1) - goto error; - } else { - bool merged = false; - for (size_t j = 0; j < self->slices.size; j++) { - StackSlice *prior_result = &self->slices.contents[j]; - if (prior_result->version == slice.version) { - ts_stack__merge_slice(self, prior_result, &slice); - merged = true; - break; - } - } - if (merged) - continue; + bool merged = false; + for (size_t j = 0; j < self->slices.size; j++) { + StackSlice *prior_slice = &self->slices.contents[j]; + StackNode *prior_node = self->heads.contents[prior_slice->version]; + if (prior_node == path->node) { + ts_stack__merge_slice(self, prior_slice, &slice); + merged = true; + break; } } - if (!array_push(&self->slices, slice)) - goto error; + if (!merged) { + slice.version = ts_stack__add_version(self, path->node); + if (slice.version == STACK_VERSION_NONE) + goto error; + if (!array_push(&self->slices, slice)) + goto error; + } } - if (self->slices.size) - stack_node_release(initial_version, &self->node_pool); return (StackPopResult){.status = StackPopSucceeded, .slices = self->slices }; error: @@ -478,7 +460,12 @@ static inline ALWAYS_INLINE StackIterateAction } StackPopResult ts_stack_pop_pending(Stack *self, StackVersion version) { - return stack__pop(self, version, stack__pop_pending_callback, NULL); + StackPopResult pop = stack__pop(self, version, stack__pop_pending_callback, NULL); + if (pop.slices.size > 0) { + ts_stack_renumber_version(self, pop.slices.contents[0].version, version); + pop.slices.contents[0].version = version; + } + return pop; } static inline ALWAYS_INLINE StackIterateAction @@ -492,7 +479,7 @@ TreeArray ts_stack_pop_all(Stack *self, StackVersion version) { if (pop.status != StackPopSucceeded) return (TreeArray)array_new(); assert(pop.slices.size == 1); - assert(pop.slices.contents[0].version == version); + ts_stack_renumber_version(self, pop.slices.contents[0].version, version); return pop.slices.contents[0].trees; } diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 99ccdca2..17c743c4 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -12,7 +12,9 @@ extern "C" { typedef struct Stack Stack; -typedef int StackVersion; +typedef unsigned int StackVersion; + +#define STACK_VERSION_NONE ((StackVersion)-1) typedef struct { TreeArray trees; @@ -21,12 +23,6 @@ typedef struct { typedef Array(StackSlice) StackSliceArray; -typedef enum { - StackPushFailed, - StackPushMerged, - StackPushContinued, -} StackPushResult; - typedef struct { enum { StackPopFailed, @@ -62,7 +58,7 @@ void ts_stack_delete(Stack *); /* * Get the stack's current number of versions. */ -int ts_stack_version_count(const Stack *); +size_t ts_stack_version_count(const Stack *); /* * Get the state at the top of the given version of the stack. If the stack is @@ -80,7 +76,7 @@ TSLength ts_stack_top_position(const Stack *, StackVersion); * Push a tree and state onto the given head of the stack. This could cause * the version to merge with an existing version. */ -StackPushResult ts_stack_push(Stack *, StackVersion, TSTree *, bool, TSStateId); +bool ts_stack_push(Stack *, StackVersion, TSTree *, bool, TSStateId); /* * Pop the given number of entries from the given version of the stack. This @@ -98,12 +94,9 @@ StackPopResult ts_stack_pop_pending(Stack *, StackVersion); TreeArray ts_stack_pop_all(Stack *, StackVersion); -/* - * Split the given stack head into two versions, so that the stack can be - * transformed from its current state in multiple alternative ways. Returns - * the ID of the newly-created version. - */ -StackVersion ts_stack_split(Stack *, StackVersion); +void ts_stack_merge(Stack *); + +void ts_stack_renumber_version(Stack *, StackVersion, StackVersion); /* * Remove the given version from the stack. diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 0d32dcac..5b43bede 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -51,6 +51,16 @@ void ts_tree_array_delete(TreeArray *self) { array_delete(self); } +size_t ts_tree_array_essential_count(const TreeArray *self) { + size_t result = 0; + for (size_t i = 0; i < self->size; i++) { + TSTree *tree = self->contents[i]; + if (!tree->extra && tree->symbol != ts_builtin_sym_error) + result++; + } + return result; +} + TSTree *ts_tree_make_error(TSLength size, TSLength padding, char lookahead_char) { TSTree *result = ts_tree_make_leaf(ts_builtin_sym_error, padding, size, (TSSymbolMetadata){ diff --git a/src/runtime/tree.h b/src/runtime/tree.h index 1c573aa0..83b9065f 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -46,6 +46,7 @@ struct TSTree { typedef Array(TSTree *) TreeArray; TreeArray ts_tree_array_copy(TreeArray *); void ts_tree_array_delete(TreeArray *); +size_t ts_tree_array_essential_count(const TreeArray *); TSTree *ts_tree_make_leaf(TSSymbol, TSLength, TSLength, TSSymbolMetadata); TSTree *ts_tree_make_node(TSSymbol, size_t, TSTree **, TSSymbolMetadata);