From 9d5a260e5bcfe59af6adce50320b425c2ce8c058 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 22 Feb 2016 21:29:46 -0800 Subject: [PATCH 1/7] :fire: unused variable in tree spec --- spec/runtime/tree_spec.cc | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/spec/runtime/tree_spec.cc b/spec/runtime/tree_spec.cc index 4761e94e..ea8d881e 100644 --- a/spec/runtime/tree_spec.cc +++ b/spec/runtime/tree_spec.cc @@ -14,17 +14,6 @@ enum { hog, }; -static const char *names[] = { - "ERROR", - "END", - "cat", - "dog", - "eel", - "fox", - "goat", - "hog", -}; - describe("Tree", []() { TSTree *tree1, *tree2, *parent1; TSSymbolMetadata visible = {true, true, false, true}; From b99c257e9e4cf6f9d68193f4d585a83c307aff25 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 23 Feb 2016 00:08:20 -0800 Subject: [PATCH 2/7] Make test symbol_names array accessible in specs --- spec/helpers/tree_helpers.cc | 3 +-- spec/helpers/tree_helpers.h | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/helpers/tree_helpers.cc b/spec/helpers/tree_helpers.cc index e4ae4dab..574fb801 100644 --- a/spec/helpers/tree_helpers.cc +++ b/spec/helpers/tree_helpers.cc @@ -7,14 +7,13 @@ using std::string; using std::to_string; using std::ostream; -static const char *symbol_names[24] = { +const char *symbol_names[24] = { "ERROR", "END", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen", "twenty", "twenty-one", "twenty-two", "twenty-three" }; - TSTree ** tree_array(std::vector trees) { TSTree ** result = (TSTree **)calloc(trees.size(), sizeof(TSTree *)); for (size_t i = 0; i < trees.size(); i++) diff --git a/spec/helpers/tree_helpers.h b/spec/helpers/tree_helpers.h index c151c9ba..acd3f0a2 100644 --- a/spec/helpers/tree_helpers.h +++ b/spec/helpers/tree_helpers.h @@ -5,6 +5,7 @@ #include #include +extern const char *symbol_names[24]; TSTree ** tree_array(std::vector trees); std::ostream &operator<<(std::ostream &stream, const TSTree *tree); From 6dd92c3abe96f70e60a370bb3c0b21a6cc6d5f4d Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 23 Feb 2016 00:08:55 -0800 Subject: [PATCH 3/7] Add function for rendering the stack as a DOT graph --- src/runtime/stack.c | 77 +++++++++++++++++++++++++++++++++++++++++++++ src/runtime/stack.h | 2 ++ 2 files changed, 79 insertions(+) diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 56e5eb07..19cb9639 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -475,3 +475,80 @@ void ts_stack_delete(Stack *self) { array_delete(&self->heads); ts_free(self); } + +size_t ts_stack__write_dot_graph(Stack *self, char *string, size_t n, + const char **symbol_names) { + char *cursor = string; + char **s = n > 0 ? &cursor : &string; + cursor += snprintf(*s, n, "digraph stack {\n"); + cursor += snprintf(*s, n, "rankdir=\"RL\";\n"); + cursor += snprintf(*s, n, "node_%p [label=\"0:NULL\"];\n", NULL); + + array_clear(&self->pop_paths); + for (int i = 0; i < self->heads.size; i++) + array_push(&self->pop_paths, ((PopPath){ + .node = self->heads.contents[i], + })); + + bool all_paths_done = false; + while (!all_paths_done) { + all_paths_done = true; + + for (size_t i = 0; i < self->pop_paths.size; i++) { + PopPath *path = &self->pop_paths.contents[i]; + StackNode *node = path->node; + + if (!node) + continue; + + all_paths_done = false; + + cursor += snprintf(*s, n, "node_%p [label=\"%d:%s\"];\n", node, + node->entry.state, + symbol_names[node->entry.tree->symbol]); + + path->node = node->successors[0]; + cursor += snprintf(*s, n, "node_%p -> node_%p;\n", node, node->successors[0]); + + for (int j = 1; j < node->successor_count; j++) { + if (!array_push(&self->pop_paths, *path)) + goto error; + cursor += snprintf(*s, n, "node_%p -> node_%p;\n", node, node->successors[j]); + + PopPath *next_path = array_back(&self->pop_paths); + next_path->node = node->successors[j]; + next_path->is_shared = true; + } + } + } + + cursor += snprintf(*s, n, "}\n"); + + return cursor - string; + +error: + return (size_t)-1; +} + +char *ts_stack_dot_graph(Stack *self, const char **symbol_names) { + static char SCRATCH[1]; + char *result = NULL; + size_t size = ts_stack__write_dot_graph(self, SCRATCH, 0, symbol_names) + 1; + if (size == -1) + goto error; + + result = ts_malloc(size * sizeof(char)); + if (!result) + goto error; + + size = ts_stack__write_dot_graph(self, result, size, symbol_names); + if (size == -1) + goto error; + + return result; + +error: + if (result) + ts_free(result); + return NULL; +} diff --git a/src/runtime/stack.h b/src/runtime/stack.h index 450f334a..ae76c2e6 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -119,6 +119,8 @@ void ts_stack_clear(Stack *); void ts_stack_set_tree_selection_callback(Stack *, void *, TreeSelectionFunction); +char *ts_stack_dot_graph(Stack *, const char **); + #ifdef __cplusplus } #endif From 867e1094c31dda32df0cc61a03833f00871cfe6f Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 23 Feb 2016 00:09:42 -0800 Subject: [PATCH 4/7] Fix sharing of lookahead token between parse stacks --- src/runtime/parser.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/runtime/parser.c b/src/runtime/parser.c index f36d83a3..2611f47e 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -152,8 +152,6 @@ static void ts_parser__pop_reusable_subtree(LookaheadState *state) { } static bool ts_parser__can_reuse(TSParser *self, int head, TSTree *subtree) { - if (!subtree) - return false; if (subtree->symbol == ts_builtin_sym_error) return false; if (ts_tree_is_fragile(subtree)) { @@ -777,31 +775,32 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input, TSTree *previous_tree) { for (;;) { TSTree *lookahead = NULL; - TSLength last_position, position = ts_length_zero(); + size_t last_position, position = 0; self->is_split = ts_stack_head_count(self->stack) > 1; for (int head = 0; head < ts_stack_head_count(self->stack);) { for (bool removed = false; !removed;) { last_position = position; - position = ts_stack_top_position(self->stack, head); + size_t new_position = ts_stack_top_position(self->stack, head).chars; - if (position.chars > max_position) { - max_position = position.chars; + if (new_position > max_position) { + max_position = new_position; + head++; + break; + } else if (new_position == max_position && head > 0) { head++; break; } - if (position.chars == max_position && head > 0) { - head++; - break; - } + position = new_position; LOG("process head:%d, head_count:%d, state:%d, pos:%lu", head, ts_stack_head_count(self->stack), - ts_stack_top_state(self->stack, head), position.chars); + ts_stack_top_state(self->stack, head), position); - if (position.chars != last_position.chars || + if (!lookahead || + (position != last_position) || !ts_parser__can_reuse(self, head, lookahead)) { ts_tree_release(lookahead); lookahead = ts_parser__get_next_lookahead(self, head); From b36143d7a7b721b346511b108c1afd4df8b35f12 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 23 Feb 2016 09:45:27 -0800 Subject: [PATCH 5/7] Add flag for logging dot graphs of the stack while parsing --- src/runtime/array.h | 16 ++++++--------- src/runtime/node.c | 7 +++---- src/runtime/parser.c | 46 ++++++++++++++++++++++++++++++-------------- src/runtime/stack.c | 36 +++++++++++++++++++++------------- 4 files changed, 64 insertions(+), 41 deletions(-) diff --git a/src/runtime/array.h b/src/runtime/array.h index 7390277e..10a5c1a7 100644 --- a/src/runtime/array.h +++ b/src/runtime/array.h @@ -43,13 +43,9 @@ extern "C" { array_grow((self), (self)->capacity * 2)) && \ ((self)->contents[(self)->size++] = (element), true)) -#define array_splice(self, index, old_count, new_count, new_elements) \ - array__splice((VoidArray *)(self), \ - array__elem_size(self), \ - index, \ - old_count, \ - new_count, \ - new_elements) \ +#define array_splice(self, index, old_count, new_count, new_elements) \ + array__splice((VoidArray *)(self), array__elem_size(self), index, old_count, \ + new_count, new_elements) #define array_pop(self) ((self)->contents[--(self)->size]) @@ -118,11 +114,11 @@ static inline bool array__splice(VoidArray *self, size_t element_size, char *contents = (char *)self->contents; if (self->size > old_end) - memmove(contents + new_end * element_size, - contents + old_end * element_size, + memmove(contents + new_end * element_size, contents + old_end * element_size, (self->size - old_end) * element_size); if (new_count > 0) - memcpy((contents + index * element_size), elements, new_count * element_size); + memcpy((contents + index * element_size), elements, + new_count * element_size); self->size += new_count - old_count; return true; } diff --git a/src/runtime/node.c b/src/runtime/node.c index a18c7745..cfe37580 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -207,9 +207,7 @@ TSSymbol ts_node_symbol(TSNode self) { TSSymbolIterator ts_node_symbols(TSNode self) { const TSTree *tree = ts_node__tree(self); return (TSSymbolIterator){ - .value = tree->symbol, - .done = false, - .data = (void *)tree, + .value = tree->symbol, .done = false, .data = (void *)tree, }; } @@ -280,7 +278,8 @@ char *ts_node_string(TSNode self, const TSDocument *document) { static char SCRATCH[1]; const TSTree *tree = ts_node__tree(self); const char **symbol_names = document->parser.language->symbol_names; - size_t size = ts_tree__write_to_string(tree, symbol_names, SCRATCH, 0, true, false) + 1; + size_t size = + ts_tree__write_to_string(tree, symbol_names, SCRATCH, 0, true, false) + 1; char *result = ts_malloc(size * sizeof(char)); ts_tree__write_to_string(tree, symbol_names, result, size, true, false); return result; diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 2611f47e..ee8109d5 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -15,6 +15,8 @@ * Debugging */ +bool TS_PARSER_PRINT_STACKS = false; + #define LOG(...) \ if (self->lexer.debugger.debug_fn) { \ snprintf(self->lexer.debug_buffer, TS_DEBUG_BUFFER_SIZE, __VA_ARGS__); \ @@ -22,6 +24,21 @@ TSDebugTypeParse, self->lexer.debug_buffer); \ } +#define LOG_ACTION(...) \ + LOG(__VA_ARGS__); \ + if (TS_PARSER_PRINT_STACKS) { \ + fprintf(stderr, "graph {\nlabel=\""); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\"\n}\n\n"); \ + } + +#define LOG_STACK() \ + if (TS_PARSER_PRINT_STACKS) { \ + fputs(ts_stack_dot_graph(self->stack, self->language->symbol_names), \ + stderr); \ + fputs("\n\n", stderr); \ + } + #define SYM_NAME(sym) self->language->symbol_names[sym] #define BOOL_STRING(value) (value ? "true" : "false") @@ -345,8 +362,8 @@ static ParseActionResult ts_parser__reduce(TSParser *self, int head, } size_t child_count = pop_result.trees.size - trailing_extra_count; - parent = - ts_tree_make_node(symbol, child_count, pop_result.trees.contents, metadata); + parent = ts_tree_make_node(symbol, child_count, pop_result.trees.contents, + metadata); if (!parent) { for (size_t i = 0; i < pop_result.trees.size; i++) ts_tree_release(pop_result.trees.contents[i]); @@ -637,7 +654,7 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self, int head, current_head = head; } else { current_head = ts_parser__split(self, head); - LOG("split_action from_head:%d, new_head:%d", head, current_head); + LOG_ACTION("split_action from_head:%d, new_head:%d", head, current_head); } LookaheadState *lookahead_state = @@ -648,9 +665,11 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self, int head, if (lookahead->symbol == ts_builtin_sym_error) action.type = TSParseActionTypeError; + LOG_STACK(); + switch (action.type) { case TSParseActionTypeError: - LOG("error_sym"); + LOG_ACTION("error_sym"); if (lookahead_state->is_verifying) { ts_parser__breakdown_top_of_stack(self, current_head); lookahead_state->is_verifying = false; @@ -667,17 +686,17 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self, int head, return ts_parser__accept(self, current_head); } } else { - LOG("bail current_head:%d", current_head); + LOG_ACTION("bail current_head:%d", current_head); ts_parser__remove_head(self, current_head); return RemovedStackHead; } case TSParseActionTypeShift: if (action.extra) { - LOG("shift_extra"); + LOG_ACTION("shift_extra"); return ts_parser__shift_extra(self, current_head, state, lookahead); } else { - LOG("shift state:%u", action.data.to_state); + LOG_ACTION("shift state:%u", action.data.to_state); lookahead_state->is_verifying = (lookahead->child_count > 0); TSStateId state = action.data.to_state; return ts_parser__shift(self, current_head, state, lookahead); @@ -687,13 +706,13 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self, int head, lookahead_state->is_verifying = false; if (action.extra) { - LOG("reduce_extra sym:%s", SYM_NAME(action.data.symbol)); + LOG_ACTION("reduce_extra sym:%s", SYM_NAME(action.data.symbol)); ts_parser__reduce(self, current_head, action.data.symbol, 1, true, false, false); } else { - LOG("reduce sym:%s, child_count:%u, fragile:%s", - SYM_NAME(action.data.symbol), action.data.child_count, - BOOL_STRING(action.fragile)); + LOG_ACTION("reduce sym:%s, child_count:%u, fragile:%s", + SYM_NAME(action.data.symbol), action.data.child_count, + BOOL_STRING(action.fragile)); switch (ts_parser__reduce(self, current_head, action.data.symbol, action.data.child_count, false, action.fragile, false)) { @@ -710,7 +729,7 @@ static ParseActionResult ts_parser__consume_lookahead(TSParser *self, int head, break; case TSParseActionTypeAccept: - LOG("accept"); + LOG_ACTION("accept"); return ts_parser__accept(self, current_head); } } @@ -799,8 +818,7 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input, TSTree *previous_tree) { ts_stack_head_count(self->stack), ts_stack_top_state(self->stack, head), position); - if (!lookahead || - (position != last_position) || + if (!lookahead || (position != last_position) || !ts_parser__can_reuse(self, head, lookahead)) { ts_tree_release(lookahead); lookahead = ts_parser__get_next_lookahead(self, head); diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 19cb9639..c98eab34 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -392,8 +392,7 @@ StackPopResultArray ts_stack_pop(Stack *self, int head_index, int child_count, array_reverse(&path->trees); StackPopResult result = { - .trees = path->trees, - .head_index = -1, + .trees = path->trees, .head_index = -1, }; if (i == 0) { @@ -476,19 +475,27 @@ void ts_stack_delete(Stack *self) { ts_free(self); } +static const char *graph_colors[] = { + "red", "blue", "orange", "green", "purple", +}; + size_t ts_stack__write_dot_graph(Stack *self, char *string, size_t n, const char **symbol_names) { char *cursor = string; char **s = n > 0 ? &cursor : &string; cursor += snprintf(*s, n, "digraph stack {\n"); cursor += snprintf(*s, n, "rankdir=\"RL\";\n"); - cursor += snprintf(*s, n, "node_%p [label=\"0:NULL\"];\n", NULL); array_clear(&self->pop_paths); - for (int i = 0; i < self->heads.size; i++) + for (size_t i = 0; i < self->heads.size; i++) { + StackNode *node = self->heads.contents[i]; + const char *color = + graph_colors[i % (sizeof(graph_colors) / sizeof(graph_colors[0]))]; + cursor += snprintf(*s, n, "node_%p [color=%s];\n", node, color); array_push(&self->pop_paths, ((PopPath){ - .node = self->heads.contents[i], - })); + .node = node, + })); + } bool all_paths_done = false; while (!all_paths_done) { @@ -503,17 +510,19 @@ size_t ts_stack__write_dot_graph(Stack *self, char *string, size_t n, all_paths_done = false; - cursor += snprintf(*s, n, "node_%p [label=\"%d:%s\"];\n", node, - node->entry.state, - symbol_names[node->entry.tree->symbol]); + cursor += + snprintf(*s, n, "node_%p [label=\"%s\\n%d\"];\n", node, + symbol_names[node->entry.tree->symbol], node->entry.state); path->node = node->successors[0]; - cursor += snprintf(*s, n, "node_%p -> node_%p;\n", node, node->successors[0]); + cursor += + snprintf(*s, n, "node_%p -> node_%p;\n", node, node->successors[0]); for (int j = 1; j < node->successor_count; j++) { if (!array_push(&self->pop_paths, *path)) goto error; - cursor += snprintf(*s, n, "node_%p -> node_%p;\n", node, node->successors[j]); + cursor += + snprintf(*s, n, "node_%p -> node_%p;\n", node, node->successors[j]); PopPath *next_path = array_back(&self->pop_paths); next_path->node = node->successors[j]; @@ -522,6 +531,7 @@ size_t ts_stack__write_dot_graph(Stack *self, char *string, size_t n, } } + cursor += snprintf(*s, n, "node_%p [label=\"-\\n0\"];\n", NULL); cursor += snprintf(*s, n, "}\n"); return cursor - string; @@ -534,7 +544,7 @@ char *ts_stack_dot_graph(Stack *self, const char **symbol_names) { static char SCRATCH[1]; char *result = NULL; size_t size = ts_stack__write_dot_graph(self, SCRATCH, 0, symbol_names) + 1; - if (size == -1) + if (size == (size_t)-1) goto error; result = ts_malloc(size * sizeof(char)); @@ -542,7 +552,7 @@ char *ts_stack_dot_graph(Stack *self, const char **symbol_names) { goto error; size = ts_stack__write_dot_graph(self, result, size, symbol_names); - if (size == -1) + if (size == (size_t)-1) goto error; return result; From abbc28295050c0b08b6730ba8da3b5d81291177a Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 23 Feb 2016 11:16:50 -0800 Subject: [PATCH 6/7] Add a public function for printing debugging graphs --- include/tree_sitter/runtime.h | 1 + src/runtime/document.c | 4 ++++ src/runtime/parser.c | 6 ++---- src/runtime/parser.h | 1 + 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index fb24d6f4..e742ace1 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -91,6 +91,7 @@ void ts_document_set_input(TSDocument *, TSInput); void ts_document_set_input_string(TSDocument *, const char *); TSDebugger ts_document_debugger(const TSDocument *); void ts_document_set_debugger(TSDocument *, TSDebugger); +void ts_document_print_debugging_graphs(TSDocument *, bool); void ts_document_edit(TSDocument *, TSInputEdit); int ts_document_parse(TSDocument *); void ts_document_invalidate(TSDocument *); diff --git a/src/runtime/document.c b/src/runtime/document.c index c852b816..3d0ef2a1 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -52,6 +52,10 @@ void ts_document_set_debugger(TSDocument *self, TSDebugger debugger) { ts_parser_set_debugger(&self->parser, debugger); } +void ts_document_print_debugging_graphs(TSDocument *self, bool should_print) { + self->parser.print_debugging_graphs = should_print; +} + TSInput ts_document_input(TSDocument *self) { return self->input; } diff --git a/src/runtime/parser.c b/src/runtime/parser.c index ee8109d5..0bb3cb9e 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -15,8 +15,6 @@ * Debugging */ -bool TS_PARSER_PRINT_STACKS = false; - #define LOG(...) \ if (self->lexer.debugger.debug_fn) { \ snprintf(self->lexer.debug_buffer, TS_DEBUG_BUFFER_SIZE, __VA_ARGS__); \ @@ -26,14 +24,14 @@ bool TS_PARSER_PRINT_STACKS = false; #define LOG_ACTION(...) \ LOG(__VA_ARGS__); \ - if (TS_PARSER_PRINT_STACKS) { \ + if (self->print_debugging_graphs) { \ fprintf(stderr, "graph {\nlabel=\""); \ fprintf(stderr, __VA_ARGS__); \ fprintf(stderr, "\"\n}\n\n"); \ } #define LOG_STACK() \ - if (TS_PARSER_PRINT_STACKS) { \ + if (self->print_debugging_graphs) { \ fputs(ts_stack_dot_graph(self->stack, self->language->symbol_names), \ stderr); \ fputs("\n\n", stderr); \ diff --git a/src/runtime/parser.h b/src/runtime/parser.h index 73223e97..d9a60f6f 100644 --- a/src/runtime/parser.h +++ b/src/runtime/parser.h @@ -18,6 +18,7 @@ typedef struct { Array(TSTree *) reduce_parents; TSTree *finished_tree; bool is_split; + bool print_debugging_graphs; } TSParser; bool ts_parser_init(TSParser *); From da2ef7ad357ec99c238a032fd9dd6f109412ade5 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 23 Feb 2016 17:35:50 -0800 Subject: [PATCH 7/7] Store trees in the links between stack nodes, not in the nodes themselves --- spec/integration/corpus_specs.cc | 1 + spec/runtime/stack_spec.cc | 364 ++++++++++++++----------------- src/runtime/array.h | 2 +- src/runtime/parser.c | 76 +++---- src/runtime/stack.c | 196 +++++++++-------- src/runtime/stack.h | 7 - 6 files changed, 295 insertions(+), 351 deletions(-) diff --git a/spec/integration/corpus_specs.cc b/spec/integration/corpus_specs.cc index 8f264e8f..8f8d9944 100644 --- a/spec/integration/corpus_specs.cc +++ b/spec/integration/corpus_specs.cc @@ -112,6 +112,7 @@ describe("The Corpus", []() { document = ts_document_make(); ts_document_set_language(document, get_test_language(language_name)); // ts_document_set_debugger(document, log_debugger_make(true)); + // ts_document_print_debugging_graphs(document, true); }); after_each([&]() { diff --git a/spec/runtime/stack_spec.cc b/spec/runtime/stack_spec.cc index eda3442f..32640561 100644 --- a/spec/runtime/stack_spec.cc +++ b/spec/runtime/stack_spec.cc @@ -12,7 +12,8 @@ enum { }; enum { - symbol0, symbol1, symbol2, symbol3, symbol4, symbol5, symbol6, symbol7, symbol8 + symbol0, symbol1, symbol2, symbol3, symbol4, symbol5, symbol6, symbol7, symbol8, + symbol9, symbol10 }; struct TreeSelectionSpy { @@ -62,7 +63,7 @@ START_TEST describe("Stack", [&]() { Stack *stack; - const size_t tree_count = 10; + const size_t tree_count = 11; TSTree *trees[tree_count]; TreeSelectionSpy tree_selection_spy{0, NULL, {NULL, NULL}}; TSLength tree_len = {2, 3, 0, 3}; @@ -99,29 +100,29 @@ describe("Stack", [&]() { AssertThat(ts_stack_head(stack, 0), Equals(nullptr)); /* - * A0. + * . <--0-- A* */ ts_stack_push(stack, 0, stateA, trees[0]); const StackEntry *entry1 = ts_stack_head(stack, 0); - AssertThat(*entry1, Equals({trees[0], stateA, tree_len})); + AssertThat(*entry1, Equals({stateA, tree_len})); AssertThat(ts_stack_entry_next_count(entry1), Equals(1)); AssertThat(ts_stack_entry_next(entry1, 0), Equals(nullptr)); /* - * A0__B1. + * . <--0-- A <--1-- B* */ ts_stack_push(stack, 0, stateB, trees[1]); const StackEntry *entry2 = ts_stack_head(stack, 0); - AssertThat(*entry2, Equals({trees[1], stateB, tree_len * 2})); + AssertThat(*entry2, Equals({stateB, tree_len * 2})); AssertThat(ts_stack_entry_next_count(entry2), Equals(1)); AssertThat(ts_stack_entry_next(entry2, 0), Equals(entry1)); /* - * A0__B1__C2. + * . <--0-- A <--1-- B <--2-- C* */ ts_stack_push(stack, 0, stateC, trees[2]); const StackEntry *entry3 = ts_stack_head(stack, 0); - AssertThat(*entry3, Equals({trees[2], stateC, tree_len * 3})); + AssertThat(*entry3, Equals({stateC, tree_len * 3})); AssertThat(ts_stack_entry_next_count(entry3), Equals(1)); AssertThat(ts_stack_entry_next(entry3, 0), Equals(entry2)); }); @@ -130,7 +131,7 @@ describe("Stack", [&]() { describe("popping nodes from the stack", [&]() { before_each([&]() { /* - * A0__B1__C2. + * . <--0-- A <--1-- B <--2-- C* */ ts_stack_push(stack, 0, stateA, trees[0]); ts_stack_push(stack, 0, stateB, trees[1]); @@ -139,14 +140,14 @@ describe("Stack", [&]() { it("removes the given number of nodes from the stack", [&]() { /* - * A0. + * . <--0-- A* */ StackPopResultArray results = ts_stack_pop(stack, 0, 2, false); AssertThat(results.size, Equals(1)); StackPopResult result = results.contents[0]; AssertThat(result.trees, Equals(vector({ trees[1], trees[2] }))); - AssertThat(*ts_stack_head(stack, 0), Equals({trees[0], stateA, tree_len})); + AssertThat(*ts_stack_head(stack, 0), Equals({stateA, tree_len})); free_pop_results(&results); /* @@ -165,6 +166,9 @@ describe("Stack", [&]() { it("does not count 'extra' trees toward the count", [&]() { trees[1]->extra = true; + /* + * . + */ StackPopResultArray results = ts_stack_pop(stack, 0, 2, false); AssertThat(results.size, Equals(1)); @@ -176,6 +180,9 @@ describe("Stack", [&]() { }); it("pops the entire stack when given a negative count", [&]() { + /* + * . + */ StackPopResultArray results = ts_stack_pop(stack, 0, -1, false); AssertThat(results.size, Equals(1)); @@ -189,49 +196,58 @@ describe("Stack", [&]() { describe("splitting the stack", [&]() { it("creates a new independent head with the same entries", [&]() { /* - * A0__B1__C2. + * . <--0-- A <--1-- B <--2-- C* */ ts_stack_push(stack, 0, stateA, trees[0]); ts_stack_push(stack, 0, stateB, trees[1]); ts_stack_push(stack, 0, stateC, trees[2]); + /* + * . <--0-- A <--1-- B <--2-- C* + * ↑ + * `-* + */ int new_index = ts_stack_split(stack, 0); AssertThat(ts_stack_head_count(stack), Equals(2)); AssertThat(new_index, Equals(1)); + AssertThat(ts_stack_top_state(stack, 1), Equals(stateC)); /* - * A0__B1__C2__D3. - * \. + * . <--0-- A <--1-- B <--2-- C <--3-- D* + * ↑ + * `-* */ ts_stack_push(stack, 0, stateD, trees[3]); StackPopResultArray pop_results = ts_stack_pop(stack, 1, 1, false); AssertThat(ts_stack_head_count(stack), Equals(2)); - AssertThat(*ts_stack_head(stack, 0), Equals({trees[3], stateD, tree_len * 4})); - AssertThat(*ts_stack_head(stack, 1), Equals({trees[1], stateB, tree_len * 2})); + AssertThat(*ts_stack_head(stack, 0), Equals({stateD, tree_len * 4})); + AssertThat(*ts_stack_head(stack, 1), Equals({stateB, tree_len * 2})); AssertThat(pop_results.size, Equals(1)); StackPopResult pop_result = pop_results.contents[0]; AssertThat(pop_result.trees.size, Equals(1)); free_pop_results(&pop_results); /* - * A0__B1__C2__D3. - * \__E4__F3. + * . <--0-- A <--1-- B <--2-- C <--3-- D* + * ↑ + * `---4--- E <--5-- F* */ ts_stack_push(stack, 1, stateE, trees[4]); - ts_stack_push(stack, 1, stateF, trees[3]); + ts_stack_push(stack, 1, stateF, trees[5]); AssertThat(ts_stack_head_count(stack), Equals(2)); - AssertThat(*ts_stack_head(stack, 0), Equals({trees[3], stateD, tree_len * 4})); - AssertThat(*ts_stack_head(stack, 1), Equals({trees[3], stateF, tree_len * 4})); + AssertThat(*ts_stack_head(stack, 0), Equals({stateD, tree_len * 4})); + AssertThat(*ts_stack_head(stack, 1), Equals({stateF, tree_len * 4})); }); }); describe("pushing the same state onto two different heads of the stack", [&]() { before_each([&]() { /* - * A0__B1__C2__D3. - * \__E4__F5. + * . <--0-- A <--1-- B <--2-- C <--3-- D* + * ↑ + * `---4--- E <--5-- F* */ ts_stack_push(stack, 0, stateA, trees[0]); ts_stack_push(stack, 0, stateB, trees[1]); @@ -242,73 +258,52 @@ describe("Stack", [&]() { ts_stack_push(stack, 1, stateF, trees[5]); AssertThat(ts_stack_head_count(stack), Equals(2)); - AssertThat(*ts_stack_head(stack, 0), Equals({trees[3], stateD, tree_len * 4})); - AssertThat(*ts_stack_head(stack, 1), Equals({trees[5], stateF, tree_len * 4})); + AssertThat(*ts_stack_head(stack, 0), Equals({stateD, tree_len * 4})); + AssertThat(*ts_stack_head(stack, 1), Equals({stateF, tree_len * 4})); }); - describe("when the trees are identical", [&]() { - it("merges the heads", [&]() { - /* - * A0__B1__C2__D3__G6. - * \__E4__F5__/ - */ - AssertThat(ts_stack_push(stack, 0, stateG, trees[6]), Equals(StackPushResultContinued)); - AssertThat(ts_stack_push(stack, 1, stateG, trees[6]), Equals(StackPushResultMerged)); + it("merges the heads", [&]() { + /* + * . <--0-- A <--1-- B <--2-- C <--3-- D <--6-- G* + * ^ | + * `---4--- E <--5-- F <--7---' + */ + AssertThat(ts_stack_push(stack, 0, stateG, trees[6]), Equals(StackPushResultContinued)); + AssertThat(ts_stack_push(stack, 1, stateG, trees[7]), Equals(StackPushResultMerged)); - AssertThat(ts_stack_head_count(stack), Equals(1)); - const StackEntry *entry1 = ts_stack_head(stack, 0); - AssertThat(*entry1, Equals({trees[6], stateG, tree_len * 5})); - AssertThat(ts_stack_entry_next_count(entry1), Equals(2)); - AssertThat(*ts_stack_entry_next(entry1, 0), Equals({trees[3], stateD, tree_len * 4})); - AssertThat(*ts_stack_entry_next(entry1, 1), Equals({trees[5], stateF, tree_len * 4})); - }); + AssertThat(ts_stack_head_count(stack), Equals(1)); + const StackEntry *entry1 = ts_stack_head(stack, 0); + AssertThat(*entry1, Equals({stateG, tree_len * 5})); + AssertThat(ts_stack_entry_next_count(entry1), Equals(2)); + AssertThat(*ts_stack_entry_next(entry1, 0), Equals({stateD, tree_len * 4})); + AssertThat(*ts_stack_entry_next(entry1, 1), Equals({stateF, tree_len * 4})); }); - describe("when the trees are different", [&]() { - before_each([&]() { - tree_selection_spy.tree_to_return = trees[7]; - AssertThat(tree_selection_spy.call_count, Equals(0)); - }); - - it("merges the heads, selecting the tree with the tree selection callback", [&]() { + describe("when the merged nodes share a successor", [&]() { + it("recursively merges the successor nodes", [&]() { /* - * A0__B1__C2__D3__G(6|7) - * \__E4__F5____/ - */ - AssertThat(ts_stack_push(stack, 0, stateG, trees[6]), Equals(StackPushResultContinued)); - AssertThat(ts_stack_push(stack, 1, stateG, trees[7]), Equals(StackPushResultMerged)); - - AssertThat(ts_stack_head_count(stack), Equals(1)); - AssertThat(tree_selection_spy.call_count, Equals(1)); - AssertThat(tree_selection_spy.arguments[0], Equals(trees[6])); - AssertThat(tree_selection_spy.arguments[1], Equals(trees[7])); - AssertThat(*ts_stack_head(stack, 0), Equals({trees[7], stateG, tree_len * 5})); - }); - }); - - describe("when successor nodes of the merged nodes have the same state", [&]() { - it("recursively merges those successor nodes", [&]() { - /* - * A0__B1__C2__D3__G6__H7. - * \__E4__F5__G6. + * . <--0-- A <--1-- B <--2-- C <--3-- D <--6-- G <--7--H* + * ↑ + * `---4--- E <--5-- F <--8-- G* */ AssertThat(ts_stack_push(stack, 0, stateG, trees[6]), Equals(StackPushResultContinued)); AssertThat(ts_stack_push(stack, 0, stateH, trees[7]), Equals(StackPushResultContinued)); AssertThat(ts_stack_push(stack, 1, stateG, trees[6]), Equals(StackPushResultContinued)); /* - * A0__B1__C2__D3__G6__H7. - * \__E4__F5_/ + * . <--0-- A <--1-- B <--2-- C <--3-- D <--6-- G <--7--H* + * ↑ | + * `---4--- E <--5-- F <--8---' */ AssertThat(ts_stack_push(stack, 1, stateH, trees[7]), Equals(StackPushResultMerged)); AssertThat(ts_stack_head_count(stack), Equals(1)); StackEntry *head = ts_stack_head(stack, 0); - AssertThat(*head, Equals({trees[7], stateH, tree_len * 6})) + AssertThat(*head, Equals({stateH, tree_len * 6})) AssertThat(ts_stack_entry_next_count(head), Equals(1)); StackEntry *next = ts_stack_entry_next(head, 0); - AssertThat(*next, Equals({trees[6], stateG, tree_len * 5})) + AssertThat(*next, Equals({stateG, tree_len * 5})) AssertThat(ts_stack_entry_next_count(next), Equals(2)); }); }); @@ -319,27 +314,24 @@ describe("Stack", [&]() { ts_tree_retain(trees[3]); TSTree *parent = ts_tree_make_node(5, 2, tree_array({ trees[2], trees[3] }), metadata); - tree_selection_spy.tree_to_return = parent; - tree_selection_spy.call_count = 0; - /* - * .__C5. - * B2.__/ + * . <--2-- B <--3-- C + * ^ | + * `--------5--------' */ ts_stack_clear(stack); ts_stack_split(stack, 0); AssertThat(ts_stack_push(stack, 0, stateC, parent), Equals(StackPushResultContinued)); AssertThat(ts_stack_push(stack, 1, stateB, trees[2]), Equals(StackPushResultContinued)); AssertThat(ts_stack_push(stack, 1, stateC, trees[3]), Equals(StackPushResultMerged)); - AssertThat(tree_selection_spy.call_count, Equals(1)); AssertThat(ts_stack_head_count(stack), Equals(1)); StackEntry *head = ts_stack_head(stack, 0); - AssertThat(*head, Equals({parent, stateC, tree_len * 2})); + AssertThat(*head, Equals({stateC, tree_len * 2})); AssertThat(ts_stack_entry_next_count(head), Equals(2)); AssertThat(ts_stack_entry_next(head, 0), Equals(nullptr)); - AssertThat(*ts_stack_entry_next(head, 1), Equals({trees[2], stateB, tree_len})); + AssertThat(*ts_stack_entry_next(head, 1), Equals({stateB, tree_len})); ts_tree_release(parent); }); @@ -349,43 +341,48 @@ describe("Stack", [&]() { describe("popping from a stack head that has been merged", [&]() { before_each([&]() { /* - * A0__B1__C2__D3__G6. - * \__E4__F5__/ + * . <--0-- A <--1-- B <--2-- C <--3-- D <--4-- E* + * ^ | + * `---5--- F <--6-- G <--7---' */ ts_stack_push(stack, 0, stateA, trees[0]); ts_stack_push(stack, 0, stateB, trees[1]); ts_stack_split(stack, 0); ts_stack_push(stack, 0, stateC, trees[2]); ts_stack_push(stack, 0, stateD, trees[3]); - ts_stack_push(stack, 0, stateG, trees[6]); - ts_stack_push(stack, 1, stateE, trees[4]); + ts_stack_push(stack, 0, stateE, trees[4]); ts_stack_push(stack, 1, stateF, trees[5]); ts_stack_push(stack, 1, stateG, trees[6]); + ts_stack_push(stack, 1, stateE, trees[7]); AssertThat(ts_stack_head_count(stack), Equals(1)); + AssertThat(ts_stack_top_state(stack, 0), Equals(stateE)); AssertThat(ts_stack_entry_next_count(ts_stack_head(stack, 0)), Equals(2)); }); describe("when there are two paths that lead to two different heads", [&]() { it("returns an entry for each revealed head", [&]() { /* - * A0__B1__C2. - * \__E4. + * . <--0-- A <--1-- B <--2-- C* + * ^ + * `---5--- F* */ StackPopResultArray results = ts_stack_pop(stack, 0, 2, false); AssertThat(results.size, Equals(2)); - StackPopResult pop1 = results.contents[0]; - AssertThat(pop1.head_index, Equals(0)); - AssertThat(pop1.trees, Equals(vector({ trees[3], trees[6] }))); + StackPopResult result1 = results.contents[0]; + AssertThat(result1.head_index, Equals(0)); + AssertThat(ts_stack_top_state(stack, 0), Equals(stateC)); + AssertThat(result1.trees, Equals(vector({ trees[3], trees[4] }))); - StackPopResult pop2 = results.contents[1]; - AssertThat(pop2.head_index, Equals(1)); - AssertThat(pop2.trees, Equals(vector({ trees[5], trees[6] }))); + StackPopResult result2 = results.contents[1]; + AssertThat(result2.head_index, Equals(1)); + AssertThat(ts_stack_top_state(stack, 1), Equals(stateF)); + AssertThat(result2.trees, Equals(vector({ trees[6], trees[7] }))); AssertThat(ts_stack_head_count(stack), Equals(2)); - AssertThat(*ts_stack_head(stack, 0), Equals({trees[2], stateC, tree_len * 3})); - AssertThat(*ts_stack_head(stack, 1), Equals({trees[4], stateE, tree_len * 3})); + AssertThat(*ts_stack_head(stack, 0), Equals({stateC, tree_len * 3})); + AssertThat(*ts_stack_head(stack, 1), Equals({stateF, tree_len * 3})); free_pop_results(&results); }); @@ -394,50 +391,27 @@ describe("Stack", [&]() { describe("when there is one path, leading to one head", [&]() { it("returns a single entry", [&]() { /* - * A0__B1__C2__D3__G6__H7. - * \__E4__F5__/ + * . <--0-- A <--1-- B <--2-- C <--3-- D <--4-- E <--8--H* + * ^ | + * `---5--- F <--6-- G <--7---' */ - AssertThat(ts_stack_push(stack, 0, stateH, trees[7]), Equals(StackPushResultContinued)); + AssertThat(ts_stack_push(stack, 0, stateH, trees[8]), Equals(StackPushResultContinued)); AssertThat(ts_stack_head_count(stack), Equals(1)); + AssertThat(ts_stack_top_state(stack, 0), Equals(stateH)); /* - * A0__B1__C2__D3__G6. - * \__E4__F5__/ + * . <--0-- A <--1-- B <--2-- C <--3-- D <--4-- E* + * ^ | + * `---5--- F <--6-- G <--7---' */ StackPopResultArray results = ts_stack_pop(stack, 0, 1, false); - AssertThat(results.size, Equals(1)); + StackPopResult result1 = results.contents[0]; + AssertThat(result1.head_index, Equals(0)); + AssertThat(result1.trees, Equals(vector({ trees[8] }))); + AssertThat(ts_stack_head_count(stack), Equals(1)); - - free_pop_results(&results); - }); - }); - - describe("when there is one path that leads to two different heads", [&]() { - it("returns two entries with the same array of trees", [&]() { - /* - * A0__B1__C2__D3__G6__H7. - * \__E4__F5__/ - */ - ts_stack_push(stack, 0, stateH, trees[7]); - - - /* - * A0__B1__C2__D3. - * \__E4__F5. - */ - StackPopResultArray results = ts_stack_pop(stack, 0, 2, false); - AssertThat(ts_stack_head_count(stack), Equals(2)); - - AssertThat(results.size, Equals(2)); - StackPopResult pop1 = results.contents[0]; - AssertThat(pop1.head_index, Equals(0)); - AssertThat(pop1.trees, Equals(vector({ trees[6], trees[7] }))); - - StackPopResult pop2 = results.contents[1]; - AssertThat(pop2.head_index, Equals(1)); - AssertThat(pop2.trees, Equals(vector({ trees[6], trees[7] }))); - AssertThat(pop2.trees.contents, Equals(pop1.trees.contents)); + AssertThat(ts_stack_top_state(stack, 0), Equals(stateE)); free_pop_results(&results); }); @@ -449,16 +423,16 @@ describe("Stack", [&]() { tree_selection_spy.tree_to_return = trees[2]; /* - * A0__B1. + * . <--0-- A <--1-- B* */ StackPopResultArray results = ts_stack_pop(stack, 0, 3, false); AssertThat(ts_stack_head_count(stack), Equals(1)); - AssertThat(*ts_stack_head(stack, 0), Equals({trees[1], stateB, tree_len * 2})); + AssertThat(*ts_stack_head(stack, 0), Equals({stateB, tree_len * 2})); AssertThat(results.size, Equals(1)); - StackPopResult pop1 = results.contents[0]; - AssertThat(pop1.head_index, Equals(0)); - AssertThat(pop1.trees, Equals(vector({ trees[2], trees[3], trees[ 6] }))); + StackPopResult result1 = results.contents[0]; + AssertThat(result1.head_index, Equals(0)); + AssertThat(result1.trees, Equals(vector({ trees[2], trees[3], trees[4] }))); free_pop_results(&results); }); @@ -469,16 +443,16 @@ describe("Stack", [&]() { tree_selection_spy.tree_to_return = trees[4]; /* - * A0__B1. + * . <--0-- A <--1-- B* */ StackPopResultArray results = ts_stack_pop(stack, 0, 3, false); AssertThat(ts_stack_head_count(stack), Equals(1)); - AssertThat(*ts_stack_head(stack, 0), Equals({trees[1], stateB, tree_len * 2})); + AssertThat(*ts_stack_head(stack, 0), Equals({stateB, tree_len * 2})); AssertThat(results.size, Equals(1)); - StackPopResult pop1 = results.contents[0]; - AssertThat(pop1.head_index, Equals(0)); - AssertThat(pop1.trees, Equals(vector({ trees[4], trees[5], trees[6] }))) + StackPopResult result1 = results.contents[0]; + AssertThat(result1.head_index, Equals(0)); + AssertThat(result1.trees, Equals(vector({ trees[5], trees[6], trees[7] }))) free_pop_results(&results); }); @@ -489,91 +463,59 @@ describe("Stack", [&]() { describe("popping from a stack head that has been 3-way merged", [&]() { before_each([&]() { /* - * A0__B1__C2__D3__I8__J9. - * \__E4__F5__/ - * \__G6__H7__/ + * . <--0-- A <--1-- B <--2-- C <--3-- D <--10-- I + * ^ | + * `---4--- E <--5-- F <--6---' + * | | + * `---7--- G <--8-- H <--9---' */ ts_stack_clear(stack); ts_stack_push(stack, 0, stateA, trees[0]); - ts_stack_push(stack, 0, stateB, trees[1]); ts_stack_split(stack, 0); ts_stack_split(stack, 1); + ts_stack_push(stack, 0, stateB, trees[1]); ts_stack_push(stack, 0, stateC, trees[2]); - ts_stack_push(stack, 1, stateE, trees[4]); - ts_stack_push(stack, 2, stateG, trees[6]); ts_stack_push(stack, 0, stateD, trees[3]); + ts_stack_push(stack, 1, stateE, trees[4]); ts_stack_push(stack, 1, stateF, trees[5]); - ts_stack_push(stack, 2, stateH, trees[7]); - ts_stack_push(stack, 0, stateI, trees[8]); - ts_stack_push(stack, 1, stateI, trees[8]); - ts_stack_push(stack, 1, stateI, trees[8]); - ts_stack_push(stack, 0, stateJ, trees[9]); - + ts_stack_push(stack, 1, stateD, trees[6]); + ts_stack_push(stack, 1, stateG, trees[7]); + ts_stack_push(stack, 1, stateH, trees[8]); + ts_stack_push(stack, 1, stateD, trees[9]); AssertThat(ts_stack_head_count(stack), Equals(1)); - StackEntry *head = ts_stack_head(stack, 0); - AssertThat(ts_stack_entry_next_count(head), Equals(1)); - AssertThat(ts_stack_entry_next_count(ts_stack_entry_next(head, 0)), Equals(3)); + AssertThat(ts_stack_entry_next_count(ts_stack_head(stack, 0)), Equals(3)); + ts_stack_push(stack, 0, stateI, trees[10]); + AssertThat(ts_stack_entry_next_count(ts_stack_head(stack, 0)), Equals(1)); }); - describe("when there is one path that leads to three different heads", [&]() { - it("returns three entries with the same array of trees", [&]() { + describe("when there are three different paths that lead to three different heads", [&]() { + it("returns three entries with different arrays of trees", [&]() { /* - * A0__B1__C2__D3. - * \__E4__F5. - * \__G6__H7. + * . <--0-- A <--1-- B <--2-- C* + * ^ + * `---4--- E <--5-- F* + * | + * `---7--- G <--8-- H* */ StackPopResultArray results = ts_stack_pop(stack, 0, 2, false); AssertThat(ts_stack_head_count(stack), Equals(3)); AssertThat(results.size, Equals(3)); - StackPopResult pop1 = results.contents[0]; - AssertThat(ts_stack_top_tree(stack, 0), Equals(trees[3])); - AssertThat(pop1.head_index, Equals(0)); - AssertThat(pop1.trees, Equals(vector({ trees[8], trees[9] }))) + StackPopResult result1 = results.contents[0]; + AssertThat(ts_stack_top_state(stack, 0), Equals(stateC)); + AssertThat(result1.head_index, Equals(0)); + AssertThat(result1.trees, Equals(vector({ trees[3], trees[10] }))) - StackPopResult pop2 = results.contents[1]; - AssertThat(ts_stack_top_tree(stack, 1), Equals(trees[5])); - AssertThat(pop2.head_index, Equals(1)); - AssertThat(pop2.trees.size, Equals(2)); - AssertThat(pop2.trees.contents, Equals(pop1.trees.contents)); + StackPopResult result2 = results.contents[1]; + AssertThat(ts_stack_top_state(stack, 1), Equals(stateF)); + AssertThat(result2.head_index, Equals(1)); + AssertThat(result2.trees, Equals(vector({ trees[6], trees[10] }))) - StackPopResult pop3 = results.contents[2]; - AssertThat(ts_stack_top_tree(stack, 2), Equals(trees[7])); - AssertThat(pop3.head_index, Equals(2)); - AssertThat(pop3.trees.size, Equals(2)); - AssertThat(pop3.trees.contents, Equals(pop1.trees.contents)); - - free_pop_results(&results); - }); - }); - - describe("when there are three different paths that lead to three different heads", [&]() { - it("returns three entries with different arrays of trees", [&]() { - /* - * A0__B1__C2. - * \__E4. - * \__G6. - */ - StackPopResultArray results = ts_stack_pop(stack, 0, 3, false); - AssertThat(ts_stack_head_count(stack), Equals(3)); - - AssertThat(results.size, Equals(3)); - - StackPopResult pop1 = results.contents[0]; - AssertThat(ts_stack_top_tree(stack, 0), Equals(trees[2])); - AssertThat(pop1.head_index, Equals(0)); - AssertThat(pop1.trees, Equals(vector({ trees[3], trees[8], trees[9] }))) - - StackPopResult pop2 = results.contents[1]; - AssertThat(ts_stack_top_tree(stack, 1), Equals(trees[4])); - AssertThat(pop2.head_index, Equals(1)); - AssertThat(pop2.trees, Equals(vector({ trees[5], trees[8], trees[9] }))) - - StackPopResult pop3 = results.contents[2]; - AssertThat(ts_stack_top_tree(stack, 2), Equals(trees[6])); - AssertThat(pop3.head_index, Equals(2)); - AssertThat(pop3.trees, Equals(vector({ trees[7], trees[8], trees[9] }))) + StackPopResult result3 = results.contents[2]; + AssertThat(ts_stack_top_state(stack, 2), Equals(stateH)); + AssertThat(result3.head_index, Equals(2)); + AssertThat(result3.trees, Equals(vector({ trees[9], trees[10] }))) free_pop_results(&results); }); @@ -584,9 +526,21 @@ describe("Stack", [&]() { END_TEST bool operator==(const StackEntry &left, const StackEntry &right) { - return left.state == right.state && ts_tree_eq(left.tree, right.tree) && ts_length_eq(left.position, right.position); + return left.state == right.state && ts_length_eq(left.position, right.position); } std::ostream &operator<<(std::ostream &stream, const StackEntry &entry) { - return stream << "{" << entry.state << ", " << entry.tree << ", " << entry.position << "}"; + return stream << "{" << entry.state << ", " << entry.position << "}"; +} + +std::ostream &operator<<(std::ostream &stream, const TreeArray &array) { + stream << "["; + bool first = true; + for (size_t i = 0; i < array.size; i++) { + if (!first) + stream << ", "; + first = false; + stream << array.contents[i]; + } + return stream << "]"; } diff --git a/src/runtime/array.h b/src/runtime/array.h index 10a5c1a7..abe0bd64 100644 --- a/src/runtime/array.h +++ b/src/runtime/array.h @@ -40,7 +40,7 @@ extern "C" { #define array_push(self, element) \ (((self)->size < (self)->capacity || \ - array_grow((self), (self)->capacity * 2)) && \ + array_grow((self), (self)->capacity ? (self)->capacity * 2 : 4)) && \ ((self)->contents[(self)->size++] = (element), true)) #define array_splice(self, index, old_count, new_count, new_elements) \ diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 0bb3cb9e..a0cbb842 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -71,16 +71,14 @@ static ParseActionResult ts_parser__breakdown_top_of_stack(TSParser *self, * Since only one entry (not counting extra trees) is being popped from the * stack, there should only be one possible array of removed trees. */ - StackPopResult first_result = pop_results.contents[0]; - TreeArray removed_trees = first_result.trees; - TSTree *parent = *array_front(&removed_trees); - LOG("breakdown_pop sym:%s, size:%lu", SYM_NAME(parent->symbol), - ts_tree_total_size(parent).chars); for (size_t i = 0; i < pop_results.size; i++) { StackPopResult pop_result = pop_results.contents[i]; - assert(pop_result.trees.contents == removed_trees.contents); + TreeArray removed_trees = pop_result.trees; + TSTree *parent = *array_front(&removed_trees); int head_index = pop_result.head_index; + LOG("breakdown_pop sym:%s, size:%lu", SYM_NAME(parent->symbol), + ts_tree_total_size(parent).chars); StackPushResult last_push = StackPushResultContinued; TSStateId state = ts_stack_top_state(self->stack, head_index); @@ -112,12 +110,11 @@ static ParseActionResult ts_parser__breakdown_top_of_stack(TSParser *self, assert(last_push != StackPushResultMerged); else assert(last_push == StackPushResultMerged); + + for (size_t j = 0, count = removed_trees.size; j < count; j++) + ts_tree_release(removed_trees.contents[j]); + array_delete(&removed_trees); } - - for (size_t j = 0, count = first_result.trees.size; j < count; j++) - ts_tree_release(first_result.trees.contents[j]); - array_delete(&removed_trees); - } while (last_child && last_child->child_count > 0); return UpdatedStackHead; @@ -268,13 +265,17 @@ static int ts_parser__select_tree(void *data, TSTree *left, TSTree *right) { TSParser *self = data; int comparison = ts_tree_compare(left, right); - if (comparison <= 0) { - LOG("select tree:%s, over_tree:%s", SYM_NAME(left->symbol), - SYM_NAME(right->symbol)); - } else { - LOG("select tree:%s, over_tree:%s", SYM_NAME(right->symbol), - SYM_NAME(left->symbol)); + switch (comparison) { + case -1: + LOG_ACTION("select tree:%s, over_tree:%s", SYM_NAME(left->symbol), + SYM_NAME(right->symbol)); + break; + case 1: + LOG_ACTION("select tree:%s, over_tree:%s", SYM_NAME(right->symbol), + SYM_NAME(left->symbol)); + break; } + return comparison; } @@ -339,35 +340,21 @@ static ParseActionResult ts_parser__reduce(TSParser *self, int head, */ TSTree *parent = NULL; size_t trailing_extra_count = 0; - for (size_t j = 0; j < i; j++) { - StackPopResult prior_result = pop_results.contents[j]; - if (pop_result.trees.contents == prior_result.trees.contents) { - parent = self->reduce_parents.contents[j]; - trailing_extra_count = pop_result.trees.size - parent->child_count; - ts_tree_retain(parent); - for (size_t k = parent->child_count; k < pop_result.trees.size; k++) - ts_tree_retain(pop_result.trees.contents[k]); + for (size_t j = pop_result.trees.size - 1; j + 1 > 0; j--) { + if (pop_result.trees.contents[j]->extra) + trailing_extra_count++; + else break; - } } + size_t popped_child_count = pop_result.trees.size - trailing_extra_count; + parent = ts_tree_make_node(symbol, popped_child_count, pop_result.trees.contents, + metadata); if (!parent) { - for (size_t j = pop_result.trees.size - 1; j + 1 > 0; j--) { - if (pop_result.trees.contents[j]->extra) { - trailing_extra_count++; - } else - break; - } - - size_t child_count = pop_result.trees.size - trailing_extra_count; - parent = ts_tree_make_node(symbol, child_count, pop_result.trees.contents, - metadata); - if (!parent) { - for (size_t i = 0; i < pop_result.trees.size; i++) - ts_tree_release(pop_result.trees.contents[i]); - array_delete(&pop_result.trees); - goto error; - } + for (size_t i = 0; i < pop_result.trees.size; i++) + ts_tree_release(pop_result.trees.contents[i]); + array_delete(&pop_result.trees); + goto error; } if (!array_push(&self->reduce_parents, parent)) @@ -487,7 +474,8 @@ static ParseActionResult ts_parser__reduce_error(TSParser *self, int head, default: { StackEntry *entry = ts_stack_head(self->stack, head); entry->position = ts_length_add(entry->position, lookahead->padding); - entry->tree->size = ts_length_add(entry->tree->size, lookahead->padding); + TSTree *tree = *array_front(&self->reduce_parents); + tree->size = ts_length_add(tree->size, lookahead->padding); lookahead->padding = ts_length_zero(); return UpdatedStackHead; } @@ -824,7 +812,7 @@ TSTree *ts_parser_parse(TSParser *self, TSInput input, TSTree *previous_tree) { return NULL; } - LOG("lookahead sym:%s, size:%lu", SYM_NAME(lookahead->symbol), + LOG("lookahead sym:(%s,%d), size:%lu", SYM_NAME(lookahead->symbol), lookahead->symbol, ts_tree_total_chars(lookahead)); switch (ts_parser__consume_lookahead(self, head, lookahead)) { diff --git a/src/runtime/stack.c b/src/runtime/stack.c index c98eab34..d93d3303 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -12,12 +12,19 @@ #define STARTING_TREE_CAPACITY 10 #define MAX_NODE_POOL_SIZE 50 -typedef struct StackNode { +typedef struct StackNode StackNode; + +typedef struct { + StackNode *node; + TSTree *tree; +} StackLink; + +struct StackNode { StackEntry entry; - struct StackNode *successors[MAX_SUCCESSOR_COUNT]; + StackLink successors[MAX_SUCCESSOR_COUNT]; short unsigned int successor_count; short unsigned int ref_count; -} StackNode; +}; typedef struct { size_t goal_tree_count; @@ -100,11 +107,6 @@ TSLength ts_stack_top_position(const Stack *self, int head) { return entry ? entry->position : ts_length_zero(); } -TSTree *ts_stack_top_tree(const Stack *self, int head) { - StackEntry *entry = ts_stack_head((Stack *)self, head); - return entry ? entry->tree : NULL; -} - StackEntry *ts_stack_head(Stack *self, int head) { StackNode *node = self->heads.contents[head]; return node ? &node->entry : NULL; @@ -119,7 +121,7 @@ int ts_stack_entry_next_count(const StackEntry *entry) { } StackEntry *ts_stack_entry_next(const StackEntry *entry, int i) { - return &((const StackNode *)entry)->successors[i]->entry; + return &((const StackNode *)entry)->successors[i].node->entry; } /* @@ -139,9 +141,10 @@ static bool stack_node_release(Stack *self, StackNode *node) { assert(node->ref_count != 0); node->ref_count--; if (node->ref_count == 0) { - for (int i = 0; i < node->successor_count; i++) - stack_node_release(self, node->successors[i]); - ts_tree_release(node->entry.tree); + for (int i = 0; i < node->successor_count; i++) { + stack_node_release(self, node->successors[i].node); + ts_tree_release(node->successors[i].tree); + } if (self->node_pool.size >= MAX_NODE_POOL_SIZE) ts_free(node); @@ -174,26 +177,12 @@ static StackNode *stack_node_new(Stack *self, StackNode *next, TSStateId state, *node = (StackNode){ .ref_count = 1, .successor_count = 1, - .successors = { next, NULL, NULL }, - .entry = {.state = state, .tree = tree, .position = position }, + .successors = { {next, tree} }, + .entry = {.state = state, .position = position }, }; return node; } -static void ts_stack__add_alternative_tree(Stack *self, StackNode *node, - TSTree *tree) { - if (tree != node->entry.tree) { - int comparison = self->tree_selection_function(self->tree_selection_payload, - node->entry.tree, tree); - - if (comparison > 0) { - ts_tree_retain(tree); - ts_tree_release(node->entry.tree); - node->entry.tree = tree; - } - } -} - static void ts_stack__clear_pop_result(Stack *self, StackPopResult *result) { for (size_t i = 0; i < result->trees.size; i++) ts_tree_release(result->trees.contents[i]); @@ -230,27 +219,31 @@ static void ts_stack__add_alternative_pop_result(Stack *self, } } -static void ts_stack__add_node_successor(Stack *self, StackNode *node, - StackNode *new_successor) { - for (int i = 0; i < node->successor_count; i++) { - StackNode *successor = node->successors[i]; - if (successor == new_successor) - return; - if (!successor) - continue; - - if (successor->entry.state == new_successor->entry.state) { - ts_stack__add_alternative_tree(self, successor, new_successor->entry.tree); - for (int j = 0; j < new_successor->successor_count; j++) - ts_stack__add_node_successor(self, successor, - new_successor->successors[j]); - return; +static void stack_node__add_successor(StackNode *self, + TSTree *new_tree, + StackNode *new_node) { + for (int i = 0; i < self->successor_count; i++) { + StackLink successor = self->successors[i]; + if (successor.tree == new_tree) { + if (successor.node == new_node) + return; + if (successor.node && new_node && + successor.node->entry.state == new_node->entry.state) { + for (int j = 0; j < new_node->successor_count; j++) { + stack_node__add_successor(successor.node, + new_node->successors[j].tree, new_node->successors[j].node); + } + return; + } } } - stack_node_retain(new_successor); - node->successors[node->successor_count] = new_successor; - node->successor_count++; + stack_node_retain(new_node); + ts_tree_retain(new_tree); + self->successors[self->successor_count++] = (StackLink){ + new_node, + new_tree, + }; } /* @@ -296,8 +289,7 @@ StackPushResult ts_stack_push(Stack *self, int head_index, TSStateId state, StackEntry prior_entry = prior_node->entry; if (prior_entry.state == state && ts_length_eq(prior_entry.position, position)) { - ts_stack__add_alternative_tree(self, prior_node, tree); - ts_stack__add_node_successor(self, prior_node, current_head); + stack_node__add_successor(prior_node, tree, current_head); ts_stack_remove_head(self, head_index); return StackPushResultMerged; } @@ -349,38 +341,43 @@ StackPopResultArray ts_stack_pop(Stack *self, int head_index, int child_count, if (!node || path->trees.size == path->goal_tree_count) continue; - all_paths_done = false; - /* - * Children that are 'extra' do not count towards the total child count. - */ - if (node->entry.tree->extra && !count_extra) - path->goal_tree_count++; - /* * If a node has more than one successor, create new paths for each of * the additional successors. */ - if (path->is_shared) { - path->trees = (TreeArray)array_copy(&path->trees); - for (size_t j = 0; j < path->trees.size; j++) - ts_tree_retain(path->trees.contents[j]); - path->is_shared = false; - } + for (int j = 0; j < node->successor_count; j++) { + StackLink successor = node->successors[j]; - ts_tree_retain(node->entry.tree); - if (!array_push(&path->trees, node->entry.tree)) - goto error; + PopPath *next_path; + if (j == 0) { + next_path = path; + } else { + if (!array_push(&self->pop_paths, *path)) + goto error; + next_path = array_back(&self->pop_paths); + next_path->is_shared = true; + } - path->node = path->node->successors[0]; - for (int j = 1; j < node->successor_count; j++) { - if (!array_push(&self->pop_paths, *path)) + if (next_path->is_shared) { + next_path->trees = (TreeArray)array_copy(&path->trees); + next_path->trees.size--; + for (size_t j = 0; j < next_path->trees.size; j++) + ts_tree_retain(next_path->trees.contents[j]); + next_path->is_shared = false; + } + + next_path->node = successor.node; + ts_tree_retain(successor.tree); + if (!array_push(&next_path->trees, successor.tree)) goto error; - PopPath *next_path = array_back(&self->pop_paths); - next_path->node = node->successors[j]; - next_path->is_shared = true; + /* + * Children that are 'extra' do not count towards the total child count. + */ + if (successor.tree->extra && !count_extra) + next_path->goal_tree_count++; } } } @@ -440,7 +437,7 @@ void ts_stack_shrink(Stack *self, int head_index, int count) { for (int i = 0; i < count; i++) { if (new_head->successor_count == 0) break; - new_head = new_head->successors[0]; + new_head = new_head->successors[0].node; } stack_node_retain(new_head); stack_node_release(self, head); @@ -475,10 +472,12 @@ void ts_stack_delete(Stack *self) { ts_free(self); } -static const char *graph_colors[] = { +static const char *COLORS[] = { "red", "blue", "orange", "green", "purple", }; +static size_t COLOR_COUNT = sizeof(COLORS) / sizeof(COLORS[0]); + size_t ts_stack__write_dot_graph(Stack *self, char *string, size_t n, const char **symbol_names) { char *cursor = string; @@ -486,15 +485,15 @@ size_t ts_stack__write_dot_graph(Stack *self, char *string, size_t n, cursor += snprintf(*s, n, "digraph stack {\n"); cursor += snprintf(*s, n, "rankdir=\"RL\";\n"); + Array(StackNode *) visited_nodes; + array_init(&visited_nodes); + array_clear(&self->pop_paths); for (size_t i = 0; i < self->heads.size; i++) { StackNode *node = self->heads.contents[i]; - const char *color = - graph_colors[i % (sizeof(graph_colors) / sizeof(graph_colors[0]))]; + const char *color = COLORS[i % COLOR_COUNT]; cursor += snprintf(*s, n, "node_%p [color=%s];\n", node, color); - array_push(&self->pop_paths, ((PopPath){ - .node = node, - })); + array_push(&self->pop_paths, ((PopPath){ .node = node })); } bool all_paths_done = false; @@ -505,38 +504,47 @@ size_t ts_stack__write_dot_graph(Stack *self, char *string, size_t n, PopPath *path = &self->pop_paths.contents[i]; StackNode *node = path->node; + for (size_t j = 0; j < visited_nodes.size; j++) { + if (visited_nodes.contents[j] == node) { + node = NULL; + break; + } + } + if (!node) continue; - all_paths_done = false; - cursor += - snprintf(*s, n, "node_%p [label=\"%s\\n%d\"];\n", node, - symbol_names[node->entry.tree->symbol], node->entry.state); + cursor += snprintf(*s, n, "node_%p [label=%d];\n", node, node->entry.state); - path->node = node->successors[0]; - cursor += - snprintf(*s, n, "node_%p -> node_%p;\n", node, node->successors[0]); + for (int j = 0; j < node->successor_count; j++) { + StackLink successor = node->successors[j]; + cursor += snprintf(*s, n, "node_%p -> node_%p [label=\"%s\"];\n", node, + successor.node, symbol_names[successor.tree->symbol]); - for (int j = 1; j < node->successor_count; j++) { - if (!array_push(&self->pop_paths, *path)) - goto error; - cursor += - snprintf(*s, n, "node_%p -> node_%p;\n", node, node->successors[j]); - - PopPath *next_path = array_back(&self->pop_paths); - next_path->node = node->successors[j]; - next_path->is_shared = true; + if (j == 0) { + path->node = successor.node; + } else { + if (!array_push(&self->pop_paths, *path)) + goto error; + PopPath *next_path = array_back(&self->pop_paths); + next_path->node = successor.node; + } } + + if (!array_push(&visited_nodes, node)) + goto error; } } - cursor += snprintf(*s, n, "node_%p [label=\"-\\n0\"];\n", NULL); + cursor += snprintf(*s, n, "node_%p [label=0];\n", NULL); cursor += snprintf(*s, n, "}\n"); + array_delete(&visited_nodes); return cursor - string; error: + array_delete(&visited_nodes); return (size_t)-1; } diff --git a/src/runtime/stack.h b/src/runtime/stack.h index ae76c2e6..55803d57 100644 --- a/src/runtime/stack.h +++ b/src/runtime/stack.h @@ -12,7 +12,6 @@ extern "C" { typedef struct Stack Stack; typedef struct { - TSTree *tree; TSStateId state; TSLength position; } StackEntry; @@ -53,12 +52,6 @@ int ts_stack_head_count(const Stack *); */ TSStateId ts_stack_top_state(const Stack *, int head); -/* - * Get the tree at given head of the stack. If the stack is empty, this - * returns NULL. - */ -TSTree *ts_stack_top_tree(const Stack *, int head); - /* * Get the position of the given head of the stack. If the stack is empty, this * returns {0, 0}.