diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 92003ff1..336aaab4 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -553,13 +553,19 @@ void ts_tree_print_dot_graph(const Tree *self, const TSLanguage *language, fprintf(f, "}\n"); } +TSExternalTokenState empty_state = { + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0, +}; + bool ts_tree_external_token_state_eq(const Tree *self, const Tree *other) { - return self == other || ( - self && - other && - self->has_external_tokens == other->has_external_tokens && ( - !self->has_external_tokens || - memcmp(&self->external_token_state, &other->external_token_state, sizeof(TSExternalTokenState)) == 0 - ) - ); + const TSExternalTokenState *state1 = &empty_state; + const TSExternalTokenState *state2 = &empty_state; + if (self && self->has_external_tokens) state1 = &self->external_token_state; + if (other && other->has_external_tokens) state2 = &other->external_token_state; + return + state1 == state2 || + memcmp(state1, state2, sizeof(TSExternalTokenState)) == 0; } diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index 14bcbc7c..ecf579e0 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -524,8 +524,10 @@ describe("Stack", [&]() { describe("setting external token state", [&]() { before_each([&]() { - trees[1]->external_token_state[0] = 'a'; - trees[2]->external_token_state[0] = 'b'; + trees[1]->has_external_tokens = true; + trees[2]->has_external_tokens = true; + memset(&trees[1]->external_token_state, 0, sizeof(TSExternalTokenState)); + memset(&trees[2]->external_token_state, 0, sizeof(TSExternalTokenState)); }); it("allows the state to be retrieved", [&]() { @@ -535,19 +537,49 @@ describe("Stack", [&]() { AssertThat(ts_stack_last_external_token(stack, 0), Equals(trees[1])); ts_stack_copy_version(stack, 0); - AssertThat(ts_stack_last_external_token(stack, 0), Equals(trees[1])); + AssertThat(ts_stack_last_external_token(stack, 1), Equals(trees[1])); + + ts_stack_set_last_external_token(stack, 0, trees[2]); + AssertThat(ts_stack_last_external_token(stack, 0), Equals(trees[2])); }); it("does not merge stack versions with different external token states", [&]() { + trees[1]->external_token_state[5] = 'a'; + trees[2]->external_token_state[5] = 'b'; + ts_stack_copy_version(stack, 0); ts_stack_push(stack, 0, trees[0], false, 5); ts_stack_push(stack, 1, trees[0], false, 5); ts_stack_set_last_external_token(stack, 0, trees[1]); - ts_stack_set_last_external_token(stack, 0, trees[1]); + ts_stack_set_last_external_token(stack, 1, trees[2]); AssertThat(ts_stack_merge(stack, 0, 1), IsFalse()); }); + + it("merges stack versions with identical external token states", [&]() { + trees[1]->external_token_state[5] = 'a'; + trees[2]->external_token_state[5] = 'a'; + + ts_stack_copy_version(stack, 0); + ts_stack_push(stack, 0, trees[0], false, 5); + ts_stack_push(stack, 1, trees[0], false, 5); + + ts_stack_set_last_external_token(stack, 0, trees[1]); + ts_stack_set_last_external_token(stack, 1, trees[2]); + + AssertThat(ts_stack_merge(stack, 0, 1), IsTrue()); + }); + + it("does not distinguish between an *empty* external token state and *no* external token state", [&]() { + ts_stack_copy_version(stack, 0); + ts_stack_push(stack, 0, trees[0], false, 5); + ts_stack_push(stack, 1, trees[0], false, 5); + + ts_stack_set_last_external_token(stack, 0, trees[1]); + + AssertThat(ts_stack_merge(stack, 0, 1), IsTrue()); + }); }); });