Revert "Remove the separator characters construct"

This reverts commit 5cd07648fd.

The separators construct is useful as an optimization. It turns out that
constructing a node for every chunk of whitespace in a document causes a
significant performance regression.

Conflicts:
	src/compiler/build_tables/build_lex_table.cc
	src/compiler/grammar.cc
	src/runtime/parser.c
This commit is contained in:
Max Brunsfeld 2014-09-02 07:41:29 -07:00
parent e941f8c175
commit 545e575508
43 changed files with 9065 additions and 11203 deletions

View file

@ -50,26 +50,7 @@ size_t ts_stack_right_position(const TSStack *stack) {
size_t result = 0;
for (size_t i = 0; i < stack->size; i++) {
TSTree *node = stack->entries[i].node;
result += node->size;
result += ts_tree_total_size(node);
}
return result;
}
TSTree **ts_stack_pop_extras(TSStack *stack, size_t *count) {
size_t first = stack->size;
while (first > 0 && ts_tree_is_extra(stack->entries[first - 1].node))
first--;
*count = (stack->size - first);
if (*count == 0)
return NULL;
TSTree **result = malloc(*count * sizeof(TSTree *));
for (size_t i = 0; i < *count; i++) {
result[i] = stack->entries[first + i].node;
ts_tree_retain(result[i]);
}
ts_stack_shrink(stack, first - 1);
return result;
}