Combine TSParser and TSStateMachine objects
My original thought was to decouple the runtime from the LR parser generator by making TSParser a generic interface that LR parsers implement. I think this was more trouble than it was worth.
This commit is contained in:
Max Brunsfeld
parent
1c7d2d2d03
commit
9da7663e99
18 changed files with 586 additions and 645 deletions
262
src/runtime/parser.c
Normal file
262
src/runtime/parser.c
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
#include <stdio.h>
|
||||
#include "tree_sitter/runtime.h"
|
||||
#include "tree_sitter/parser.h"
|
||||
#include "runtime/tree.h"
|
||||
|
||||
/*
|
||||
* Private
|
||||
*/
|
||||
|
||||
static const TSParseAction * actions_for_state(TSParserConfig config, TSStateId state) {
|
||||
return config.parse_table + (state * config.symbol_count);
|
||||
}
|
||||
|
||||
static size_t breakdown_stack(TSParser *parser, TSInputEdit *edit) {
|
||||
if (!edit) return 0;
|
||||
|
||||
TSStack *stack = &parser->stack;
|
||||
size_t position = 0;
|
||||
|
||||
for (;;) {
|
||||
TSTree *node = ts_stack_top_node(stack);
|
||||
if (!node) break;
|
||||
|
||||
position = ts_stack_right_position(stack);
|
||||
size_t child_count;
|
||||
TSTree **children = ts_tree_immediate_children(node, &child_count);
|
||||
if (position <= edit->position && !children) break;
|
||||
|
||||
stack->size--;
|
||||
position -= ts_tree_total_size(node);
|
||||
|
||||
for (size_t i = 0; i < child_count && position < edit->position; i++) {
|
||||
TSTree *child = children[i];
|
||||
TSStateId state = ts_stack_top_state(stack);
|
||||
TSStateId next_state = actions_for_state(parser->config, state)[ts_tree_symbol(child)].data.to_state;
|
||||
ts_stack_push(stack, next_state, child);
|
||||
ts_tree_retain(child);
|
||||
position += ts_tree_total_size(child);
|
||||
}
|
||||
|
||||
ts_tree_release(node);
|
||||
}
|
||||
|
||||
return position;
|
||||
}
|
||||
|
||||
static TSSymbol * expected_symbols(TSParser *parser, size_t *count) {
|
||||
*count = 0;
|
||||
const TSParseAction *actions = actions_for_state(parser->config, ts_stack_top_state(&parser->stack));
|
||||
for (size_t i = 0; i < parser->config.symbol_count; i++)
|
||||
if (actions[i].type != TSParseActionTypeError)
|
||||
(*count)++;
|
||||
|
||||
size_t n = 0;
|
||||
TSSymbol *result = malloc(*count * sizeof(*result));
|
||||
for (TSSymbol i = 0; i < parser->config.symbol_count; i++)
|
||||
if (actions[i].type != TSParseActionTypeError)
|
||||
result[n++] = i;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* Public
|
||||
*/
|
||||
|
||||
TSParser * ts_parser_make(TSParserConfig config) {
|
||||
TSParser *result = malloc(sizeof(*result));
|
||||
*result = (TSParser) {
|
||||
.lexer = ts_lexer_make(),
|
||||
.stack = ts_stack_make(),
|
||||
.debug = 0,
|
||||
.config = config,
|
||||
};
|
||||
return result;
|
||||
}
|
||||
|
||||
void ts_parser_free(TSParser *parser) {
|
||||
if (parser->lookahead) ts_tree_release(parser->lookahead);
|
||||
if (parser->next_lookahead) ts_tree_release(parser->next_lookahead);
|
||||
ts_stack_delete(&parser->stack);
|
||||
free(parser);
|
||||
}
|
||||
|
||||
void ts_parser_start(TSParser *parser, TSInput input, TSInputEdit *edit) {
|
||||
if (!edit) ts_stack_shrink(&parser->stack, 0);
|
||||
parser->lookahead = NULL;
|
||||
parser->next_lookahead = NULL;
|
||||
|
||||
size_t position = breakdown_stack(parser, edit);
|
||||
input.seek_fn(input.data, position);
|
||||
|
||||
parser->lexer = ts_lexer_make();
|
||||
parser->lexer.input = input;
|
||||
ts_lexer_advance(&parser->lexer);
|
||||
}
|
||||
|
||||
void ts_parser_shift(TSParser *parser, TSStateId parse_state) {
|
||||
if (parser->lookahead->is_extra)
|
||||
parse_state = ts_stack_top_state(&parser->stack);
|
||||
ts_stack_push(&parser->stack, parse_state, parser->lookahead);
|
||||
parser->lookahead = parser->next_lookahead;
|
||||
parser->next_lookahead = NULL;
|
||||
}
|
||||
|
||||
void ts_parser_shift_extra(TSParser *parser) {
|
||||
parser->lookahead->is_extra = 1;
|
||||
ts_parser_shift(parser, 0);
|
||||
}
|
||||
|
||||
void ts_parser_reduce(TSParser *parser, TSSymbol symbol, size_t child_count) {
|
||||
parser->next_lookahead = parser->lookahead;
|
||||
parser->lookahead = ts_stack_reduce(
|
||||
&parser->stack,
|
||||
symbol,
|
||||
child_count,
|
||||
parser->config.hidden_symbol_flags, 1);
|
||||
}
|
||||
|
||||
int ts_parser_reduce_extra(TSParser *parser, TSSymbol symbol) {
|
||||
TSTree *top_node = ts_stack_top_node(&parser->stack);
|
||||
if (top_node->symbol == symbol && !top_node->is_extra) {
|
||||
ts_parser_reduce(parser, symbol, 1);
|
||||
parser->lookahead->is_extra = 1;
|
||||
return 1;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
int ts_parser_handle_error(TSParser *parser) {
|
||||
size_t count = 0;
|
||||
const TSSymbol *inputs = expected_symbols(parser, &count);
|
||||
TSTree *error = ts_tree_make_error(ts_lexer_lookahead_char(&parser->lexer),
|
||||
count,
|
||||
inputs,
|
||||
0,
|
||||
0);
|
||||
|
||||
for (;;) {
|
||||
ts_tree_release(parser->lookahead);
|
||||
size_t position = ts_lexer_position(&parser->lexer);
|
||||
parser->lookahead = parser->config.lex_fn(parser, ts_lex_state_error);
|
||||
|
||||
int at_end = 0;
|
||||
if (ts_lexer_position(&parser->lexer) == position)
|
||||
at_end = !ts_lexer_advance(&parser->lexer);
|
||||
|
||||
if (at_end || ts_tree_symbol(parser->lookahead) == ts_builtin_sym_end) {
|
||||
ts_stack_push(&parser->stack, 0, error);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Unwind the stack, looking for a state in which this token
|
||||
* may appear after an error.
|
||||
*/
|
||||
for (size_t j = 0; j < parser->stack.size; j++) {
|
||||
size_t i = parser->stack.size - 1 - j;
|
||||
TSStateId stack_state = parser->stack.entries[i].state;
|
||||
TSParseAction action_on_error = actions_for_state(parser->config, stack_state)[ts_builtin_sym_error];
|
||||
if (action_on_error.type == TSParseActionTypeShift) {
|
||||
TSStateId state_after_error = action_on_error.data.to_state;
|
||||
if (actions_for_state(parser->config, state_after_error)[ts_tree_symbol(parser->lookahead)].type != TSParseActionTypeError) {
|
||||
ts_stack_shrink(&parser->stack, i + 1);
|
||||
ts_stack_push(&parser->stack, state_after_error, error);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TSTree * ts_parser_tree_root(TSParser *parser) {
|
||||
TSStack *stack = &parser->stack;
|
||||
TSTree *top_node = ts_stack_top_node(stack);
|
||||
if (stack->size <= 1)
|
||||
return top_node;
|
||||
if (ts_tree_symbol(top_node) == ts_builtin_sym_error)
|
||||
return top_node;
|
||||
|
||||
size_t immediate_child_count;
|
||||
TSTree **immedate_children = ts_tree_immediate_children(top_node, &immediate_child_count);
|
||||
|
||||
stack->size--;
|
||||
for (size_t i = 0; i < immediate_child_count; i++) {
|
||||
TSTree *child = immedate_children[i];
|
||||
child->is_extra = 0;
|
||||
ts_tree_retain(child);
|
||||
TSStateId state = ts_stack_top_state(stack);
|
||||
TSStateId next_state = actions_for_state(parser->config, state)[ts_tree_symbol(child)].data.to_state;
|
||||
ts_stack_push(stack, next_state, child);
|
||||
}
|
||||
|
||||
TSTree *new_node = ts_stack_reduce(stack,
|
||||
top_node->symbol,
|
||||
stack->size,
|
||||
parser->config.hidden_symbol_flags,
|
||||
0);
|
||||
ts_tree_release(top_node);
|
||||
return new_node;
|
||||
}
|
||||
|
||||
TSParseAction ts_parser_next_action(TSParser *parser) {
|
||||
TSStateId state = ts_stack_top_state(&parser->stack);
|
||||
if (!parser->lookahead)
|
||||
parser->lookahead = parser->config.lex_fn(parser, parser->config.lex_states[state]);
|
||||
return actions_for_state(parser->config, state)[ts_tree_symbol(parser->lookahead)];
|
||||
}
|
||||
|
||||
#define DEBUG_PARSE(...) \
|
||||
if (parser->debug) { fprintf(stderr, "\n" __VA_ARGS__); }
|
||||
|
||||
TSTree * ts_parser_step(TSParser *parser) {
|
||||
TSParseAction action = ts_parser_next_action(parser);
|
||||
DEBUG_PARSE("LOOKAHEAD %s", parser->config.symbol_names[ts_tree_symbol(parser->lookahead)]);
|
||||
switch (action.type) {
|
||||
case TSParseActionTypeShift:
|
||||
DEBUG_PARSE("SHIFT %d", action.data.to_state);
|
||||
ts_parser_shift(parser, action.data.to_state);
|
||||
return NULL;
|
||||
case TSParseActionTypeShiftExtra:
|
||||
DEBUG_PARSE("SHIFT EXTRA");
|
||||
ts_parser_shift_extra(parser);
|
||||
return NULL;
|
||||
case TSParseActionTypeReduce:
|
||||
DEBUG_PARSE("REDUCE %s %d", parser->config.symbol_names[action.data.symbol], action.data.child_count);
|
||||
ts_parser_reduce(parser, action.data.symbol, action.data.child_count);
|
||||
return NULL;
|
||||
case TSParseActionTypeReduceExtra:
|
||||
if (!ts_parser_reduce_extra(parser, action.data.symbol))
|
||||
goto error;
|
||||
DEBUG_PARSE("REDUCE EXTRA");
|
||||
return NULL;
|
||||
case TSParseActionTypeAccept:
|
||||
DEBUG_PARSE("ACCEPT");
|
||||
return ts_parser_tree_root(parser);
|
||||
case TSParseActionTypeError:
|
||||
goto error;
|
||||
default:
|
||||
return NULL;
|
||||
}
|
||||
error:
|
||||
DEBUG_PARSE("ERROR");
|
||||
if (!ts_parser_handle_error(parser))
|
||||
return ts_parser_tree_root(parser);
|
||||
else
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const TSTree * ts_parser_parse(TSParser *parser, TSInput input, TSInputEdit *edit) {
|
||||
ts_parser_start(parser, input, edit);
|
||||
|
||||
for (;;) {
|
||||
const TSTree *tree = ts_parser_step(parser);
|
||||
if (tree) return tree;
|
||||
}
|
||||
}
|
||||
|
||||
TSParserConfig ts_parser_config(TSParser *parser) {
|
||||
return parser->config;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue