traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Combine TSParser and TSStateMachine objects

Browse source 
My original thought was to decouple the runtime from
the LR parser generator by making TSParser a generic
interface that LR parsers implement.

I think this was more trouble than it was worth.
This commit is contained in:
Max Brunsfeld 2014-07-10 13:14:52 -07:00
parent 1c7d2d2d03
commit 9da7663e99
18 changed files with 586 additions and 645 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/runtime/document.c
View file

@ -1,8 +1,9 @@
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser.h"
#include <string.h>
struct TSDocument {
TSParser parser;
TSParser *parser;
const TSTree *tree;
TSInput input;
size_t error_count;
@ -13,14 +14,13 @@ TSDocument * ts_document_make() {
}
void ts_document_free(TSDocument *document) {
if (document->parser.free_fn)
document->parser.free_fn(document->parser.data);
ts_parser_free(document->parser);
if (document->input.release_fn)
document->input.release_fn(document->input.data);
free(document);
}
void ts_document_set_parser(TSDocument *document, TSParser parser) {
void ts_document_set_parser(TSDocument *document, TSParser *parser) {
document->parser = parser;
}
@ -29,20 +29,20 @@ const TSTree * ts_document_tree(const TSDocument *document) {
}
const char * ts_document_string(const TSDocument *document) {
return ts_tree_string(document->tree, document->parser.symbol_names);
return ts_tree_string(document->tree, ts_parser_config(document->parser).symbol_names);
}
void ts_document_set_input(TSDocument *document, TSInput input) {
document->input = input;
document->tree = document->parser.parse_fn(document->parser.data, input, NULL);
document->tree = ts_parser_parse(document->parser, document->input, NULL);
}
void ts_document_edit(TSDocument *document, TSInputEdit edit) {
document->tree = document->parser.parse_fn(document->parser.data, document->input, &edit);
document->tree = ts_parser_parse(document->parser, document->input, &edit);
}
const char * ts_document_symbol_name(const TSDocument *document, const TSTree *tree) {
return document->parser.symbol_names[ts_tree_symbol(tree)];
return ts_parser_config(document->parser).symbol_names[ts_tree_symbol(tree)];
}
typedef struct {

43
src/runtime/lexer.c Normal file
View file

@ -0,0 +1,43 @@
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser.h"
TSLexer ts_lexer_make() {
return (TSLexer) {
.chunk = NULL,
.debug = 0,
.chunk_start = 0,
.chunk_size = 0,
.position_in_chunk = 0,
.token_start_position = 0,
.token_end_position = 0,
.reached_end = 0
};
}
int ts_lexer_advance(TSLexer *lexer) {
static const char *empty_chunk = "";
if (lexer->position_in_chunk + 1 < lexer->chunk_size) {
lexer->position_in_chunk++;
} else {
lexer->chunk_start += lexer->chunk_size;
lexer->chunk = lexer->input.read_fn(lexer->input.data, &lexer->chunk_size);
lexer->position_in_chunk = 0;
if (lexer->reached_end) {
return 0;
}
if (lexer->chunk_size == 0) {
lexer->reached_end = 1;
lexer->chunk = empty_chunk;
}
}
return 1;
}
TSTree * ts_lexer_build_node(TSLexer *lexer, TSSymbol symbol) {
size_t current_position = ts_lexer_position(lexer);
size_t size = current_position - lexer->token_start_position;
size_t offset = lexer->token_start_position - lexer->token_end_position;
lexer->token_end_position = current_position;
return ts_tree_make_leaf(symbol, size, offset);
}

262
src/runtime/parser.c Normal file
View file

@ -0,0 +1,262 @@
#include <stdio.h>
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser.h"
#include "runtime/tree.h"
/*
* Private
*/
static const TSParseAction * actions_for_state(TSParserConfig config, TSStateId state) {
return config.parse_table + (state * config.symbol_count);
}
static size_t breakdown_stack(TSParser *parser, TSInputEdit *edit) {
if (!edit) return 0;
TSStack *stack = &parser->stack;
size_t position = 0;
for (;;) {
TSTree *node = ts_stack_top_node(stack);
if (!node) break;
position = ts_stack_right_position(stack);
size_t child_count;
TSTree **children = ts_tree_immediate_children(node, &child_count);
if (position <= edit->position && !children) break;
stack->size--;
position -= ts_tree_total_size(node);
for (size_t i = 0; i < child_count && position < edit->position; i++) {
TSTree *child = children[i];
TSStateId state = ts_stack_top_state(stack);
TSStateId next_state = actions_for_state(parser->config, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child);
ts_tree_retain(child);
position += ts_tree_total_size(child);
}
ts_tree_release(node);
}
return position;
}
static TSSymbol * expected_symbols(TSParser *parser, size_t *count) {
*count = 0;
const TSParseAction *actions = actions_for_state(parser->config, ts_stack_top_state(&parser->stack));
for (size_t i = 0; i < parser->config.symbol_count; i++)
if (actions[i].type != TSParseActionTypeError)
(*count)++;
size_t n = 0;
TSSymbol *result = malloc(*count * sizeof(*result));
for (TSSymbol i = 0; i < parser->config.symbol_count; i++)
if (actions[i].type != TSParseActionTypeError)
result[n++] = i;
return result;
}
/*
* Public
*/
TSParser * ts_parser_make(TSParserConfig config) {
TSParser *result = malloc(sizeof(*result));
*result = (TSParser) {
.lexer = ts_lexer_make(),
.stack = ts_stack_make(),
.debug = 0,
.config = config,
};
return result;
}
void ts_parser_free(TSParser *parser) {
if (parser->lookahead) ts_tree_release(parser->lookahead);
if (parser->next_lookahead) ts_tree_release(parser->next_lookahead);
ts_stack_delete(&parser->stack);
free(parser);
}
void ts_parser_start(TSParser *parser, TSInput input, TSInputEdit *edit) {
if (!edit) ts_stack_shrink(&parser->stack, 0);
parser->lookahead = NULL;
parser->next_lookahead = NULL;
size_t position = breakdown_stack(parser, edit);
input.seek_fn(input.data, position);
parser->lexer = ts_lexer_make();
parser->lexer.input = input;
ts_lexer_advance(&parser->lexer);
}
void ts_parser_shift(TSParser *parser, TSStateId parse_state) {
if (parser->lookahead->is_extra)
parse_state = ts_stack_top_state(&parser->stack);
ts_stack_push(&parser->stack, parse_state, parser->lookahead);
parser->lookahead = parser->next_lookahead;
parser->next_lookahead = NULL;
}
void ts_parser_shift_extra(TSParser *parser) {
parser->lookahead->is_extra = 1;
ts_parser_shift(parser, 0);
}
void ts_parser_reduce(TSParser *parser, TSSymbol symbol, size_t child_count) {
parser->next_lookahead = parser->lookahead;
parser->lookahead = ts_stack_reduce(
&parser->stack,
symbol,
child_count,
parser->config.hidden_symbol_flags, 1);
}
int ts_parser_reduce_extra(TSParser *parser, TSSymbol symbol) {
TSTree *top_node = ts_stack_top_node(&parser->stack);
if (top_node->symbol == symbol && !top_node->is_extra) {
ts_parser_reduce(parser, symbol, 1);
parser->lookahead->is_extra = 1;
return 1;
} else {
return 0;
}
}
int ts_parser_handle_error(TSParser *parser) {
size_t count = 0;
const TSSymbol *inputs = expected_symbols(parser, &count);
TSTree *error = ts_tree_make_error(ts_lexer_lookahead_char(&parser->lexer),
count,
inputs,
0,
0);
for (;;) {
ts_tree_release(parser->lookahead);
size_t position = ts_lexer_position(&parser->lexer);
parser->lookahead = parser->config.lex_fn(parser, ts_lex_state_error);
int at_end = 0;
if (ts_lexer_position(&parser->lexer) == position)
at_end = !ts_lexer_advance(&parser->lexer);
if (at_end || ts_tree_symbol(parser->lookahead) == ts_builtin_sym_end) {
ts_stack_push(&parser->stack, 0, error);
return 0;
}
/*
* Unwind the stack, looking for a state in which this token
* may appear after an error.
*/
for (size_t j = 0; j < parser->stack.size; j++) {
size_t i = parser->stack.size - 1 - j;
TSStateId stack_state = parser->stack.entries[i].state;
TSParseAction action_on_error = actions_for_state(parser->config, stack_state)[ts_builtin_sym_error];
if (action_on_error.type == TSParseActionTypeShift) {
TSStateId state_after_error = action_on_error.data.to_state;
if (actions_for_state(parser->config, state_after_error)[ts_tree_symbol(parser->lookahead)].type != TSParseActionTypeError) {
ts_stack_shrink(&parser->stack, i + 1);
ts_stack_push(&parser->stack, state_after_error, error);
return 1;
}
}
}
}
}
TSTree * ts_parser_tree_root(TSParser *parser) {
TSStack *stack = &parser->stack;
TSTree *top_node = ts_stack_top_node(stack);
if (stack->size <= 1)
return top_node;
if (ts_tree_symbol(top_node) == ts_builtin_sym_error)
return top_node;
size_t immediate_child_count;
TSTree **immedate_children = ts_tree_immediate_children(top_node, &immediate_child_count);
stack->size--;
for (size_t i = 0; i < immediate_child_count; i++) {
TSTree *child = immedate_children[i];
child->is_extra = 0;
ts_tree_retain(child);
TSStateId state = ts_stack_top_state(stack);
TSStateId next_state = actions_for_state(parser->config, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child);
}
TSTree *new_node = ts_stack_reduce(stack,
top_node->symbol,
stack->size,
parser->config.hidden_symbol_flags,
0);
ts_tree_release(top_node);
return new_node;
}
TSParseAction ts_parser_next_action(TSParser *parser) {
TSStateId state = ts_stack_top_state(&parser->stack);
if (!parser->lookahead)
parser->lookahead = parser->config.lex_fn(parser, parser->config.lex_states[state]);
return actions_for_state(parser->config, state)[ts_tree_symbol(parser->lookahead)];
}
#define DEBUG_PARSE(...) \
if (parser->debug) { fprintf(stderr, "\n" __VA_ARGS__); }
TSTree * ts_parser_step(TSParser *parser) {
TSParseAction action = ts_parser_next_action(parser);
DEBUG_PARSE("LOOKAHEAD %s", parser->config.symbol_names[ts_tree_symbol(parser->lookahead)]);
switch (action.type) {
case TSParseActionTypeShift:
DEBUG_PARSE("SHIFT %d", action.data.to_state);
ts_parser_shift(parser, action.data.to_state);
return NULL;
case TSParseActionTypeShiftExtra:
DEBUG_PARSE("SHIFT EXTRA");
ts_parser_shift_extra(parser);
return NULL;
case TSParseActionTypeReduce:
DEBUG_PARSE("REDUCE %s %d", parser->config.symbol_names[action.data.symbol], action.data.child_count);
ts_parser_reduce(parser, action.data.symbol, action.data.child_count);
return NULL;
case TSParseActionTypeReduceExtra:
if (!ts_parser_reduce_extra(parser, action.data.symbol))
goto error;
DEBUG_PARSE("REDUCE EXTRA");
return NULL;
case TSParseActionTypeAccept:
DEBUG_PARSE("ACCEPT");
return ts_parser_tree_root(parser);
case TSParseActionTypeError:
goto error;
default:
return NULL;
}
error:
DEBUG_PARSE("ERROR");
if (!ts_parser_handle_error(parser))
return ts_parser_tree_root(parser);
else
return NULL;
}
const TSTree * ts_parser_parse(TSParser *parser, TSInput input, TSInputEdit *edit) {
ts_parser_start(parser, input, edit);
for (;;) {
const TSTree *tree = ts_parser_step(parser);
if (tree) return tree;
}
}
TSParserConfig ts_parser_config(TSParser *parser) {
return parser->config;
}

2
src/runtime/stack.c
View file

@ -1,5 +1,5 @@
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser/stack.h"
#include "tree_sitter/parser.h"
#include "runtime/tree.h"
#include <string.h>

263
src/runtime/state_machine.c
View file

@ -1,263 +0,0 @@
#include "tree_sitter/parser/state_machine.h"
#include "runtime/tree.h"
/*
* Private
*/
static const TSParseAction * actions_for_state(TSStateMachine *machine, TSStateId state) {
return machine->config.parse_table + (state * machine->config.symbol_count);
}
void shift(TSStateMachine *machine, TSStateId parse_state) {
if (machine->lookahead->is_extra)
parse_state = ts_stack_top_state(&machine->stack);
ts_stack_push(&machine->stack, parse_state, machine->lookahead);
machine->lookahead = machine->next_lookahead;
machine->next_lookahead = NULL;
}
void shift_extra(TSStateMachine *machine) {
machine->lookahead->is_extra = 1;
shift(machine, 0);
}
void reduce(TSStateMachine *machine, TSSymbol symbol, size_t child_count) {
machine->next_lookahead = machine->lookahead;
machine->lookahead = ts_stack_reduce(
&machine->stack,
symbol,
child_count,
machine->config.hidden_symbol_flags, 1);
}
int reduce_extra(TSStateMachine *machine, TSSymbol symbol) {
TSTree *top_node = ts_stack_top_node(&machine->stack);
if (top_node->symbol == symbol && !top_node->is_extra) {
reduce(machine, symbol, 1);
machine->lookahead->is_extra = 1;
return 1;
} else {
return 0;
}
}
static size_t breakdown_stack(TSStateMachine *machine, TSInputEdit *edit) {
if (!edit) return 0;
TSStack *stack = &machine->stack;
size_t position = 0;
for (;;) {
TSTree *node = ts_stack_top_node(stack);
if (!node) break;
position = ts_stack_right_position(stack);
size_t child_count;
TSTree **children = ts_tree_immediate_children(node, &child_count);
if (position <= edit->position && !children) break;
stack->size--;
position -= ts_tree_total_size(node);
for (size_t i = 0; i < child_count && position < edit->position; i++) {
TSTree *child = children[i];
TSStateId state = ts_stack_top_state(stack);
TSStateId next_state = actions_for_state(machine, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child);
ts_tree_retain(child);
position += ts_tree_total_size(child);
}
ts_tree_release(node);
}
return position;
}
TSSymbol * expected_symbols(TSStateMachine *machine, size_t *count) {
*count = 0;
const TSParseAction *actions = actions_for_state(machine, ts_stack_top_state(&machine->stack));
for (size_t i = 0; i < machine->config.symbol_count; i++)
if (actions[i].type != TSParseActionTypeError)
++(*count);
size_t n = 0;
TSSymbol *result = malloc(*count * sizeof(*result));
for (TSSymbol i = 0; i < machine->config.symbol_count; i++)
if (actions[i].type != TSParseActionTypeError)
result[n++] = i;
return result;
}
int handle_error(TSStateMachine *machine) {
size_t count = 0;
const TSSymbol *inputs = expected_symbols(machine, &count);
TSTree *error = ts_tree_make_error(ts_lexer_lookahead_char(&machine->lexer),
count,
inputs,
0,
0);
for (;;) {
ts_tree_release(machine->lookahead);
size_t position = ts_lexer_position(&machine->lexer);
machine->lookahead = machine->config.lex_fn(&machine->lexer, ts_lex_state_error);
int at_end = 0;
if (ts_lexer_position(&machine->lexer) == position)
at_end = !ts_lexer_advance(&machine->lexer);
if (at_end || ts_tree_symbol(machine->lookahead) == ts_builtin_sym_end) {
ts_stack_push(&machine->stack, 0, error);
return 0;
}
/*
* Unwind the stack, looking for a state in which this token
* may appear after an error.
*/
for (size_t j = 0; j < machine->stack.size; j++) {
size_t i = machine->stack.size - 1 - j;
TSStateId stack_state = machine->stack.entries[i].state;
TSParseAction action_on_error = actions_for_state(machine, stack_state)[ts_builtin_sym_error];
if (action_on_error.type == TSParseActionTypeShift) {
TSStateId state_after_error = action_on_error.data.to_state;
if (actions_for_state(machine, state_after_error)[ts_tree_symbol(machine->lookahead)].type != TSParseActionTypeError) {
ts_stack_shrink(&machine->stack, i + 1);
ts_stack_push(&machine->stack, state_after_error, error);
return 1;
}
}
}
}
}
TSTree * get_tree_root(TSStateMachine *machine) {
TSStack *stack = &machine->stack;
TSTree *top_node = ts_stack_top_node(stack);
if (stack->size <= 1)
return top_node;
if (ts_tree_symbol(top_node) == ts_builtin_sym_error)
return top_node;
size_t immediate_child_count;
TSTree **immedate_children = ts_tree_immediate_children(top_node, &immediate_child_count);
stack->size--;
for (size_t i = 0; i < immediate_child_count; i++) {
TSTree *child = immedate_children[i];
child->is_extra = 0;
ts_tree_retain(child);
TSStateId state = ts_stack_top_state(stack);
TSStateId next_state = actions_for_state(machine, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child);
}
TSTree *new_node = ts_stack_reduce(stack,
top_node->symbol,
stack->size,
machine->config.hidden_symbol_flags,
0);
ts_tree_release(top_node);
return new_node;
}
TSParseAction get_next_action(TSStateMachine *machine) {
TSStateId state = ts_stack_top_state(&machine->stack);
if (!machine->lookahead)
machine->lookahead = machine->config.lex_fn(&machine->lexer, machine->config.lex_states[state]);
return actions_for_state(machine, state)[ts_tree_symbol(machine->lookahead)];
}
/*
* Public API
*/
TSStateMachine * ts_state_machine_make(size_t symbol_count,
const TSParseAction *parse_table,
const TSStateId *lex_states,
TSTree * (* lex_fn)(TSLexer *, TSStateId),
const int *hidden_symbol_flags) {
TSStateMachine *result = malloc(sizeof(TSStateMachine));
*result = (TSStateMachine) {
.lexer = ts_lexer_make(),
.stack = ts_stack_make(),
.config = {
.symbol_count = symbol_count,
.parse_table = parse_table,
.lex_states = lex_states,
.lex_fn = lex_fn,
.hidden_symbol_flags = hidden_symbol_flags,
},
};
return result;
}
void ts_state_machine_free(void *data) {
TSStateMachine *machine = (TSStateMachine *)data;
if (machine->lookahead) ts_tree_release(machine->lookahead);
if (machine->next_lookahead) ts_tree_release(machine->next_lookahead);
ts_stack_delete(&machine->stack);
free(machine);
}
void ts_state_machine_initialize(TSStateMachine *machine, TSInput input, TSInputEdit *edit) {
if (!edit) ts_stack_shrink(&machine->stack, 0);
machine->lookahead = NULL;
machine->next_lookahead = NULL;
size_t position = breakdown_stack(machine, edit);
input.seek_fn(input.data, position);
machine->lexer = ts_lexer_make();
machine->lexer.input = input;
ts_lexer_advance(&machine->lexer);
}
// #define TS_DEBUG_PARSE
#ifdef TS_DEBUG_PARSE
#include <stdio.h>
#define DEBUG_PARSE(...) fprintf(stderr, "\n" __VA_ARGS__)
#else
#define DEBUG_PARSE(...)
#endif
TSTree * ts_state_machine_parse(TSStateMachine *machine, const char **symbol_names) {
TSParseAction action = get_next_action(machine);
DEBUG_PARSE("LOOKAHEAD %s", symbol_names[ts_tree_symbol(machine->lookahead)]);
switch (action.type) {
case TSParseActionTypeShift:
DEBUG_PARSE("SHIFT %d", action.data.to_state);
shift(machine, action.data.to_state);
return NULL;
case TSParseActionTypeShiftExtra:
DEBUG_PARSE("SHIFT EXTRA");
shift_extra(machine);
return NULL;
case TSParseActionTypeReduce:
DEBUG_PARSE("REDUCE %s %d", symbol_names[action.data.symbol], action.data.child_count);
reduce(machine, action.data.symbol, action.data.child_count);
return NULL;
case TSParseActionTypeReduceExtra:
if (!reduce_extra(machine, action.data.symbol))
goto error;
DEBUG_PARSE("REDUCE EXTRA");
return NULL;
case TSParseActionTypeAccept:
DEBUG_PARSE("ACCEPT");
return get_tree_root(machine);
case TSParseActionTypeError:
goto error;
default:
return NULL;
}
error:
DEBUG_PARSE("ERROR");
if (handle_error(machine))
return NULL;
else
return get_tree_root(machine);
}