traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rename type ts_lr_parser -> TSStateMachine

Browse source
This commit is contained in:
Max Brunsfeld 2014-06-28 19:22:16 -07:00
parent 27f6eb725d
commit 0ec3faba3e
5 changed files with 96 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

12
include/tree_sitter/parser.h
View file

@ -11,7 +11,7 @@ extern "C" {
#include "tree_sitter/runtime.h" #include "tree_sitter/runtime.h"
#include "tree_sitter/parser/lexer.h" #include "tree_sitter/parser/lexer.h"
#include "tree_sitter/parser/stack.h" #include "tree_sitter/parser/stack.h"
#include "tree_sitter/parser/lr_parser.h" #include "tree_sitter/parser/state_machine.h"
#define SYMBOL_NAMES \ #define SYMBOL_NAMES \
static const char *ts_symbol_names[] static const char *ts_symbol_names[]
@ -73,10 +73,10 @@ ts_lexer_start_token(lexer);
SYMBOL_NAMES; SYMBOL_NAMES;
static const TSTree * ts_parse(void *data, TSInput input, TSInputEdit *edit) { static const TSTree * ts_parse(void *data, TSInput input, TSInputEdit *edit) {
ts_lr_parser *parser = (ts_lr_parser *)data; TSStateMachine *parser = (TSStateMachine *)data;
ts_lr_parser_initialize(parser, input, edit); ts_state_machine_initialize(parser, input, edit);
for (;;) { for (;;) {
const TSTree *tree = ts_lr_parser_parse(parser, ts_symbol_names); const TSTree *tree = ts_state_machine_parse(parser, ts_symbol_names);
if (tree) return tree; if (tree) return tree;
} }
} }
@ -85,9 +85,9 @@ static const TSTree * ts_parse(void *data, TSInput input, TSInputEdit *edit) {
TSParser constructor_name() { \ TSParser constructor_name() { \
return (TSParser) { \ return (TSParser) { \
.parse_fn = ts_parse, \ .parse_fn = ts_parse, \
.free_fn = ts_lr_parser_free, \ .free_fn = ts_state_machine_free, \
.symbol_names = ts_symbol_names, \ .symbol_names = ts_symbol_names, \
.data = ts_lr_parser_make( \ .data = ts_state_machine_make( \
SYMBOL_COUNT, \ SYMBOL_COUNT, \
(const TSParseAction *)ts_parse_actions, \ (const TSParseAction *)ts_parse_actions, \
ts_lex_states, \ ts_lex_states, \

19
include/tree_sitter/parser/lr_parser.h → include/tree_sitter/parser/state_machine.h
View file

@ -51,16 +51,17 @@ typedef struct {
const TSStateId *lex_states; const TSStateId *lex_states;
TSTree * (* lex_fn)(TSLexer *, TSStateId); TSTree * (* lex_fn)(TSLexer *, TSStateId);
} config; } config;
} ts_lr_parser; } TSStateMachine;
ts_lr_parser * ts_lr_parser_make(size_t symbol_count, TSStateMachine * ts_state_machine_make(
const TSParseAction *parse_table, size_t symbol_count,
const TSStateId *lex_states, const TSParseAction *parse_table,
TSTree * (* lex_fn)(TSLexer *, TSStateId), const TSStateId *lex_states,
const int *hidden_symbol_flags); TSTree * (* lex_fn)(TSLexer *, TSStateId),
void ts_lr_parser_free(void *data); const int *hidden_symbol_flags);
void ts_lr_parser_initialize(ts_lr_parser *parser, TSInput input, TSInputEdit *edit); void ts_state_machine_free(void *data);
TSTree * ts_lr_parser_parse(ts_lr_parser *parser, const char **symbol_names); void ts_state_machine_initialize(TSStateMachine *, TSInput, TSInputEdit *);
TSTree * ts_state_machine_parse(TSStateMachine *, const char **symbol_names);
#ifdef __cplusplus #ifdef __cplusplus
} }

2
spec/runtime/helpers/dummy_parser.h
View file

@ -5,7 +5,7 @@
extern "C" { extern "C" {
#endif #endif
#include "tree_sitter/parser/lr_parser.h" #include "tree_sitter/parser/state_machine.h"
enum { enum {
dummy_sym1 = 2, dummy_sym1 = 2,

16
spec/runtime/lr_parser_spec.cc → spec/runtime/state_machine_spec.cc
View file

@ -1,7 +1,7 @@
#include "runtime/runtime_spec_helper.h" #include "runtime/runtime_spec_helper.h"
#include "runtime/helpers/spy_reader.h" #include "runtime/helpers/spy_reader.h"
#include "runtime/helpers/dummy_parser.h" #include "runtime/helpers/dummy_parser.h"
#include "tree_sitter/parser/lr_parser.h" #include "tree_sitter/parser/state_machine.h"
TSTree *lex_fn_node_to_return; TSTree *lex_fn_node_to_return;
TSStateId lex_fn_state_received; TSStateId lex_fn_state_received;
@ -16,12 +16,12 @@ TSTree * fake_lex(TSLexer *lexer, TSStateId state_id) {
START_TEST START_TEST
describe("LR Parsers", [&]() { describe("LR Parsers", [&]() {
ts_lr_parser *parser; TSStateMachine *parser;
SpyReader *reader; SpyReader *reader;
before_each([&]() { before_each([&]() {
reader = new SpyReader("some structured text", 5); reader = new SpyReader("some structured text", 5);
parser = ts_lr_parser_make(dummy_parser.symbol_count, parser = ts_state_machine_make(dummy_parser.symbol_count,
(const TSParseAction *)dummy_parser.parse_table, (const TSParseAction *)dummy_parser.parse_table,
dummy_parser.lex_states, dummy_parser.lex_states,
fake_lex, fake_lex,
@ -34,12 +34,12 @@ describe("LR Parsers", [&]() {
describe("when starting at the beginning of the input (edit is NULL)", [&]() { describe("when starting at the beginning of the input (edit is NULL)", [&]() {
before_each([&]() { before_each([&]() {
ts_lr_parser_initialize(parser, reader->input, nullptr); ts_state_machine_initialize(parser, reader->input, nullptr);
}); });
it("runs the lexer with the lex state corresponding to the initial state", [&]() { it("runs the lexer with the lex state corresponding to the initial state", [&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1); lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1);
ts_lr_parser_parse(parser, nullptr); ts_state_machine_parse(parser, nullptr);
AssertThat(lex_fn_state_received, Equals(100)); AssertThat(lex_fn_state_received, Equals(100));
}); });
@ -49,12 +49,12 @@ describe("LR Parsers", [&]() {
}); });
it("advances to the state specified in the action", [&]() { it("advances to the state specified in the action", [&]() {
ts_lr_parser_parse(parser, nullptr); ts_state_machine_parse(parser, nullptr);
AssertThat(ts_stack_top_state(&parser->stack), Equals(12)); AssertThat(ts_stack_top_state(&parser->stack), Equals(12));
}); });
it("continues parsing (returns NULL)", [&]() { it("continues parsing (returns NULL)", [&]() {
auto result = ts_lr_parser_parse(parser, nullptr); auto result = ts_state_machine_parse(parser, nullptr);
AssertThat(result, Equals((TSTree *)nullptr)); AssertThat(result, Equals((TSTree *)nullptr));
}); });
}); });
@ -65,7 +65,7 @@ describe("LR Parsers", [&]() {
}); });
it("ends the parse, returning an error tree", [&]() { it("ends the parse, returning an error tree", [&]() {
auto result = ts_lr_parser_parse(parser, nullptr); auto result = ts_state_machine_parse(parser, nullptr);
AssertThat(ts_tree_symbol(result), Equals(ts_builtin_sym_error)); AssertThat(ts_tree_symbol(result), Equals(ts_builtin_sym_error));
}); });
}); });

142
src/runtime/lr_parser.c → src/runtime/state_machine.c
View file

@ -1,34 +1,34 @@
#include "tree_sitter/parser/lr_parser.h" #include "tree_sitter/parser/state_machine.h"
#include "runtime/tree.h" #include "runtime/tree.h"
/* /*
* Private * Private
*/ */
static const TSParseAction * actions_for_state(ts_lr_parser *parser, TSStateId state) { static const TSParseAction * actions_for_state(TSStateMachine *machine, TSStateId state) {
return parser->config.parse_table + (state * parser->config.symbol_count); return machine->config.parse_table + (state * machine->config.symbol_count);
} }
void shift(ts_lr_parser *parser, TSStateId parse_state, int is_extra) { void shift(TSStateMachine *machine, TSStateId parse_state, int is_extra) {
parser->lookahead->is_extra = is_extra; machine->lookahead->is_extra = is_extra;
ts_stack_push(&parser->stack, parse_state, parser->lookahead); ts_stack_push(&machine->stack, parse_state, machine->lookahead);
parser->lookahead = parser->next_lookahead; machine->lookahead = machine->next_lookahead;
parser->next_lookahead = NULL; machine->next_lookahead = NULL;
} }
void reduce(ts_lr_parser *parser, TSSymbol symbol, size_t child_count) { void reduce(TSStateMachine *machine, TSSymbol symbol, size_t child_count) {
parser->next_lookahead = parser->lookahead; machine->next_lookahead = machine->lookahead;
parser->lookahead = ts_stack_reduce(&parser->stack, machine->lookahead = ts_stack_reduce(&machine->stack,
symbol, symbol,
child_count, child_count,
parser->config.hidden_symbol_flags, machine->config.hidden_symbol_flags,
1); 1);
} }
static size_t breakdown_stack(ts_lr_parser *parser, TSInputEdit *edit) { static size_t breakdown_stack(TSStateMachine *machine, TSInputEdit *edit) {
if (!edit) return 0; if (!edit) return 0;
TSStack *stack = &parser->stack; TSStack *stack = &machine->stack;
size_t position = 0; size_t position = 0;
for (;;) { for (;;) {
@ -46,7 +46,7 @@ static size_t breakdown_stack(ts_lr_parser *parser, TSInputEdit *edit) {
for (size_t i = 0; i < child_count && position < edit->position; i++) { for (size_t i = 0; i < child_count && position < edit->position; i++) {
TSTree *child = children[i]; TSTree *child = children[i];
TSStateId state = ts_stack_top_state(stack); TSStateId state = ts_stack_top_state(stack);
TSStateId next_state = actions_for_state(parser, state)[ts_tree_symbol(child)].data.to_state; TSStateId next_state = actions_for_state(machine, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child); ts_stack_push(stack, next_state, child);
ts_tree_retain(child); ts_tree_retain(child);
position += ts_tree_total_size(child); position += ts_tree_total_size(child);
@ -58,42 +58,42 @@ static size_t breakdown_stack(ts_lr_parser *parser, TSInputEdit *edit) {
return position; return position;
} }
TSSymbol * expected_symbols(ts_lr_parser *parser, size_t *count) { TSSymbol * expected_symbols(TSStateMachine *machine, size_t *count) {
*count = 0; *count = 0;
const TSParseAction *actions = actions_for_state(parser, ts_stack_top_state(&parser->stack)); const TSParseAction *actions = actions_for_state(machine, ts_stack_top_state(&machine->stack));
for (size_t i = 0; i < parser->config.symbol_count; i++) for (size_t i = 0; i < machine->config.symbol_count; i++)
if (actions[i].type != ts_parse_action_type_error) if (actions[i].type != ts_parse_action_type_error)
++(*count); ++(*count);
size_t n = 0; size_t n = 0;
TSSymbol *result = malloc(*count * sizeof(*result)); TSSymbol *result = malloc(*count * sizeof(*result));
for (TSSymbol i = 0; i < parser->config.symbol_count; i++) for (TSSymbol i = 0; i < machine->config.symbol_count; i++)
if (actions[i].type != ts_parse_action_type_error) if (actions[i].type != ts_parse_action_type_error)
result[n++] = i; result[n++] = i;
return result; return result;
} }
int handle_error(ts_lr_parser *parser) { int handle_error(TSStateMachine *machine) {
size_t count = 0; size_t count = 0;
const TSSymbol *inputs = expected_symbols(parser, &count); const TSSymbol *inputs = expected_symbols(machine, &count);
TSTree *error = ts_tree_make_error(ts_lexer_lookahead_char(&parser->lexer), TSTree *error = ts_tree_make_error(ts_lexer_lookahead_char(&machine->lexer),
count, count,
inputs, inputs,
0, 0,
0); 0);
for (;;) { for (;;) {
ts_tree_release(parser->lookahead); ts_tree_release(machine->lookahead);
size_t position = ts_lexer_position(&parser->lexer); size_t position = ts_lexer_position(&machine->lexer);
parser->lookahead = parser->config.lex_fn(&parser->lexer, ts_lex_state_error); machine->lookahead = machine->config.lex_fn(&machine->lexer, ts_lex_state_error);
int at_end = 0; int at_end = 0;
if (ts_lexer_position(&parser->lexer) == position) if (ts_lexer_position(&machine->lexer) == position)
at_end = !ts_lexer_advance(&parser->lexer); at_end = !ts_lexer_advance(&machine->lexer);
if (at_end || ts_tree_symbol(parser->lookahead) == ts_builtin_sym_end) { if (at_end || ts_tree_symbol(machine->lookahead) == ts_builtin_sym_end) {
ts_stack_push(&parser->stack, 0, error); ts_stack_push(&machine->stack, 0, error);
return 0; return 0;
} }
@ -101,15 +101,15 @@ int handle_error(ts_lr_parser *parser) {
* Unwind the stack, looking for a state in which this token * Unwind the stack, looking for a state in which this token
* may appear after an error. * may appear after an error.
*/ */
for (size_t j = 0; j < parser->stack.size; j++) { for (size_t j = 0; j < machine->stack.size; j++) {
size_t i = parser->stack.size - 1 - j; size_t i = machine->stack.size - 1 - j;
TSStateId stack_state = parser->stack.entries[i].state; TSStateId stack_state = machine->stack.entries[i].state;
TSParseAction action_on_error = actions_for_state(parser, stack_state)[ts_builtin_sym_error]; TSParseAction action_on_error = actions_for_state(machine, stack_state)[ts_builtin_sym_error];
if (action_on_error.type == ts_parse_action_type_shift) { if (action_on_error.type == ts_parse_action_type_shift) {
TSStateId state_after_error = action_on_error.data.to_state; TSStateId state_after_error = action_on_error.data.to_state;
if (actions_for_state(parser, state_after_error)[ts_tree_symbol(parser->lookahead)].type != ts_parse_action_type_error) { if (actions_for_state(machine, state_after_error)[ts_tree_symbol(machine->lookahead)].type != ts_parse_action_type_error) {
ts_stack_shrink(&parser->stack, i + 1); ts_stack_shrink(&machine->stack, i + 1);
ts_stack_push(&parser->stack, state_after_error, error); ts_stack_push(&machine->stack, state_after_error, error);
return 1; return 1;
} }
} }
@ -117,8 +117,8 @@ int handle_error(ts_lr_parser *parser) {
} }
} }
TSTree * get_tree_root(ts_lr_parser *parser) { TSTree * get_tree_root(TSStateMachine *machine) {
TSStack *stack = &parser->stack; TSStack *stack = &machine->stack;
TSTree *top_node = ts_stack_top_node(stack); TSTree *top_node = ts_stack_top_node(stack);
if (stack->size <= 1) if (stack->size <= 1)
return top_node; return top_node;
@ -134,37 +134,37 @@ TSTree * get_tree_root(ts_lr_parser *parser) {
child->is_extra = 0; child->is_extra = 0;
ts_tree_retain(child); ts_tree_retain(child);
TSStateId state = ts_stack_top_state(stack); TSStateId state = ts_stack_top_state(stack);
TSStateId next_state = actions_for_state(parser, state)[ts_tree_symbol(child)].data.to_state; TSStateId next_state = actions_for_state(machine, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child); ts_stack_push(stack, next_state, child);
} }
TSTree *new_node = ts_stack_reduce(stack, TSTree *new_node = ts_stack_reduce(stack,
top_node->symbol, top_node->symbol,
stack->size, stack->size,
parser->config.hidden_symbol_flags, machine->config.hidden_symbol_flags,
0); 0);
ts_tree_release(top_node); ts_tree_release(top_node);
return new_node; return new_node;
} }
TSParseAction get_next_action(ts_lr_parser *parser) { TSParseAction get_next_action(TSStateMachine *machine) {
TSStateId state = ts_stack_top_state(&parser->stack); TSStateId state = ts_stack_top_state(&machine->stack);
if (!parser->lookahead) if (!machine->lookahead)
parser->lookahead = parser->config.lex_fn(&parser->lexer, parser->config.lex_states[state]); machine->lookahead = machine->config.lex_fn(&machine->lexer, machine->config.lex_states[state]);
return actions_for_state(parser, state)[ts_tree_symbol(parser->lookahead)]; return actions_for_state(machine, state)[ts_tree_symbol(machine->lookahead)];
} }
/* /*
* Public API * Public API
*/ */
ts_lr_parser * ts_lr_parser_make(size_t symbol_count, TSStateMachine * ts_state_machine_make(size_t symbol_count,
const TSParseAction *parse_table, const TSParseAction *parse_table,
const TSStateId *lex_states, const TSStateId *lex_states,
TSTree * (* lex_fn)(TSLexer *, TSStateId), TSTree * (* lex_fn)(TSLexer *, TSStateId),
const int *hidden_symbol_flags) { const int *hidden_symbol_flags) {
ts_lr_parser *result = malloc(sizeof(ts_lr_parser)); TSStateMachine *result = malloc(sizeof(TSStateMachine));
*result = (ts_lr_parser) { *result = (TSStateMachine) {
.lexer = ts_lexer_make(), .lexer = ts_lexer_make(),
.stack = ts_stack_make(), .stack = ts_stack_make(),
.config = { .config = {
@ -178,25 +178,25 @@ ts_lr_parser * ts_lr_parser_make(size_t symbol_count,
return result; return result;
} }
void ts_lr_parser_free(void *data) { void ts_state_machine_free(void *data) {
ts_lr_parser *parser = (ts_lr_parser *)data; TSStateMachine *machine = (TSStateMachine *)data;
if (parser->lookahead) ts_tree_release(parser->lookahead); if (machine->lookahead) ts_tree_release(machine->lookahead);
if (parser->next_lookahead) ts_tree_release(parser->next_lookahead); if (machine->next_lookahead) ts_tree_release(machine->next_lookahead);
ts_stack_delete(&parser->stack); ts_stack_delete(&machine->stack);
free(parser); free(machine);
} }
void ts_lr_parser_initialize(ts_lr_parser *parser, TSInput input, TSInputEdit *edit) { void ts_state_machine_initialize(TSStateMachine *machine, TSInput input, TSInputEdit *edit) {
if (!edit) ts_stack_shrink(&parser->stack, 0); if (!edit) ts_stack_shrink(&machine->stack, 0);
parser->lookahead = NULL; machine->lookahead = NULL;
parser->next_lookahead = NULL; machine->next_lookahead = NULL;
size_t position = breakdown_stack(parser, edit); size_t position = breakdown_stack(machine, edit);
input.seek_fn(input.data, position); input.seek_fn(input.data, position);
parser->lexer = ts_lexer_make(); machine->lexer = ts_lexer_make();
parser->lexer.input = input; machine->lexer.input = input;
ts_lexer_advance(&parser->lexer); ts_lexer_advance(&machine->lexer);
} }
/* #define TS_DEBUG_PARSE */ /* #define TS_DEBUG_PARSE */
@ -208,31 +208,31 @@ void ts_lr_parser_initialize(ts_lr_parser *parser, TSInput input, TSInputEdit *e
#define DEBUG_PARSE(...) #define DEBUG_PARSE(...)
#endif #endif
TSTree * ts_lr_parser_parse(ts_lr_parser *parser, const char **symbol_names) { TSTree * ts_state_machine_parse(TSStateMachine *machine, const char **symbol_names) {
TSParseAction action = get_next_action(parser); TSParseAction action = get_next_action(machine);
DEBUG_PARSE("LOOKAHEAD %s", symbol_names[ts_tree_symbol(parser->lookahead)]); DEBUG_PARSE("LOOKAHEAD %s", symbol_names[ts_tree_symbol(machine->lookahead)]);
switch (action.type) { switch (action.type) {
case ts_parse_action_type_shift: case ts_parse_action_type_shift:
DEBUG_PARSE("SHIFT %d", action.data.to_state); DEBUG_PARSE("SHIFT %d", action.data.to_state);
shift(parser, action.data.to_state, 0); shift(machine, action.data.to_state, 0);
return NULL; return NULL;
case ts_parse_action_type_shift_extra: case ts_parse_action_type_shift_extra:
DEBUG_PARSE("SHIFT EXTRA"); DEBUG_PARSE("SHIFT EXTRA");
shift(parser, ts_stack_top_state(&parser->stack), 1); shift(machine, ts_stack_top_state(&machine->stack), 1);
return NULL; return NULL;
case ts_parse_action_type_reduce: case ts_parse_action_type_reduce:
DEBUG_PARSE("REDUCE %s %d", symbol_names[action.data.symbol], action.data.child_count); DEBUG_PARSE("REDUCE %s %d", symbol_names[action.data.symbol], action.data.child_count);
reduce(parser, action.data.symbol, action.data.child_count); reduce(machine, action.data.symbol, action.data.child_count);
return NULL; return NULL;
case ts_parse_action_type_accept: case ts_parse_action_type_accept:
DEBUG_PARSE("ACCEPT"); DEBUG_PARSE("ACCEPT");
return get_tree_root(parser); return get_tree_root(machine);
case ts_parse_action_type_error: case ts_parse_action_type_error:
DEBUG_PARSE("ERROR"); DEBUG_PARSE("ERROR");
if (handle_error(parser)) if (handle_error(machine))
return NULL; return NULL;
else else
return get_tree_root(parser); return get_tree_root(machine);
default: default:
return NULL; return NULL;
} }