traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Move lr_parser implementation into a separate .c file

Browse source
This commit is contained in:
Max Brunsfeld 2014-06-04 13:34:37 -07:00
parent 63cde3967c
commit 9a4889176e
5 changed files with 371 additions and 246 deletions
Download patch file Download diff file Expand all files Collapse all files

69
include/tree_sitter/parser.h
View file

@ -16,8 +16,6 @@ extern "C" {
#define SYMBOL_NAMES \
static const char *ts_symbol_names[]
SYMBOL_NAMES;
#define HIDDEN_SYMBOLS \
static const int hidden_symbol_flags[SYMBOL_COUNT]
@ -27,9 +25,19 @@ static const int ubiquitous_symbol_flags[SYMBOL_COUNT]
#define LEX_STATES \
static ts_state_id ts_lex_states[STATE_COUNT]
#define PARSE_TABLE \
static const ts_parse_action ts_parse_actions[STATE_COUNT][SYMBOL_COUNT]
#define LEX_FN() \
static ts_tree * ts_lex(ts_lexer *lexer, ts_state_id lex_state)
#ifdef TS_DEBUG_LEX
#include <stdio.h>
#define DEBUG_LEX(...) fprintf(stderr, "\n" __VA_ARGS__)
#else
#define DEBUG_LEX(...)
#endif
#define START_LEXER() \
DEBUG_LEX("LEX %d", lex_state); \
char lookahead; \
@ -52,62 +60,15 @@ ts_lexer_start_token(lexer);
#define LEX_PANIC() \
{ DEBUG_LEX("LEX ERROR: unexpected state %d", lex_state); return NULL; }
#define PARSE_TABLE \
static const ts_parse_action ts_parse_actions[STATE_COUNT][SYMBOL_COUNT]
SYMBOL_NAMES;
#define SHIFT(to_state_value) \
{ .type = ts_parse_action_type_shift, .data = { .to_state = to_state_value } }
#define REDUCE(symbol_val, child_count_val) \
{ .type = ts_parse_action_type_reduce, .data = { .symbol = symbol_val, .child_count = child_count_val } }
#define ACCEPT_INPUT() \
{ .type = ts_parse_action_type_accept }
#ifdef TS_DEBUG_LEX
#include <stdio.h>
#define DEBUG_LEX(...) fprintf(stderr, "\n" __VA_ARGS__)
#else
#define DEBUG_LEX(...)
#endif
#ifdef TS_DEBUG_PARSE
#include <stdio.h>
#define DEBUG_PARSE(...) fprintf(stderr, "\n" __VA_ARGS__)
#else
#define DEBUG_PARSE(...)
#endif
static const ts_tree *
ts_parse(void *data, ts_input input, ts_input_edit *edit) {
static const ts_tree * ts_parse(void *data, ts_input input, ts_input_edit *edit) {
ts_lr_parser *parser = (ts_lr_parser *)data;
ts_lr_parser_initialize(parser, input, edit);
int done = 0;
while (!done) {
ts_parse_action action = ts_lr_parser_next_action(parser);
DEBUG_PARSE("LOOKAHEAD %s", ts_symbol_names[ts_tree_symbol(parser->lookahead)]);
switch (action.type) {
case ts_parse_action_type_shift:
DEBUG_PARSE("SHIFT %d", action.data.to_state);
ts_lr_parser_shift(parser, action.data.to_state);
break;
case ts_parse_action_type_reduce:
DEBUG_PARSE("REDUCE %s %d", ts_symbol_names[action.data.symbol], action.data.child_count);
ts_lr_parser_reduce(parser, action.data.symbol, action.data.child_count);
break;
case ts_parse_action_type_accept:
DEBUG_PARSE("ACCEPT");
done = 1;
break;
case ts_parse_action_type_error:
DEBUG_PARSE("ERROR");
done = !ts_lr_parser_handle_error(parser);
break;
}
for (;;) {
ts_tree *tree = ts_lr_parser_parse(parser, ts_symbol_names);
if (tree) return tree;
}
return ts_lr_parser_tree_root(parser);
}
#define EXPORT_PARSER(constructor_name) \

12
include/tree_sitter/parser/lexer.h
View file

@ -16,7 +16,7 @@ typedef struct {
int reached_end;
} ts_lexer;
static ts_lexer ts_lexer_make() {
static inline ts_lexer ts_lexer_make() {
ts_lexer result = {
.chunk = NULL,
.chunk_start = 0,
@ -29,15 +29,15 @@ static ts_lexer ts_lexer_make() {
return result;
}
static size_t ts_lexer_position(const ts_lexer *lexer) {
static inline size_t ts_lexer_position(const ts_lexer *lexer) {
return lexer->chunk_start + lexer->position_in_chunk;
}
static char ts_lexer_lookahead_char(const ts_lexer *lexer) {
static inline char ts_lexer_lookahead_char(const ts_lexer *lexer) {
return lexer->chunk[lexer->position_in_chunk];
}
static int ts_lexer_advance(ts_lexer *lexer) {
static inline int ts_lexer_advance(ts_lexer *lexer) {
static const char empty_chunk[1] = "";
if (lexer->position_in_chunk + 1 < lexer->chunk_size) {
lexer->position_in_chunk++;
@ -56,11 +56,11 @@ static int ts_lexer_advance(ts_lexer *lexer) {
return 1;
}
static void ts_lexer_start_token(ts_lexer *lexer) {
static inline void ts_lexer_start_token(ts_lexer *lexer) {
lexer->token_start_position = ts_lexer_position(lexer);
}
static ts_tree * ts_lexer_build_node(ts_lexer *lexer, ts_symbol symbol) {
static inline ts_tree * ts_lexer_build_node(ts_lexer *lexer, ts_symbol symbol) {
size_t current_position = ts_lexer_position(lexer);
size_t size = current_position - lexer->token_start_position;
size_t offset = lexer->token_start_position - lexer->token_end_position;

210
include/tree_sitter/parser/lr_parser.h
View file

@ -1,6 +1,10 @@
#ifndef TREE_SITTER_PARSER_LR_PARSER_H_
#define TREE_SITTER_PARSER_LR_PARSER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "tree_sitter/parser/stack.h"
#include "tree_sitter/parser/lexer.h"
@ -22,6 +26,15 @@ typedef struct {
} data;
} ts_parse_action;
#define SHIFT(to_state_value) \
{ .type = ts_parse_action_type_shift, .data = { .to_state = to_state_value } }
#define REDUCE(symbol_val, child_count_val) \
{ .type = ts_parse_action_type_reduce, .data = { .symbol = symbol_val, .child_count = child_count_val } }
#define ACCEPT_INPUT() \
{ .type = ts_parse_action_type_accept }
typedef struct {
ts_lexer lexer;
ts_stack stack;
@ -37,192 +50,17 @@ typedef struct {
} config;
} ts_lr_parser;
static ts_lr_parser *
ts_lr_parser_make(size_t symbol_count,
const ts_parse_action *parse_table,
const ts_state_id *lex_states,
ts_tree * (* lex_fn)(ts_lexer *, ts_state_id),
const int *hidden_symbol_flags,
const int *ubiquitous_symbol_flags) {
ts_lr_parser *result = malloc(sizeof(ts_lr_parser));
*result = (ts_lr_parser) {
.lexer = ts_lexer_make(),
.stack = ts_stack_make(),
.config = {
.symbol_count = symbol_count,
.parse_table = parse_table,
.lex_states = lex_states,
.lex_fn = lex_fn,
.hidden_symbol_flags = hidden_symbol_flags,
.ubiquitous_symbol_flags = ubiquitous_symbol_flags,
},
};
return result;
}
static const ts_parse_action *
ts_lr_parser_table_actions(ts_lr_parser *parser, ts_state_id state) {
return parser->config.parse_table + (state * parser->config.symbol_count);
}
static size_t
ts_lr_parser_breakdown_stack(ts_lr_parser *parser, ts_input_edit *edit) {
if (!edit) return 0;
ts_stack *stack = &parser->stack;
size_t position = 0;
for (;;) {
ts_tree *node = ts_stack_top_node(stack);
if (!node) break;
position = ts_stack_right_position(stack);
size_t child_count;
ts_tree **children = ts_tree_immediate_children(node, &child_count);
if (position <= edit->position && !children) break;
stack->size--;
position -= ts_tree_total_size(node);
for (size_t i = 0; i < child_count && position < edit->position; i++) {
ts_tree *child = children[i];
ts_state_id state = ts_stack_top_state(stack);
ts_state_id next_state = ts_lr_parser_table_actions(parser, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child);
ts_tree_retain(child);
position += ts_tree_total_size(child);
}
ts_tree_release(node);
}
return position;
}
static void
ts_lr_parser_initialize(ts_lr_parser *parser, ts_input input, ts_input_edit *edit) {
if (!edit) ts_stack_shrink(&parser->stack, 0);
parser->lookahead = NULL;
parser->next_lookahead = NULL;
size_t position = ts_lr_parser_breakdown_stack(parser, edit);
input.seek_fn(input.data, position);
parser->lexer = ts_lexer_make();
parser->lexer.input = input;
ts_lexer_advance(&parser->lexer);
}
static void
ts_lr_parser_shift(ts_lr_parser *parser, ts_state_id parse_state) {
ts_stack_push(&parser->stack, parse_state, parser->lookahead);
parser->lookahead = parser->next_lookahead;
parser->next_lookahead = NULL;
}
static void
ts_lr_parser_reduce(ts_lr_parser *parser, ts_symbol symbol, size_t child_count) {
parser->next_lookahead = parser->lookahead;
parser->lookahead = ts_stack_reduce(&parser->stack,
symbol,
child_count,
parser->config.hidden_symbol_flags,
parser->config.ubiquitous_symbol_flags);
}
static ts_symbol *
ts_lr_parser_expected_symbols(ts_lr_parser *parser, size_t *count) {
*count = 0;
const ts_parse_action *actions = ts_lr_parser_table_actions(parser, ts_stack_top_state(&parser->stack));
for (size_t i = 0; i < parser->config.symbol_count; i++)
if (actions[i].type != ts_parse_action_type_error)
++(*count);
size_t n = 0;
ts_symbol *result = malloc(*count * sizeof(*result));
for (ts_symbol i = 0; i < parser->config.symbol_count; i++)
if (actions[i].type != ts_parse_action_type_error)
result[n++] = i;
return result;
}
static int
ts_lr_parser_handle_error(ts_lr_parser *parser) {
size_t count = 0;
ts_symbol *expected_symbols = ts_lr_parser_expected_symbols(parser, &count);
ts_tree *error = ts_tree_make_error(ts_lexer_lookahead_char(&parser->lexer), count, expected_symbols, 0, 0);
for (;;) {
ts_tree_release(parser->lookahead);
size_t position = ts_lexer_position(&parser->lexer);
parser->lookahead = parser->config.lex_fn(&parser->lexer, ts_lex_state_error);
int at_end = 0;
if (ts_lexer_position(&parser->lexer) == position)
at_end = !ts_lexer_advance(&parser->lexer);
if (at_end || ts_tree_symbol(parser->lookahead) == ts_builtin_sym_end) {
ts_stack_push(&parser->stack, 0, error);
return 0;
}
/*
* Unwind the stack, looking for a state in which this token
* may appear after an error.
*/
for (size_t j = 0; j < parser->stack.size; j++) {
size_t i = parser->stack.size - 1 - j;
ts_state_id stack_state = parser->stack.entries[i].state;
ts_parse_action action_on_error = ts_lr_parser_table_actions(parser, stack_state)[ts_builtin_sym_error];
if (action_on_error.type == ts_parse_action_type_shift) {
ts_state_id state_after_error = action_on_error.data.to_state;
if (ts_lr_parser_table_actions(parser, state_after_error)[ts_tree_symbol(parser->lookahead)].type != ts_parse_action_type_error) {
ts_stack_shrink(&parser->stack, i + 1);
ts_stack_push(&parser->stack, state_after_error, error);
return 1;
}
}
}
}
}
static ts_tree *
ts_lr_parser_tree_root(ts_lr_parser *parser) {
ts_stack *stack = &parser->stack;
ts_tree *top_node = ts_stack_top_node(stack);
if (stack->size <= 1)
return top_node;
if (ts_tree_symbol(top_node) == ts_builtin_sym_error)
return top_node;
size_t immediate_child_count;
ts_tree **immedate_children = ts_tree_immediate_children(top_node, &immediate_child_count);
stack->size--;
for (size_t i = 0; i < immediate_child_count; i++) {
ts_tree *child = immedate_children[i];
ts_tree_retain(child);
ts_state_id state = ts_stack_top_state(stack);
ts_state_id next_state = ts_lr_parser_table_actions(parser, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child);
}
ts_tree *new_node = ts_stack_reduce(stack,
ts_tree_symbol(top_node),
stack->size,
parser->config.hidden_symbol_flags,
NULL);
ts_tree_release(top_node);
return new_node;
}
static ts_parse_action
ts_lr_parser_next_action(ts_lr_parser *parser) {
ts_state_id state = ts_stack_top_state(&parser->stack);
if (!parser->lookahead)
parser->lookahead = parser->config.lex_fn(&parser->lexer, parser->config.lex_states[state]);
return ts_lr_parser_table_actions(parser, state)[ts_tree_symbol(parser->lookahead)];
ts_lr_parser * ts_lr_parser_make(size_t symbol_count,
const ts_parse_action *parse_table,
const ts_state_id *lex_states,
ts_tree * (* lex_fn)(ts_lexer *, ts_state_id),
const int *hidden_symbol_flags,
const int *ubiquitous_symbol_flags);
void ts_lr_parser_initialize(ts_lr_parser *parser, ts_input input, ts_input_edit *edit);
ts_tree * ts_lr_parser_parse(ts_lr_parser *parser, const char **symbol_names);
#ifdef __cplusplus
}
#endif
#endif

105
spec/runtime/lr_parser_spec.cc Normal file
View file

@ -0,0 +1,105 @@
#include "runtime_spec_helper.h"
#include "helpers/spy_reader.h"
#include "tree_sitter/parser/lr_parser.h"
enum {
sym1 = 2,
sym2 = 3,
hidden_sym = 4,
};
const ts_parse_action parse_table[3][5] = {
[0] = {
[sym2] = SHIFT(12),
[hidden_sym] = SHIFT(12),
},
[1] = {
[sym1] = ACCEPT_INPUT(),
[sym2] = SHIFT(2),
[hidden_sym] = SHIFT(4),
},
[2] = {
[sym1] = SHIFT(3),
[sym2] = SHIFT(12),
[hidden_sym] = SHIFT(12),
},
};
ts_state_id lex_states[3] = {
[0] = 100,
[1] = 101,
[2] = 102,
};
int hidden_symbols[5] = {
[sym1] = 0,
[sym2] = 0,
[hidden_sym] = 1,
};
ts_tree *lex_fn_node_to_return;
ts_state_id lex_fn_state_received;
ts_lexer *lex_fn_lexer_received;
ts_tree * fake_lex(ts_lexer *lexer, ts_state_id state_id) {
lex_fn_lexer_received = lexer;
lex_fn_state_received = state_id;
return lex_fn_node_to_return;
}
START_TEST
describe("LR Parsers", [&]() {
ts_lr_parser *parser;
SpyReader *reader;
before_each([&]() {
reader = new SpyReader("some structured text", 5);
parser = ts_lr_parser_make(3, (const ts_parse_action *)parse_table, lex_states, fake_lex, hidden_symbols, nullptr);
});
after_each([&]() {
delete reader;
});
describe("when starting at the beginning of the input (edit is NULL)", [&]() {
before_each([&]() {
ts_lr_parser_initialize(parser, reader->input, nullptr);
});
it("runs the lexer with the lex state corresponding to the initial state", [&]() {
lex_fn_node_to_return = ts_tree_make_leaf(sym2, 5, 1);
ts_lr_parser_parse(parser, nullptr);
AssertThat(lex_fn_state_received, Equals(100));
});
describe("when the returned symbol indicates a shift action", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(sym2, 5, 1);
});
it("advances to the state specified in the action", [&]() {
ts_lr_parser_parse(parser, nullptr);
AssertThat(ts_stack_top_state(&parser->stack), Equals(12));
});
it("continues parsing (returns NULL)", [&]() {
auto result = ts_lr_parser_parse(parser, nullptr);
AssertThat(result, Equals((ts_tree *)nullptr));
});
});
describe("when the returned symbol indicates an error", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(sym1, 5, 1);
});
it("ends the parse, returning an error tree", [&]() {
auto result = ts_lr_parser_parse(parser, nullptr);
AssertThat(ts_tree_symbol(result), Equals(ts_builtin_sym_error));
});
});
});
});
END_TEST

221
src/runtime/lr_parser.c Normal file
View file

@ -0,0 +1,221 @@
#include "tree_sitter/parser/lr_parser.h"
/*
* Private
*/
static const ts_parse_action * actions_for_state(ts_lr_parser *parser, ts_state_id state) {
return parser->config.parse_table + (state * parser->config.symbol_count);
}
void shift(ts_lr_parser *parser, ts_state_id parse_state) {
ts_stack_push(&parser->stack, parse_state, parser->lookahead);
parser->lookahead = parser->next_lookahead;
parser->next_lookahead = NULL;
}
void reduce(ts_lr_parser *parser, ts_symbol symbol, size_t child_count) {
parser->next_lookahead = parser->lookahead;
parser->lookahead = ts_stack_reduce(&parser->stack,
symbol,
child_count,
parser->config.hidden_symbol_flags,
parser->config.ubiquitous_symbol_flags);
}
static size_t breakdown_stack(ts_lr_parser *parser, ts_input_edit *edit) {
if (!edit) return 0;
ts_stack *stack = &parser->stack;
size_t position = 0;
for (;;) {
ts_tree *node = ts_stack_top_node(stack);
if (!node) break;
position = ts_stack_right_position(stack);
size_t child_count;
ts_tree **children = ts_tree_immediate_children(node, &child_count);
if (position <= edit->position && !children) break;
stack->size--;
position -= ts_tree_total_size(node);
for (size_t i = 0; i < child_count && position < edit->position; i++) {
ts_tree *child = children[i];
ts_state_id state = ts_stack_top_state(stack);
ts_state_id next_state = actions_for_state(parser, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child);
ts_tree_retain(child);
position += ts_tree_total_size(child);
}
ts_tree_release(node);
}
return position;
}
ts_symbol * expected_symbols(ts_lr_parser *parser, size_t *count) {
*count = 0;
const ts_parse_action *actions = actions_for_state(parser, ts_stack_top_state(&parser->stack));
for (size_t i = 0; i < parser->config.symbol_count; i++)
if (actions[i].type != ts_parse_action_type_error)
++(*count);
size_t n = 0;
ts_symbol *result = malloc(*count * sizeof(*result));
for (ts_symbol i = 0; i < parser->config.symbol_count; i++)
if (actions[i].type != ts_parse_action_type_error)
result[n++] = i;
return result;
}
int handle_error(ts_lr_parser *parser) {
size_t count = 0;
ts_tree *error = ts_tree_make_error(ts_lexer_lookahead_char(&parser->lexer),
count,
expected_symbols(parser, &count),
0,
0);
for (;;) {
ts_tree_release(parser->lookahead);
size_t position = ts_lexer_position(&parser->lexer);
parser->lookahead = parser->config.lex_fn(&parser->lexer, ts_lex_state_error);
int at_end = 0;
if (ts_lexer_position(&parser->lexer) == position)
at_end = !ts_lexer_advance(&parser->lexer);
if (at_end || ts_tree_symbol(parser->lookahead) == ts_builtin_sym_end) {
ts_stack_push(&parser->stack, 0, error);
return 0;
}
/*
* Unwind the stack, looking for a state in which this token
* may appear after an error.
*/
for (size_t j = 0; j < parser->stack.size; j++) {
size_t i = parser->stack.size - 1 - j;
ts_state_id stack_state = parser->stack.entries[i].state;
ts_parse_action action_on_error = actions_for_state(parser, stack_state)[ts_builtin_sym_error];
if (action_on_error.type == ts_parse_action_type_shift) {
ts_state_id state_after_error = action_on_error.data.to_state;
if (actions_for_state(parser, state_after_error)[ts_tree_symbol(parser->lookahead)].type != ts_parse_action_type_error) {
ts_stack_shrink(&parser->stack, i + 1);
ts_stack_push(&parser->stack, state_after_error, error);
return 1;
}
}
}
}
}
ts_tree * get_tree_root(ts_lr_parser *parser) {
ts_stack *stack = &parser->stack;
ts_tree *top_node = ts_stack_top_node(stack);
if (stack->size <= 1)
return top_node;
if (ts_tree_symbol(top_node) == ts_builtin_sym_error)
return top_node;
size_t immediate_child_count;
ts_tree **immedate_children = ts_tree_immediate_children(top_node, &immediate_child_count);
stack->size--;
for (size_t i = 0; i < immediate_child_count; i++) {
ts_tree *child = immedate_children[i];
ts_tree_retain(child);
ts_state_id state = ts_stack_top_state(stack);
ts_state_id next_state = actions_for_state(parser, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child);
}
ts_tree *new_node = ts_stack_reduce(stack,
ts_tree_symbol(top_node),
stack->size,
parser->config.hidden_symbol_flags,
NULL);
ts_tree_release(top_node);
return new_node;
}
ts_parse_action get_next_action(ts_lr_parser *parser) {
ts_state_id state = ts_stack_top_state(&parser->stack);
if (!parser->lookahead)
parser->lookahead = parser->config.lex_fn(&parser->lexer, parser->config.lex_states[state]);
return actions_for_state(parser, state)[ts_tree_symbol(parser->lookahead)];
}
/*
* Public API
*/
ts_lr_parser * ts_lr_parser_make(size_t symbol_count,
const ts_parse_action *parse_table,
const ts_state_id *lex_states,
ts_tree * (* lex_fn)(ts_lexer *, ts_state_id),
const int *hidden_symbol_flags,
const int *ubiquitous_symbol_flags) {
ts_lr_parser *result = malloc(sizeof(ts_lr_parser));
*result = (ts_lr_parser) {
.lexer = ts_lexer_make(),
.stack = ts_stack_make(),
.config = {
.symbol_count = symbol_count,
.parse_table = parse_table,
.lex_states = lex_states,
.lex_fn = lex_fn,
.hidden_symbol_flags = hidden_symbol_flags,
.ubiquitous_symbol_flags = ubiquitous_symbol_flags,
},
};
return result;
}
void ts_lr_parser_initialize(ts_lr_parser *parser, ts_input input, ts_input_edit *edit) {
if (!edit) ts_stack_shrink(&parser->stack, 0);
parser->lookahead = NULL;
parser->next_lookahead = NULL;
size_t position = breakdown_stack(parser, edit);
input.seek_fn(input.data, position);
parser->lexer = ts_lexer_make();
parser->lexer.input = input;
ts_lexer_advance(&parser->lexer);
}
#ifdef TS_DEBUG_PARSE
#include <stdio.h>
#define DEBUG_PARSE(...) fprintf(stderr, "\n" __VA_ARGS__)
#else
#define DEBUG_PARSE(...)
#endif
ts_tree * ts_lr_parser_parse(ts_lr_parser *parser, const char **symbol_names) {
ts_parse_action action = get_next_action(parser);
DEBUG_PARSE("LOOKAHEAD %s", symbol_names[ts_tree_symbol(parser->lookahead)]);
switch (action.type) {
case ts_parse_action_type_shift:
DEBUG_PARSE("SHIFT %d", action.data.to_state);
shift(parser, action.data.to_state);
return NULL;
case ts_parse_action_type_reduce:
DEBUG_PARSE("REDUCE %s %d", symbol_names[action.data.symbol], action.data.child_count);
reduce(parser, action.data.symbol, action.data.child_count);
return NULL;
case ts_parse_action_type_accept:
DEBUG_PARSE("ACCEPT");
return get_tree_root(parser);
case ts_parse_action_type_error:
DEBUG_PARSE("ERROR");
if (handle_error(parser))
return NULL;
else
return get_tree_root(parser);
}
}