traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove generated parsers' dependency on the runtime library

Browse source 
Generated parsers no longer export a parser constructor function.
They now export an opaque Language object which can be set on
Documents directly. This way, the logic for constructing parsers
lives entirely in the runtime. The Languages are just structs which
have no load-time dependency on the runtime
This commit is contained in:
Max Brunsfeld 2014-07-30 23:40:02 -07:00
parent 048a479b5f
commit eecbcccee0
21 changed files with 219 additions and 173 deletions
Download patch file Download diff file Expand all files Collapse all files

26
src/runtime/document.c
View file

@ -2,10 +2,11 @@
#include "tree_sitter/parser.h"
#include "runtime/tree.h"
#include "runtime/node.h"
#include "runtime/parser.h"
#include <string.h>
struct TSDocument {
TSParser *parser;
TSParser parser;
const TSTree *tree;
TSInput input;
};
@ -14,24 +15,21 @@ TSDocument *ts_document_make() {
TSDocument *document = malloc(sizeof(TSDocument));
*document = (TSDocument) {
.input = (TSInput) { .data = NULL, .read_fn = NULL, .seek_fn = NULL },
.parser = NULL,
.tree = NULL
};
return document;
}
void ts_document_free(TSDocument *document) {
if (document->parser)
ts_parser_free(document->parser);
ts_parser_destroy(&document->parser);
if (document->input.release_fn)
document->input.release_fn(document->input.data);
free(document);
}
void ts_document_set_parser(TSDocument *document, TSParser *parser) {
if (document->parser)
ts_parser_free(document->parser);
document->parser = parser;
void ts_document_set_language(TSDocument *document, const TSLanguage *language) {
ts_parser_destroy(&document->parser);
document->parser = ts_parser_make(language);
}
const TSTree *ts_document_tree(const TSDocument *document) {
@ -39,22 +37,21 @@ const TSTree *ts_document_tree(const TSDocument *document) {
}
const char *ts_document_string(const TSDocument *document) {
return ts_tree_string(document->tree,
ts_parser_config(document->parser).symbol_names);
return ts_tree_string(document->tree, document->parser.language->symbol_names);
}
void ts_document_set_input(TSDocument *document, TSInput input) {
document->input = input;
document->tree = ts_parser_parse(document->parser, document->input, NULL);
document->tree = ts_parser_parse(&document->parser, document->input, NULL);
}
void ts_document_edit(TSDocument *document, TSInputEdit edit) {
document->tree = ts_parser_parse(document->parser, document->input, &edit);
document->tree = ts_parser_parse(&document->parser, document->input, &edit);
}
const char *ts_document_symbol_name(const TSDocument *document,
const TSTree *tree) {
return ts_parser_config(document->parser).symbol_names[tree->symbol];
return document->parser.language->symbol_names[tree->symbol];
}
typedef struct {
@ -97,8 +94,7 @@ void ts_document_set_input_string(TSDocument *document, const char *text) {
}
TSNode *ts_document_root_node(const TSDocument *document) {
return ts_node_make_root(document->tree,
document->parser->config.symbol_names);
return ts_node_make_root(document->tree, document->parser.language->symbol_names);
}
TSNode *ts_document_get_node(const TSDocument *document, size_t pos) {

29
src/runtime/lexer.c
View file

@ -1,18 +1,7 @@
#include "tree_sitter/parser.h"
#include "runtime/tree.h"
TSLexer ts_lexer_make() {
return (TSLexer) { .chunk = NULL,
.debug = 0,
.chunk_start = 0,
.chunk_size = 0,
.position_in_chunk = 0,
.token_start_position = 0,
.token_end_position = 0,
.reached_end = 0 };
}
int ts_lexer_advance(TSLexer *lexer) {
static int advance(TSLexer *lexer) {
static const char *empty_chunk = "";
if (lexer->position_in_chunk + 1 < lexer->chunk_size) {
lexer->position_in_chunk++;
@ -31,10 +20,24 @@ int ts_lexer_advance(TSLexer *lexer) {
return 1;
}
TSTree *ts_lexer_build_node(TSLexer *lexer, TSSymbol symbol, int is_hidden) {
static TSTree * accept(TSLexer *lexer, TSSymbol symbol, int is_hidden) {
size_t current_position = ts_lexer_position(lexer);
size_t size = current_position - lexer->token_start_position;
size_t offset = lexer->token_start_position - lexer->token_end_position;
lexer->token_end_position = current_position;
return ts_tree_make_leaf(symbol, size, offset, is_hidden);
}
TSLexer ts_lexer_make() {
return (TSLexer) { .chunk = NULL,
.debug = 0,
.chunk_start = 0,
.chunk_size = 0,
.position_in_chunk = 0,
.token_start_position = 0,
.token_end_position = 0,
.reached_end = 0,
.advance_fn = advance,
.accept_fn = accept,
};
}

16
src/runtime/lexer.h Normal file
View file

@ -0,0 +1,16 @@
#ifndef RUNTIME_LEXER_H_
#define RUNTIME_LEXER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "tree_sitter/parser.h"
TSLexer ts_lexer_make();
#ifdef __cplusplus
}
#endif
#endif // RUNTIME_LEXER_H_

52
src/runtime/parser.c
View file

@ -2,14 +2,17 @@
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser.h"
#include "runtime/tree.h"
#include "runtime/lexer.h"
#include "runtime/stack.h"
#include "runtime/parser.h"
/*
* Private
*/
static const TSParseAction *actions_for_state(TSParserConfig config,
static const TSParseAction *actions_for_state(const TSLanguage *language,
TSStateId state) {
return config.parse_table + (state * config.symbol_count);
return language->parse_table + (state * language->symbol_count);
}
static size_t breakdown_stack(TSParser *parser, TSInputEdit *edit) {
@ -37,7 +40,7 @@ static size_t breakdown_stack(TSParser *parser, TSInputEdit *edit) {
TSTree *child = children[i];
TSStateId state = ts_stack_top_state(stack);
TSStateId next_state =
actions_for_state(parser->config, state)[child->symbol].data.to_state;
actions_for_state(parser->language, state)[child->symbol].data.to_state;
ts_stack_push(stack, next_state, child);
ts_tree_retain(child);
position += ts_tree_total_size(child);
@ -52,14 +55,14 @@ static size_t breakdown_stack(TSParser *parser, TSInputEdit *edit) {
static TSSymbol *expected_symbols(TSParser *parser, size_t *count) {
*count = 0;
const TSParseAction *actions =
actions_for_state(parser->config, ts_stack_top_state(&parser->stack));
for (size_t i = 0; i < parser->config.symbol_count; i++)
actions_for_state(parser->language, ts_stack_top_state(&parser->stack));
for (size_t i = 0; i < parser->language->symbol_count; i++)
if (actions[i].type != TSParseActionTypeError)
(*count)++;
size_t n = 0;
TSSymbol *result = malloc(*count * sizeof(*result));
for (TSSymbol i = 0; i < parser->config.symbol_count; i++)
for (TSSymbol i = 0; i < parser->language->symbol_count; i++)
if (actions[i].type != TSParseActionTypeError)
result[n++] = i;
@ -70,22 +73,19 @@ static TSSymbol *expected_symbols(TSParser *parser, size_t *count) {
* Public
*/
TSParser *ts_parser_make(TSParserConfig config) {
TSParser *result = malloc(sizeof(*result));
*result = (TSParser) { .lexer = ts_lexer_make(),
.stack = ts_stack_make(),
.debug = 0,
.config = config, };
return result;
TSParser ts_parser_make(const TSLanguage *language) {
return (TSParser) { .lexer = ts_lexer_make(),
.stack = ts_stack_make(),
.debug = 0,
.language = language, };
}
void ts_parser_free(TSParser *parser) {
void ts_parser_destroy(TSParser *parser) {
if (parser->lookahead)
ts_tree_release(parser->lookahead);
if (parser->next_lookahead)
ts_tree_release(parser->next_lookahead);
ts_stack_delete(&parser->stack);
free(parser);
}
void ts_parser_start(TSParser *parser, TSInput input, TSInputEdit *edit) {
@ -118,7 +118,7 @@ void ts_parser_shift_extra(TSParser *parser) {
void ts_parser_reduce(TSParser *parser, TSSymbol symbol, size_t child_count) {
parser->next_lookahead = parser->lookahead;
parser->lookahead = ts_stack_reduce(&parser->stack, symbol, child_count,
parser->config.hidden_symbol_flags, 1);
parser->language->hidden_symbol_flags, 1);
}
int ts_parser_reduce_extra(TSParser *parser, TSSymbol symbol) {
@ -141,7 +141,7 @@ int ts_parser_handle_error(TSParser *parser) {
for (;;) {
ts_tree_release(parser->lookahead);
size_t position = ts_lexer_position(&parser->lexer);
parser->lookahead = parser->config.lex_fn(parser, ts_lex_state_error);
parser->lookahead = parser->language->lex_fn(&parser->lexer, ts_lex_state_error);
int at_end = 0;
if (ts_lexer_position(&parser->lexer) == position)
@ -160,10 +160,10 @@ int ts_parser_handle_error(TSParser *parser) {
size_t i = parser->stack.size - 1 - j;
TSStateId stack_state = parser->stack.entries[i].state;
TSParseAction action_on_error =
actions_for_state(parser->config, stack_state)[ts_builtin_sym_error];
actions_for_state(parser->language, stack_state)[ts_builtin_sym_error];
if (action_on_error.type == TSParseActionTypeShift) {
TSStateId state_after_error = action_on_error.data.to_state;
if (actions_for_state(parser->config,
if (actions_for_state(parser->language,
state_after_error)[parser->lookahead->symbol]
.type != TSParseActionTypeError) {
ts_stack_shrink(&parser->stack, i + 1);
@ -180,13 +180,13 @@ TSTree *ts_parser_tree_root(TSParser *parser) {
size_t node_count = 0;
for (size_t i = 0; i < stack->size; i++) {
TSTree *node = stack->entries[i].node;
if (!parser->config.hidden_symbol_flags[node->symbol])
if (!parser->language->hidden_symbol_flags[node->symbol])
node_count++;
}
if (node_count > 1)
return ts_stack_reduce(stack, 2, stack->size,
parser->config.hidden_symbol_flags, 0);
parser->language->hidden_symbol_flags, 0);
else
return ts_stack_top_node(stack);
}
@ -195,8 +195,8 @@ TSParseAction ts_parser_next_action(TSParser *parser) {
TSStateId state = ts_stack_top_state(&parser->stack);
if (!parser->lookahead)
parser->lookahead =
parser->config.lex_fn(parser, parser->config.lex_states[state]);
return actions_for_state(parser->config, state)[parser->lookahead->symbol];
parser->language->lex_fn(&parser->lexer, parser->language->lex_states[state]);
return actions_for_state(parser->language, state)[parser->lookahead->symbol];
}
#define DEBUG_PARSE(...) \
@ -207,7 +207,7 @@ TSParseAction ts_parser_next_action(TSParser *parser) {
TSTree *ts_parser_step(TSParser *parser) {
TSParseAction action = ts_parser_next_action(parser);
DEBUG_PARSE("LOOKAHEAD %s",
parser->config.symbol_names[parser->lookahead->symbol]);
parser->language->symbol_names[parser->lookahead->symbol]);
switch (action.type) {
case TSParseActionTypeShift:
DEBUG_PARSE("SHIFT %d", action.data.to_state);
@ -219,7 +219,7 @@ TSTree *ts_parser_step(TSParser *parser) {
return NULL;
case TSParseActionTypeReduce:
DEBUG_PARSE("REDUCE %s %d",
parser->config.symbol_names[action.data.symbol],
parser->language->symbol_names[action.data.symbol],
action.data.child_count);
ts_parser_reduce(parser, action.data.symbol, action.data.child_count);
return NULL;
@ -255,5 +255,3 @@ const TSTree *ts_parser_parse(TSParser *parser, TSInput input,
return tree;
}
}
TSParserConfig ts_parser_config(TSParser *parser) { return parser->config; }

30
src/runtime/parser.h Normal file
View file

@ -0,0 +1,30 @@
#ifndef RUNTIME_PARSER_H_
#define RUNTIME_PARSER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "runtime/stack.h"
typedef struct {
TSLexer lexer;
TSStack stack;
int debug;
TSTree *lookahead;
TSTree *next_lookahead;
const TSLanguage *language;
} TSParser;
TSParser ts_parser_make(const TSLanguage *);
void ts_parser_destroy(TSParser *);
const TSTree *ts_parser_parse(TSParser *parser, TSInput input,
TSInputEdit *edit);
void ts_parser_start(TSParser *parser, TSInput input, TSInputEdit *edit);
TSTree *ts_parser_step(TSParser *parser);
#ifdef __cplusplus
}
#endif
#endif // RUNTIME_PARSER_H_

3
src/runtime/stack.c
View file

@ -1,7 +1,6 @@
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser.h"
#include "runtime/tree.h"
#include <string.h>
#include "runtime/stack.h"
static size_t INITIAL_STACK_SIZE = 100;
static TSStateId INITIAL_STATE = 0;

34
src/runtime/stack.h Normal file
View file

@ -0,0 +1,34 @@
#ifndef RUNTIME_STACK_H_
#define RUNTIME_STACK_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "tree_sitter/parser.h"
typedef struct {
size_t size;
struct {
TSTree *node;
TSStateId state;
int is_extra;
} *entries;
} TSStack;
TSStack ts_stack_make();
void ts_stack_delete(TSStack *);
TSTree *ts_stack_reduce(TSStack *stack, TSSymbol symbol,
size_t immediate_child_count,
const int *hidden_symbol_flags, int gather_extras);
void ts_stack_shrink(TSStack *stack, size_t new_size);
void ts_stack_push(TSStack *stack, TSStateId state, TSTree *node);
TSStateId ts_stack_top_state(const TSStack *stack);
TSTree *ts_stack_top_node(const TSStack *stack);
size_t ts_stack_right_position(const TSStack *stack);
#ifdef __cplusplus
}
#endif
#endif // RUNTIME_STACK_H_