Combine TSParser and TSStateMachine objects
My original thought was to decouple the runtime from the LR parser generator by making TSParser a generic interface that LR parsers implement. I think this was more trouble than it was worth.
This commit is contained in:
Max Brunsfeld
parent
1c7d2d2d03
commit
9da7663e99
18 changed files with 586 additions and 645 deletions
|
|
@ -5,18 +5,109 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
// #define TS_DEBUG_LEX
|
||||
|
||||
#include <stdio.h>
|
||||
#include "tree_sitter/runtime.h"
|
||||
#include "tree_sitter/parser/lexer.h"
|
||||
#include "tree_sitter/parser/stack.h"
|
||||
#include "tree_sitter/parser/state_machine.h"
|
||||
|
||||
#define ts_lex_state_error 0
|
||||
|
||||
typedef struct {
|
||||
TSInput input;
|
||||
int debug;
|
||||
const char *chunk;
|
||||
size_t chunk_start;
|
||||
size_t chunk_size;
|
||||
size_t position_in_chunk;
|
||||
size_t token_end_position;
|
||||
size_t token_start_position;
|
||||
int reached_end;
|
||||
} TSLexer;
|
||||
|
||||
TSLexer ts_lexer_make();
|
||||
int ts_lexer_advance(TSLexer *lexer);
|
||||
TSTree * ts_lexer_build_node(TSLexer *lexer, TSSymbol symbol);
|
||||
|
||||
static inline size_t ts_lexer_position(const TSLexer *lexer) {
|
||||
return lexer->chunk_start + lexer->position_in_chunk;
|
||||
}
|
||||
|
||||
static inline char ts_lexer_lookahead_char(const TSLexer *lexer) {
|
||||
return lexer->chunk[lexer->position_in_chunk];
|
||||
}
|
||||
|
||||
static inline void ts_lexer_start_token(TSLexer *lexer) {
|
||||
lexer->token_start_position = ts_lexer_position(lexer);
|
||||
}
|
||||
|
||||
typedef unsigned short TSStateId;
|
||||
|
||||
typedef struct {
|
||||
size_t size;
|
||||
struct {
|
||||
TSTree *node;
|
||||
TSStateId state;
|
||||
int is_extra;
|
||||
} *entries;
|
||||
} TSStack;
|
||||
|
||||
TSStack ts_stack_make();
|
||||
void ts_stack_delete(TSStack *);
|
||||
TSTree * ts_stack_reduce(TSStack *stack, TSSymbol symbol, size_t immediate_child_count, const int *hidden_symbol_flags, int gather_extras);
|
||||
void ts_stack_shrink(TSStack *stack, size_t new_size);
|
||||
void ts_stack_push(TSStack *stack, TSStateId state, TSTree *node);
|
||||
TSStateId ts_stack_top_state(const TSStack *stack);
|
||||
TSTree * ts_stack_top_node(const TSStack *stack);
|
||||
size_t ts_stack_right_position(const TSStack *stack);
|
||||
|
||||
typedef enum {
|
||||
TSParseActionTypeError,
|
||||
TSParseActionTypeShift,
|
||||
TSParseActionTypeShiftExtra,
|
||||
TSParseActionTypeReduce,
|
||||
TSParseActionTypeReduceExtra,
|
||||
TSParseActionTypeAccept,
|
||||
} TSParseActionType;
|
||||
|
||||
typedef struct {
|
||||
TSParseActionType type;
|
||||
union {
|
||||
TSStateId to_state;
|
||||
struct {
|
||||
TSSymbol symbol;
|
||||
unsigned short child_count;
|
||||
};
|
||||
} data;
|
||||
} TSParseAction;
|
||||
|
||||
typedef struct {
|
||||
size_t symbol_count;
|
||||
const char **symbol_names;
|
||||
const int *hidden_symbol_flags;
|
||||
const TSParseAction *parse_table;
|
||||
const TSStateId *lex_states;
|
||||
TSTree * (* lex_fn)(TSParser *, TSStateId);
|
||||
} TSParserConfig;
|
||||
|
||||
struct TSParser {
|
||||
TSLexer lexer;
|
||||
TSStack stack;
|
||||
int debug;
|
||||
TSTree *lookahead;
|
||||
TSTree *next_lookahead;
|
||||
TSParserConfig config;
|
||||
};
|
||||
|
||||
TSParser * ts_parser_make(TSParserConfig);
|
||||
void ts_parser_free(TSParser *);
|
||||
TSParserConfig ts_parser_config(TSParser *);
|
||||
const TSTree * ts_parser_parse(TSParser *parser, TSInput input, TSInputEdit *edit);
|
||||
void ts_parser_start(TSParser *parser, TSInput input, TSInputEdit *edit);
|
||||
TSTree * ts_parser_step(TSParser *parser);
|
||||
|
||||
#define SYMBOL_NAMES \
|
||||
static const char *ts_symbol_names[]
|
||||
|
||||
#define HIDDEN_SYMBOLS \
|
||||
static const int hidden_symbol_flags[SYMBOL_COUNT]
|
||||
static const int ts_hidden_symbol_flags[SYMBOL_COUNT]
|
||||
|
||||
#define LEX_STATES \
|
||||
static TSStateId ts_lex_states[STATE_COUNT]
|
||||
|
|
@ -25,42 +116,38 @@ static TSStateId ts_lex_states[STATE_COUNT]
|
|||
static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT]
|
||||
|
||||
#define LEX_FN() \
|
||||
static TSTree * ts_lex(TSLexer *lexer, TSStateId lex_state)
|
||||
static TSTree * ts_lex(TSParser *parser, TSStateId lex_state)
|
||||
|
||||
#ifdef TS_DEBUG_LEX
|
||||
#include <stdio.h>
|
||||
#define DEBUG_LEX(...) fprintf(stderr, "\n" __VA_ARGS__)
|
||||
#else
|
||||
#define DEBUG_LEX(...)
|
||||
#endif
|
||||
#define DEBUG_LEX(...) \
|
||||
if (parser->lexer.debug) { fprintf(stderr, "\n" __VA_ARGS__); }
|
||||
|
||||
#define START_LEXER() \
|
||||
DEBUG_LEX("LEX %d", lex_state); \
|
||||
char lookahead; \
|
||||
next_state: \
|
||||
lookahead = ts_lexer_lookahead_char(lexer); \
|
||||
lookahead = ts_lexer_lookahead_char(&parser->lexer); \
|
||||
DEBUG_LEX("CHAR '%c'", lookahead);
|
||||
|
||||
#define START_TOKEN() \
|
||||
ts_lexer_start_token(lexer);
|
||||
ts_lexer_start_token(&parser->lexer);
|
||||
|
||||
#define ADVANCE(state_index) \
|
||||
{ \
|
||||
DEBUG_LEX("ADVANCE %d", state_index); \
|
||||
if (!ts_lexer_advance(lexer)) ACCEPT_TOKEN(ts_builtin_sym_end); \
|
||||
if (!ts_lexer_advance(&parser->lexer)) ACCEPT_TOKEN(ts_builtin_sym_end); \
|
||||
lex_state = state_index; goto next_state; \
|
||||
}
|
||||
|
||||
#define ACCEPT_TOKEN(symbol) \
|
||||
{ \
|
||||
DEBUG_LEX("TOKEN %s", ts_symbol_names[symbol]); \
|
||||
return ts_lexer_build_node(lexer, symbol); \
|
||||
return ts_lexer_build_node(&parser->lexer, symbol); \
|
||||
}
|
||||
|
||||
#define LEX_ERROR() \
|
||||
{ \
|
||||
DEBUG_LEX("ERROR"); \
|
||||
return ts_lexer_build_node(lexer, ts_builtin_sym_error); \
|
||||
return ts_lexer_build_node(&parser->lexer, ts_builtin_sym_error); \
|
||||
}
|
||||
|
||||
#define LEX_PANIC() \
|
||||
|
|
@ -69,31 +156,32 @@ ts_lexer_start_token(lexer);
|
|||
return NULL; \
|
||||
}
|
||||
|
||||
SYMBOL_NAMES;
|
||||
#define SHIFT(to_state_value) \
|
||||
{ .type = TSParseActionTypeShift, .data = { .to_state = to_state_value } }
|
||||
|
||||
#define SHIFT_EXTRA() \
|
||||
{ .type = TSParseActionTypeShiftExtra }
|
||||
|
||||
#define REDUCE_EXTRA(symbol_val) \
|
||||
{ .type = TSParseActionTypeReduceExtra, .data = { .symbol = symbol_val } }
|
||||
|
||||
#define REDUCE(symbol_val, child_count_val) \
|
||||
{ .type = TSParseActionTypeReduce, .data = { .symbol = symbol_val, .child_count = child_count_val } }
|
||||
|
||||
#define ACCEPT_INPUT() \
|
||||
{ .type = TSParseActionTypeAccept }
|
||||
|
||||
static const TSTree * ts_parse(void *data, TSInput input, TSInputEdit *edit) {
|
||||
TSStateMachine *parser = (TSStateMachine *)data;
|
||||
ts_state_machine_initialize(parser, input, edit);
|
||||
for (;;) {
|
||||
const TSTree *tree = ts_state_machine_parse(parser, ts_symbol_names);
|
||||
if (tree) return tree;
|
||||
}
|
||||
}
|
||||
|
||||
#define EXPORT_PARSER(constructor_name) \
|
||||
TSParser constructor_name() { \
|
||||
return (TSParser) { \
|
||||
.parse_fn = ts_parse, \
|
||||
.free_fn = ts_state_machine_free, \
|
||||
.symbol_names = ts_symbol_names, \
|
||||
.data = ts_state_machine_make( \
|
||||
SYMBOL_COUNT, \
|
||||
(const TSParseAction *)ts_parse_actions, \
|
||||
ts_lex_states, \
|
||||
ts_lex, \
|
||||
hidden_symbol_flags \
|
||||
), \
|
||||
}; \
|
||||
TSParser * constructor_name() { \
|
||||
return ts_parser_make((TSParserConfig) { \
|
||||
.symbol_count = SYMBOL_COUNT, \
|
||||
.hidden_symbol_flags = ts_hidden_symbol_flags, \
|
||||
.parse_table = (const TSParseAction *)ts_parse_actions, \
|
||||
.lex_states = ts_lex_states, \
|
||||
.symbol_names = ts_symbol_names, \
|
||||
.lex_fn = ts_lex, \
|
||||
}); \
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
|
|
|
|||
|
|
@ -1,70 +0,0 @@
|
|||
#ifndef TREE_SITTER_PARSER_LEXER_H_
|
||||
#define TREE_SITTER_PARSER_LEXER_H_
|
||||
|
||||
#include "tree_sitter/runtime.h"
|
||||
|
||||
#define ts_lex_state_error 0
|
||||
|
||||
typedef struct {
|
||||
TSInput input;
|
||||
const char *chunk;
|
||||
size_t chunk_start;
|
||||
size_t chunk_size;
|
||||
size_t position_in_chunk;
|
||||
size_t token_end_position;
|
||||
size_t token_start_position;
|
||||
int reached_end;
|
||||
} TSLexer;
|
||||
|
||||
static inline TSLexer ts_lexer_make() {
|
||||
TSLexer result;
|
||||
result.chunk = NULL;
|
||||
result.chunk_start = 0;
|
||||
result.chunk_size = 0;
|
||||
result.position_in_chunk = 0;
|
||||
result.token_start_position = 0;
|
||||
result.token_end_position = 0;
|
||||
result.reached_end = 0;
|
||||
return result;
|
||||
}
|
||||
|
||||
static inline size_t ts_lexer_position(const TSLexer *lexer) {
|
||||
return lexer->chunk_start + lexer->position_in_chunk;
|
||||
}
|
||||
|
||||
static inline char ts_lexer_lookahead_char(const TSLexer *lexer) {
|
||||
return lexer->chunk[lexer->position_in_chunk];
|
||||
}
|
||||
|
||||
static inline int ts_lexer_advance(TSLexer *lexer) {
|
||||
static const char *empty_chunk = "";
|
||||
if (lexer->position_in_chunk + 1 < lexer->chunk_size) {
|
||||
lexer->position_in_chunk++;
|
||||
} else {
|
||||
lexer->chunk_start += lexer->chunk_size;
|
||||
lexer->chunk = lexer->input.read_fn(lexer->input.data, &lexer->chunk_size);
|
||||
lexer->position_in_chunk = 0;
|
||||
if (lexer->reached_end) {
|
||||
return 0;
|
||||
}
|
||||
if (lexer->chunk_size == 0) {
|
||||
lexer->reached_end = 1;
|
||||
lexer->chunk = empty_chunk;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static inline void ts_lexer_start_token(TSLexer *lexer) {
|
||||
lexer->token_start_position = ts_lexer_position(lexer);
|
||||
}
|
||||
|
||||
static inline TSTree * ts_lexer_build_node(TSLexer *lexer, TSSymbol symbol) {
|
||||
size_t current_position = ts_lexer_position(lexer);
|
||||
size_t size = current_position - lexer->token_start_position;
|
||||
size_t offset = lexer->token_start_position - lexer->token_end_position;
|
||||
lexer->token_end_position = current_position;
|
||||
return ts_tree_make_leaf(symbol, size, offset);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
#ifndef TREE_SITTER_PARSER_STACK_H_
|
||||
#define TREE_SITTER_PARSER_STACK_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "tree_sitter/runtime.h"
|
||||
|
||||
typedef unsigned short TSStateId;
|
||||
|
||||
typedef struct {
|
||||
size_t size;
|
||||
struct {
|
||||
TSTree *node;
|
||||
TSStateId state;
|
||||
int is_extra;
|
||||
} *entries;
|
||||
} TSStack;
|
||||
|
||||
TSStack ts_stack_make();
|
||||
void ts_stack_delete(TSStack *);
|
||||
TSTree * ts_stack_reduce(TSStack *stack, TSSymbol symbol, size_t immediate_child_count, const int *hidden_symbol_flags, int gather_extras);
|
||||
void ts_stack_shrink(TSStack *stack, size_t new_size);
|
||||
void ts_stack_push(TSStack *stack, TSStateId state, TSTree *node);
|
||||
TSStateId ts_stack_top_state(const TSStack *stack);
|
||||
TSTree * ts_stack_top_node(const TSStack *stack);
|
||||
size_t ts_stack_right_position(const TSStack *stack);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
#ifndef TREE_SITTER_PARSER_LR_PARSER_H_
|
||||
#define TREE_SITTER_PARSER_LR_PARSER_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "tree_sitter/parser/stack.h"
|
||||
#include "tree_sitter/parser/lexer.h"
|
||||
|
||||
typedef enum {
|
||||
TSParseActionTypeError,
|
||||
TSParseActionTypeShift,
|
||||
TSParseActionTypeShiftExtra,
|
||||
TSParseActionTypeReduce,
|
||||
TSParseActionTypeReduceExtra,
|
||||
TSParseActionTypeAccept,
|
||||
} TSParseActionType;
|
||||
|
||||
typedef struct {
|
||||
TSParseActionType type;
|
||||
union {
|
||||
TSStateId to_state;
|
||||
struct {
|
||||
TSSymbol symbol;
|
||||
unsigned short child_count;
|
||||
};
|
||||
} data;
|
||||
} TSParseAction;
|
||||
|
||||
#define SHIFT(to_state_value) \
|
||||
{ .type = TSParseActionTypeShift, .data = { .to_state = to_state_value } }
|
||||
|
||||
#define SHIFT_EXTRA() \
|
||||
{ .type = TSParseActionTypeShiftExtra }
|
||||
|
||||
#define REDUCE_EXTRA(symbol_val) \
|
||||
{ .type = TSParseActionTypeReduceExtra, .data = { .symbol = symbol_val } }
|
||||
|
||||
#define REDUCE(symbol_val, child_count_val) \
|
||||
{ .type = TSParseActionTypeReduce, .data = { .symbol = symbol_val, .child_count = child_count_val } }
|
||||
|
||||
#define ACCEPT_INPUT() \
|
||||
{ .type = TSParseActionTypeAccept }
|
||||
|
||||
typedef struct {
|
||||
TSLexer lexer;
|
||||
TSStack stack;
|
||||
TSTree *lookahead;
|
||||
TSTree *next_lookahead;
|
||||
struct {
|
||||
size_t symbol_count;
|
||||
const int *hidden_symbol_flags;
|
||||
const TSParseAction *parse_table;
|
||||
const TSStateId *lex_states;
|
||||
TSTree * (* lex_fn)(TSLexer *, TSStateId);
|
||||
} config;
|
||||
} TSStateMachine;
|
||||
|
||||
TSStateMachine * ts_state_machine_make(
|
||||
size_t symbol_count,
|
||||
const TSParseAction *parse_table,
|
||||
const TSStateId *lex_states,
|
||||
TSTree * (* lex_fn)(TSLexer *, TSStateId),
|
||||
const int *hidden_symbol_flags);
|
||||
void ts_state_machine_free(void *data);
|
||||
void ts_state_machine_initialize(TSStateMachine *, TSInput, TSInputEdit *);
|
||||
TSTree * ts_state_machine_parse(TSStateMachine *, const char **symbol_names);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
@ -41,17 +41,12 @@ typedef struct {
|
|||
size_t bytes_removed;
|
||||
} TSInputEdit;
|
||||
|
||||
typedef struct {
|
||||
const TSTree * (* parse_fn)(void *data, TSInput input, TSInputEdit *edit);
|
||||
void (* free_fn)(void *data);
|
||||
const char **symbol_names;
|
||||
void *data;
|
||||
} TSParser;
|
||||
typedef struct TSParser TSParser;
|
||||
|
||||
typedef struct TSDocument TSDocument;
|
||||
TSDocument * ts_document_make();
|
||||
void ts_document_free(TSDocument *doc);
|
||||
void ts_document_set_parser(TSDocument *doc, TSParser parser);
|
||||
void ts_document_set_parser(TSDocument *doc, TSParser *parser);
|
||||
void ts_document_set_input(TSDocument *doc, TSInput input);
|
||||
void ts_document_set_input_string(TSDocument *doc, const char *text);
|
||||
void ts_document_edit(TSDocument *doc, TSInputEdit edit);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue