traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Combine TSParser and TSStateMachine objects

Browse source 
My original thought was to decouple the runtime from
the LR parser generator by making TSParser a generic
interface that LR parsers implement.

I think this was more trouble than it was worth.
This commit is contained in:
Max Brunsfeld 2014-07-10 13:14:52 -07:00
parent 1c7d2d2d03
commit 9da7663e99
18 changed files with 586 additions and 645 deletions
Download patch file Download diff file Expand all files Collapse all files

168
include/tree_sitter/parser.h
View file

@ -5,18 +5,109 @@
extern "C" {
#endif
// #define TS_DEBUG_LEX
#include <stdio.h>
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser/lexer.h"
#include "tree_sitter/parser/stack.h"
#include "tree_sitter/parser/state_machine.h"
#define ts_lex_state_error 0
typedef struct {
TSInput input;
int debug;
const char *chunk;
size_t chunk_start;
size_t chunk_size;
size_t position_in_chunk;
size_t token_end_position;
size_t token_start_position;
int reached_end;
} TSLexer;
TSLexer ts_lexer_make();
int ts_lexer_advance(TSLexer *lexer);
TSTree * ts_lexer_build_node(TSLexer *lexer, TSSymbol symbol);
static inline size_t ts_lexer_position(const TSLexer *lexer) {
return lexer->chunk_start + lexer->position_in_chunk;
}
static inline char ts_lexer_lookahead_char(const TSLexer *lexer) {
return lexer->chunk[lexer->position_in_chunk];
}
static inline void ts_lexer_start_token(TSLexer *lexer) {
lexer->token_start_position = ts_lexer_position(lexer);
}
typedef unsigned short TSStateId;
typedef struct {
size_t size;
struct {
TSTree *node;
TSStateId state;
int is_extra;
} *entries;
} TSStack;
TSStack ts_stack_make();
void ts_stack_delete(TSStack *);
TSTree * ts_stack_reduce(TSStack *stack, TSSymbol symbol, size_t immediate_child_count, const int *hidden_symbol_flags, int gather_extras);
void ts_stack_shrink(TSStack *stack, size_t new_size);
void ts_stack_push(TSStack *stack, TSStateId state, TSTree *node);
TSStateId ts_stack_top_state(const TSStack *stack);
TSTree * ts_stack_top_node(const TSStack *stack);
size_t ts_stack_right_position(const TSStack *stack);
typedef enum {
TSParseActionTypeError,
TSParseActionTypeShift,
TSParseActionTypeShiftExtra,
TSParseActionTypeReduce,
TSParseActionTypeReduceExtra,
TSParseActionTypeAccept,
} TSParseActionType;
typedef struct {
TSParseActionType type;
union {
TSStateId to_state;
struct {
TSSymbol symbol;
unsigned short child_count;
};
} data;
} TSParseAction;
typedef struct {
size_t symbol_count;
const char **symbol_names;
const int *hidden_symbol_flags;
const TSParseAction *parse_table;
const TSStateId *lex_states;
TSTree * (* lex_fn)(TSParser *, TSStateId);
} TSParserConfig;
struct TSParser {
TSLexer lexer;
TSStack stack;
int debug;
TSTree *lookahead;
TSTree *next_lookahead;
TSParserConfig config;
};
TSParser * ts_parser_make(TSParserConfig);
void ts_parser_free(TSParser *);
TSParserConfig ts_parser_config(TSParser *);
const TSTree * ts_parser_parse(TSParser *parser, TSInput input, TSInputEdit *edit);
void ts_parser_start(TSParser *parser, TSInput input, TSInputEdit *edit);
TSTree * ts_parser_step(TSParser *parser);
#define SYMBOL_NAMES \
static const char *ts_symbol_names[]
#define HIDDEN_SYMBOLS \
static const int hidden_symbol_flags[SYMBOL_COUNT]
static const int ts_hidden_symbol_flags[SYMBOL_COUNT]
#define LEX_STATES \
static TSStateId ts_lex_states[STATE_COUNT]
@ -25,42 +116,38 @@ static TSStateId ts_lex_states[STATE_COUNT]
static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT]
#define LEX_FN() \
static TSTree * ts_lex(TSLexer *lexer, TSStateId lex_state)
static TSTree * ts_lex(TSParser *parser, TSStateId lex_state)
#ifdef TS_DEBUG_LEX
#include <stdio.h>
#define DEBUG_LEX(...) fprintf(stderr, "\n" __VA_ARGS__)
#else
#define DEBUG_LEX(...)
#endif
#define DEBUG_LEX(...) \
if (parser->lexer.debug) { fprintf(stderr, "\n" __VA_ARGS__); }
#define START_LEXER() \
DEBUG_LEX("LEX %d", lex_state); \
char lookahead; \
next_state: \
lookahead = ts_lexer_lookahead_char(lexer); \
lookahead = ts_lexer_lookahead_char(&parser->lexer); \
DEBUG_LEX("CHAR '%c'", lookahead);
#define START_TOKEN() \
ts_lexer_start_token(lexer);
ts_lexer_start_token(&parser->lexer);
#define ADVANCE(state_index) \
{ \
DEBUG_LEX("ADVANCE %d", state_index); \
if (!ts_lexer_advance(lexer)) ACCEPT_TOKEN(ts_builtin_sym_end); \
if (!ts_lexer_advance(&parser->lexer)) ACCEPT_TOKEN(ts_builtin_sym_end); \
lex_state = state_index; goto next_state; \
}
#define ACCEPT_TOKEN(symbol) \
{ \
DEBUG_LEX("TOKEN %s", ts_symbol_names[symbol]); \
return ts_lexer_build_node(lexer, symbol); \
return ts_lexer_build_node(&parser->lexer, symbol); \
}
#define LEX_ERROR() \
{ \
DEBUG_LEX("ERROR"); \
return ts_lexer_build_node(lexer, ts_builtin_sym_error); \
return ts_lexer_build_node(&parser->lexer, ts_builtin_sym_error); \
}
#define LEX_PANIC() \
@ -69,31 +156,32 @@ ts_lexer_start_token(lexer);
return NULL; \
}
SYMBOL_NAMES;
#define SHIFT(to_state_value) \
{ .type = TSParseActionTypeShift, .data = { .to_state = to_state_value } }
#define SHIFT_EXTRA() \
{ .type = TSParseActionTypeShiftExtra }
#define REDUCE_EXTRA(symbol_val) \
{ .type = TSParseActionTypeReduceExtra, .data = { .symbol = symbol_val } }
#define REDUCE(symbol_val, child_count_val) \
{ .type = TSParseActionTypeReduce, .data = { .symbol = symbol_val, .child_count = child_count_val } }
#define ACCEPT_INPUT() \
{ .type = TSParseActionTypeAccept }
static const TSTree * ts_parse(void *data, TSInput input, TSInputEdit *edit) {
TSStateMachine *parser = (TSStateMachine *)data;
ts_state_machine_initialize(parser, input, edit);
for (;;) {
const TSTree *tree = ts_state_machine_parse(parser, ts_symbol_names);
if (tree) return tree;
}
}
#define EXPORT_PARSER(constructor_name) \
TSParser constructor_name() { \
return (TSParser) { \
.parse_fn = ts_parse, \
.free_fn = ts_state_machine_free, \
.symbol_names = ts_symbol_names, \
.data = ts_state_machine_make( \
SYMBOL_COUNT, \
(const TSParseAction *)ts_parse_actions, \
ts_lex_states, \
ts_lex, \
hidden_symbol_flags \
), \
}; \
TSParser * constructor_name() { \
return ts_parser_make((TSParserConfig) { \
.symbol_count = SYMBOL_COUNT, \
.hidden_symbol_flags = ts_hidden_symbol_flags, \
.parse_table = (const TSParseAction *)ts_parse_actions, \
.lex_states = ts_lex_states, \
.symbol_names = ts_symbol_names, \
.lex_fn = ts_lex, \
}); \
}
#ifdef __cplusplus

70
include/tree_sitter/parser/lexer.h
View file

@ -1,70 +0,0 @@
#ifndef TREE_SITTER_PARSER_LEXER_H_
#define TREE_SITTER_PARSER_LEXER_H_
#include "tree_sitter/runtime.h"
#define ts_lex_state_error 0
typedef struct {
TSInput input;
const char *chunk;
size_t chunk_start;
size_t chunk_size;
size_t position_in_chunk;
size_t token_end_position;
size_t token_start_position;
int reached_end;
} TSLexer;
static inline TSLexer ts_lexer_make() {
TSLexer result;
result.chunk = NULL;
result.chunk_start = 0;
result.chunk_size = 0;
result.position_in_chunk = 0;
result.token_start_position = 0;
result.token_end_position = 0;
result.reached_end = 0;
return result;
}
static inline size_t ts_lexer_position(const TSLexer *lexer) {
return lexer->chunk_start + lexer->position_in_chunk;
}
static inline char ts_lexer_lookahead_char(const TSLexer *lexer) {
return lexer->chunk[lexer->position_in_chunk];
}
static inline int ts_lexer_advance(TSLexer *lexer) {
static const char *empty_chunk = "";
if (lexer->position_in_chunk + 1 < lexer->chunk_size) {
lexer->position_in_chunk++;
} else {
lexer->chunk_start += lexer->chunk_size;
lexer->chunk = lexer->input.read_fn(lexer->input.data, &lexer->chunk_size);
lexer->position_in_chunk = 0;
if (lexer->reached_end) {
return 0;
}
if (lexer->chunk_size == 0) {
lexer->reached_end = 1;
lexer->chunk = empty_chunk;
}
}
return 1;
}
static inline void ts_lexer_start_token(TSLexer *lexer) {
lexer->token_start_position = ts_lexer_position(lexer);
}
static inline TSTree * ts_lexer_build_node(TSLexer *lexer, TSSymbol symbol) {
size_t current_position = ts_lexer_position(lexer);
size_t size = current_position - lexer->token_start_position;
size_t offset = lexer->token_start_position - lexer->token_end_position;
lexer->token_end_position = current_position;
return ts_tree_make_leaf(symbol, size, offset);
}
#endif

34
include/tree_sitter/parser/stack.h
View file

@ -1,34 +0,0 @@
#ifndef TREE_SITTER_PARSER_STACK_H_
#define TREE_SITTER_PARSER_STACK_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "tree_sitter/runtime.h"
typedef unsigned short TSStateId;
typedef struct {
size_t size;
struct {
TSTree *node;
TSStateId state;
int is_extra;
} *entries;
} TSStack;
TSStack ts_stack_make();
void ts_stack_delete(TSStack *);
TSTree * ts_stack_reduce(TSStack *stack, TSSymbol symbol, size_t immediate_child_count, const int *hidden_symbol_flags, int gather_extras);
void ts_stack_shrink(TSStack *stack, size_t new_size);
void ts_stack_push(TSStack *stack, TSStateId state, TSTree *node);
TSStateId ts_stack_top_state(const TSStack *stack);
TSTree * ts_stack_top_node(const TSStack *stack);
size_t ts_stack_right_position(const TSStack *stack);
#ifdef __cplusplus
}
#endif
#endif

74
include/tree_sitter/parser/state_machine.h
View file

@ -1,74 +0,0 @@
#ifndef TREE_SITTER_PARSER_LR_PARSER_H_
#define TREE_SITTER_PARSER_LR_PARSER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "tree_sitter/parser/stack.h"
#include "tree_sitter/parser/lexer.h"
typedef enum {
TSParseActionTypeError,
TSParseActionTypeShift,
TSParseActionTypeShiftExtra,
TSParseActionTypeReduce,
TSParseActionTypeReduceExtra,
TSParseActionTypeAccept,
} TSParseActionType;
typedef struct {
TSParseActionType type;
union {
TSStateId to_state;
struct {
TSSymbol symbol;
unsigned short child_count;
};
} data;
} TSParseAction;
#define SHIFT(to_state_value) \
{ .type = TSParseActionTypeShift, .data = { .to_state = to_state_value } }
#define SHIFT_EXTRA() \
{ .type = TSParseActionTypeShiftExtra }
#define REDUCE_EXTRA(symbol_val) \
{ .type = TSParseActionTypeReduceExtra, .data = { .symbol = symbol_val } }
#define REDUCE(symbol_val, child_count_val) \
{ .type = TSParseActionTypeReduce, .data = { .symbol = symbol_val, .child_count = child_count_val } }
#define ACCEPT_INPUT() \
{ .type = TSParseActionTypeAccept }
typedef struct {
TSLexer lexer;
TSStack stack;
TSTree *lookahead;
TSTree *next_lookahead;
struct {
size_t symbol_count;
const int *hidden_symbol_flags;
const TSParseAction *parse_table;
const TSStateId *lex_states;
TSTree * (* lex_fn)(TSLexer *, TSStateId);
} config;
} TSStateMachine;
TSStateMachine * ts_state_machine_make(
size_t symbol_count,
const TSParseAction *parse_table,
const TSStateId *lex_states,
TSTree * (* lex_fn)(TSLexer *, TSStateId),
const int *hidden_symbol_flags);
void ts_state_machine_free(void *data);
void ts_state_machine_initialize(TSStateMachine *, TSInput, TSInputEdit *);
TSTree * ts_state_machine_parse(TSStateMachine *, const char **symbol_names);
#ifdef __cplusplus
}
#endif
#endif

9
include/tree_sitter/runtime.h
View file

@ -41,17 +41,12 @@ typedef struct {
size_t bytes_removed;
} TSInputEdit;
typedef struct {
const TSTree * (* parse_fn)(void *data, TSInput input, TSInputEdit *edit);
void (* free_fn)(void *data);
const char **symbol_names;
void *data;
} TSParser;
typedef struct TSParser TSParser;
typedef struct TSDocument TSDocument;
TSDocument * ts_document_make();
void ts_document_free(TSDocument *doc);
void ts_document_set_parser(TSDocument *doc, TSParser parser);
void ts_document_set_parser(TSDocument *doc, TSParser *parser);
void ts_document_set_input(TSDocument *doc, TSInput input);
void ts_document_set_input_string(TSDocument *doc, const char *text);
void ts_document_edit(TSDocument *doc, TSInputEdit edit);

78
spec/runtime/document_spec.cc Normal file
View file

@ -0,0 +1,78 @@
#include "runtime/runtime_spec_helper.h"
#include "runtime/helpers/spy_reader.h"
extern "C" TSParser * ts_parser_json();
START_TEST
describe("incremental parsing", [&]() {
TSDocument *doc;
SpyReader *reader;
before_each([&]() {
doc = ts_document_make();
ts_document_set_parser(doc, ts_parser_json());
reader = new SpyReader("{ \"key\": [1, 2] }", 5);
ts_document_set_input(doc, reader->input);
});
after_each([&]() {
ts_document_free(doc);
delete reader;
});
it("parses the input", [&]() {
AssertThat(string(ts_document_string(doc)), Equals(
"(value (object (string) (array (number) (number))))"));
});
it("reads the entire input", [&]() {
AssertThat(reader->strings_read, Equals(vector<string>({
"{ \"key\": [1, 2] }"
})));
});
describe("modifying the end of the input", [&]() {
before_each([&]() {
size_t position(string("{ \"key\": [1, 2]").length());
string inserted_text(", \"key2\": 4");
reader->content.insert(position, inserted_text);
ts_document_edit(doc, { position, 0, inserted_text.length() });
});
it("updates the parse tree", [&]() {
AssertThat(string(ts_document_string(doc)), Equals(
"(value (object (string) (array (number) (number)) (string) (number)))"));
});
it("re-reads only the changed portion of the input", [&]() {
AssertThat(reader->strings_read.size(), Equals<size_t>(2));
AssertThat(reader->strings_read[1], Equals(", \"key2\": 4 }"));
});
});
describe("modifying the beginning of the input", [&]() {
before_each([&]() {
size_t position(string("{ ").length());
string inserted_text("\"key2\": 4, ");
reader->content.insert(position, inserted_text);
ts_document_edit(doc, { position, 0, inserted_text.length() });
});
it("2 updates the parse tree", [&]() {
AssertThat(string(ts_document_string(doc)), Equals(
"(value (object (string) (number) (string) (array (number) (number))))"));
});
it_skip("re-reads only the changed portion of the input", [&]() {
AssertThat(reader->strings_read.size(), Equals<size_t>(2));
AssertThat(reader->strings_read[1], Equals("\"key2\": 4, "));
});
});
});
END_TEST

8
spec/runtime/helpers/dummy_parser.c
View file

@ -1,4 +1,5 @@
#include "runtime/helpers/dummy_parser.h"
#include "tree_sitter/parser.h"
const TSParseAction parse_table[3][5] = {
[0] = {
@ -29,10 +30,9 @@ const int hidden_symbols[5] = {
[dummy_sym3] = 1,
};
struct test_parser dummy_parser = {
.state_count = 3,
TSParserConfig dummy_parser = {
.symbol_count = 5,
.parse_table = (const TSParseAction **)parse_table,
.parse_table = (const TSParseAction *)parse_table,
.lex_states = lex_states,
.hidden_symbols = hidden_symbols,
.hidden_symbol_flags = hidden_symbols,
};

13
spec/runtime/helpers/dummy_parser.h
View file

@ -5,7 +5,8 @@
extern "C" {
#endif
#include "tree_sitter/parser/state_machine.h"
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser.h"
enum {
dummy_sym1 = 2,
@ -13,15 +14,7 @@ enum {
dummy_sym3 = 4,
};
struct test_parser {
size_t state_count;
size_t symbol_count;
const TSParseAction **parse_table;
const TSStateId *lex_states;
const int *hidden_symbols;
};
extern struct test_parser dummy_parser;
extern TSParserConfig dummy_parser;
#ifdef __cplusplus
}

10
spec/runtime/language_specs.cc
View file

@ -1,10 +1,10 @@
#include "runtime/runtime_spec_helper.h"
#include "runtime/helpers/read_test_entries.h"
extern "C" TSParser ts_parser_javascript();
extern "C" TSParser ts_parser_json();
extern "C" TSParser ts_parser_arithmetic();
extern "C" TSParser ts_parser_golang();
extern "C" TSParser * ts_parser_javascript();
extern "C" TSParser * ts_parser_json();
extern "C" TSParser * ts_parser_arithmetic();
extern "C" TSParser * ts_parser_golang();
START_TEST
@ -19,7 +19,7 @@ describe("Languages", [&]() {
ts_document_free(doc);
});
auto run_tests_for_language = [&](string language, TSParser (parser_constructor)()) {
auto run_tests_for_language = [&](string language, TSParser * (parser_constructor)()) {
describe(language.c_str(), [&]() {
before_each([&]() {
ts_document_set_parser(doc, parser_constructor());

2
spec/runtime/node_position_spec.cc
View file

@ -1,6 +1,6 @@
#include "runtime/runtime_spec_helper.h"
extern "C" TSParser ts_parser_json();
extern "C" TSParser * ts_parser_json();
START_TEST

102
spec/runtime/parser_spec.cc
View file

@ -1,78 +1,76 @@
#include "runtime/runtime_spec_helper.h"
#include "runtime/helpers/spy_reader.h"
#include "runtime/helpers/dummy_parser.h"
#include "tree_sitter/parser.h"
extern "C" TSParser ts_parser_json();
TSTree *lex_fn_node_to_return;
TSStateId lex_fn_state_received;
TSParser *lex_fn_parser_received;
TSTree * fake_lex(TSParser *parser, TSStateId state_id) {
lex_fn_parser_received = parser;
lex_fn_state_received = state_id;
return lex_fn_node_to_return;
}
START_TEST
describe("incremental parsing", [&]() {
TSDocument *doc;
describe("LR Parsers", [&]() {
TSParser *parser;
SpyReader *reader;
before_each([&]() {
doc = ts_document_make();
ts_document_set_parser(doc, ts_parser_json());
TSParserConfig config = dummy_parser;
config.lex_fn = fake_lex;
reader = new SpyReader("{ \"key\": [1, 2] }", 5);
ts_document_set_input(doc, reader->input);
parser = ts_parser_make(config);
reader = new SpyReader("some structured text", 5);
});
after_each([&]() {
ts_document_free(doc);
ts_parser_free(parser);
delete reader;
});
it("parses the input", [&]() {
AssertThat(string(ts_document_string(doc)), Equals(
"(value (object (string) (array (number) (number))))"));
});
it("reads the entire input", [&]() {
AssertThat(reader->strings_read, Equals(vector<string>({
"{ \"key\": [1, 2] }"
})));
});
describe("modifying the end of the input", [&]() {
describe("when starting at the beginning of the input (edit is NULL)", [&]() {
before_each([&]() {
size_t position(string("{ \"key\": [1, 2]").length());
string inserted_text(", \"key2\": 4");
reader->content.insert(position, inserted_text);
ts_document_edit(doc, { position, 0, inserted_text.length() });
ts_parser_start(parser, reader->input, nullptr);
});
it("updates the parse tree", [&]() {
AssertThat(string(ts_document_string(doc)), Equals(
"(value (object (string) (array (number) (number)) (string) (number)))"));
it("runs the lexer with the lex state corresponding to the initial state", [&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1);
ts_parser_step(parser);
AssertThat(lex_fn_state_received, Equals(100));
});
it("re-reads only the changed portion of the input", [&]() {
AssertThat(reader->strings_read.size(), Equals<size_t>(2));
AssertThat(reader->strings_read[1], Equals(", \"key2\": 4 }"));
describe("when the returned symbol indicates a shift action", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1);
});
it("advances to the state specified in the action", [&]() {
ts_parser_step(parser);
AssertThat(ts_stack_top_state(&parser->stack), Equals(12));
});
it("continues parsing (returns NULL)", [&]() {
auto result = ts_parser_step(parser);
AssertThat(result, Equals((TSTree *)nullptr));
});
});
describe("when the returned symbol indicates an error", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym1, 5, 1);
});
it("ends the parse, returning an error tree", [&]() {
auto result = ts_parser_step(parser);
AssertThat(ts_tree_symbol(result), Equals(ts_builtin_sym_error));
});
});
});
describe("modifying the beginning of the input", [&]() {
before_each([&]() {
size_t position(string("{ ").length());
string inserted_text("\"key2\": 4, ");
reader->content.insert(position, inserted_text);
ts_document_edit(doc, { position, 0, inserted_text.length() });
});
it("2 updates the parse tree", [&]() {
AssertThat(string(ts_document_string(doc)), Equals(
"(value (object (string) (number) (string) (array (number) (number))))"));
});
it_skip("re-reads only the changed portion of the input", [&]() {
AssertThat(reader->strings_read.size(), Equals<size_t>(2));
AssertThat(reader->strings_read[1], Equals("\"key2\": 4, "));
});
});
});
END_TEST

2
spec/runtime/stack_spec.cc
View file

@ -1,5 +1,5 @@
#include "runtime/runtime_spec_helper.h"
#include "tree_sitter/parser/stack.h"
#include "tree_sitter/parser.h"
START_TEST

75
spec/runtime/state_machine_spec.cc
View file

@ -1,75 +0,0 @@
#include "runtime/runtime_spec_helper.h"
#include "runtime/helpers/spy_reader.h"
#include "runtime/helpers/dummy_parser.h"
#include "tree_sitter/parser/state_machine.h"
TSTree *lex_fn_node_to_return;
TSStateId lex_fn_state_received;
TSLexer *lex_fn_lexer_received;
TSTree * fake_lex(TSLexer *lexer, TSStateId state_id) {
lex_fn_lexer_received = lexer;
lex_fn_state_received = state_id;
return lex_fn_node_to_return;
}
START_TEST
describe("LR Parsers", [&]() {
TSStateMachine *parser;
SpyReader *reader;
before_each([&]() {
reader = new SpyReader("some structured text", 5);
parser = ts_state_machine_make(dummy_parser.symbol_count,
(const TSParseAction *)dummy_parser.parse_table,
dummy_parser.lex_states,
fake_lex,
dummy_parser.hidden_symbols);
});
after_each([&]() {
delete reader;
});
describe("when starting at the beginning of the input (edit is NULL)", [&]() {
before_each([&]() {
ts_state_machine_initialize(parser, reader->input, nullptr);
});
it("runs the lexer with the lex state corresponding to the initial state", [&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1);
ts_state_machine_parse(parser, nullptr);
AssertThat(lex_fn_state_received, Equals(100));
});
describe("when the returned symbol indicates a shift action", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1);
});
it("advances to the state specified in the action", [&]() {
ts_state_machine_parse(parser, nullptr);
AssertThat(ts_stack_top_state(&parser->stack), Equals(12));
});
it("continues parsing (returns NULL)", [&]() {
auto result = ts_state_machine_parse(parser, nullptr);
AssertThat(result, Equals((TSTree *)nullptr));
});
});
describe("when the returned symbol indicates an error", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym1, 5, 1);
});
it("ends the parse, returning an error tree", [&]() {
auto result = ts_state_machine_parse(parser, nullptr);
AssertThat(ts_tree_symbol(result), Equals(ts_builtin_sym_error));
});
});
});
});
END_TEST

16
src/runtime/document.c
View file

@ -1,8 +1,9 @@
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser.h"
#include <string.h>
struct TSDocument {
TSParser parser;
TSParser *parser;
const TSTree *tree;
TSInput input;
size_t error_count;
@ -13,14 +14,13 @@ TSDocument * ts_document_make() {
}
void ts_document_free(TSDocument *document) {
if (document->parser.free_fn)
document->parser.free_fn(document->parser.data);
ts_parser_free(document->parser);
if (document->input.release_fn)
document->input.release_fn(document->input.data);
free(document);
}
void ts_document_set_parser(TSDocument *document, TSParser parser) {
void ts_document_set_parser(TSDocument *document, TSParser *parser) {
document->parser = parser;
}
@ -29,20 +29,20 @@ const TSTree * ts_document_tree(const TSDocument *document) {
}
const char * ts_document_string(const TSDocument *document) {
return ts_tree_string(document->tree, document->parser.symbol_names);
return ts_tree_string(document->tree, ts_parser_config(document->parser).symbol_names);
}
void ts_document_set_input(TSDocument *document, TSInput input) {
document->input = input;
document->tree = document->parser.parse_fn(document->parser.data, input, NULL);
document->tree = ts_parser_parse(document->parser, document->input, NULL);
}
void ts_document_edit(TSDocument *document, TSInputEdit edit) {
document->tree = document->parser.parse_fn(document->parser.data, document->input, &edit);
document->tree = ts_parser_parse(document->parser, document->input, &edit);
}
const char * ts_document_symbol_name(const TSDocument *document, const TSTree *tree) {
return document->parser.symbol_names[ts_tree_symbol(tree)];
return ts_parser_config(document->parser).symbol_names[ts_tree_symbol(tree)];
}
typedef struct {

43
src/runtime/lexer.c Normal file
View file

@ -0,0 +1,43 @@
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser.h"
TSLexer ts_lexer_make() {
return (TSLexer) {
.chunk = NULL,
.debug = 0,
.chunk_start = 0,
.chunk_size = 0,
.position_in_chunk = 0,
.token_start_position = 0,
.token_end_position = 0,
.reached_end = 0
};
}
int ts_lexer_advance(TSLexer *lexer) {
static const char *empty_chunk = "";
if (lexer->position_in_chunk + 1 < lexer->chunk_size) {
lexer->position_in_chunk++;
} else {
lexer->chunk_start += lexer->chunk_size;
lexer->chunk = lexer->input.read_fn(lexer->input.data, &lexer->chunk_size);
lexer->position_in_chunk = 0;
if (lexer->reached_end) {
return 0;
}
if (lexer->chunk_size == 0) {
lexer->reached_end = 1;
lexer->chunk = empty_chunk;
}
}
return 1;
}
TSTree * ts_lexer_build_node(TSLexer *lexer, TSSymbol symbol) {
size_t current_position = ts_lexer_position(lexer);
size_t size = current_position - lexer->token_start_position;
size_t offset = lexer->token_start_position - lexer->token_end_position;
lexer->token_end_position = current_position;
return ts_tree_make_leaf(symbol, size, offset);
}

262
src/runtime/parser.c Normal file
View file

@ -0,0 +1,262 @@
#include <stdio.h>
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser.h"
#include "runtime/tree.h"
/*
* Private
*/
static const TSParseAction * actions_for_state(TSParserConfig config, TSStateId state) {
return config.parse_table + (state * config.symbol_count);
}
static size_t breakdown_stack(TSParser *parser, TSInputEdit *edit) {
if (!edit) return 0;
TSStack *stack = &parser->stack;
size_t position = 0;
for (;;) {
TSTree *node = ts_stack_top_node(stack);
if (!node) break;
position = ts_stack_right_position(stack);
size_t child_count;
TSTree **children = ts_tree_immediate_children(node, &child_count);
if (position <= edit->position && !children) break;
stack->size--;
position -= ts_tree_total_size(node);
for (size_t i = 0; i < child_count && position < edit->position; i++) {
TSTree *child = children[i];
TSStateId state = ts_stack_top_state(stack);
TSStateId next_state = actions_for_state(parser->config, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child);
ts_tree_retain(child);
position += ts_tree_total_size(child);
}
ts_tree_release(node);
}
return position;
}
static TSSymbol * expected_symbols(TSParser *parser, size_t *count) {
*count = 0;
const TSParseAction *actions = actions_for_state(parser->config, ts_stack_top_state(&parser->stack));
for (size_t i = 0; i < parser->config.symbol_count; i++)
if (actions[i].type != TSParseActionTypeError)
(*count)++;
size_t n = 0;
TSSymbol *result = malloc(*count * sizeof(*result));
for (TSSymbol i = 0; i < parser->config.symbol_count; i++)
if (actions[i].type != TSParseActionTypeError)
result[n++] = i;
return result;
}
/*
* Public
*/
TSParser * ts_parser_make(TSParserConfig config) {
TSParser *result = malloc(sizeof(*result));
*result = (TSParser) {
.lexer = ts_lexer_make(),
.stack = ts_stack_make(),
.debug = 0,
.config = config,
};
return result;
}
void ts_parser_free(TSParser *parser) {
if (parser->lookahead) ts_tree_release(parser->lookahead);
if (parser->next_lookahead) ts_tree_release(parser->next_lookahead);
ts_stack_delete(&parser->stack);
free(parser);
}
void ts_parser_start(TSParser *parser, TSInput input, TSInputEdit *edit) {
if (!edit) ts_stack_shrink(&parser->stack, 0);
parser->lookahead = NULL;
parser->next_lookahead = NULL;
size_t position = breakdown_stack(parser, edit);
input.seek_fn(input.data, position);
parser->lexer = ts_lexer_make();
parser->lexer.input = input;
ts_lexer_advance(&parser->lexer);
}
void ts_parser_shift(TSParser *parser, TSStateId parse_state) {
if (parser->lookahead->is_extra)
parse_state = ts_stack_top_state(&parser->stack);
ts_stack_push(&parser->stack, parse_state, parser->lookahead);
parser->lookahead = parser->next_lookahead;
parser->next_lookahead = NULL;
}
void ts_parser_shift_extra(TSParser *parser) {
parser->lookahead->is_extra = 1;
ts_parser_shift(parser, 0);
}
void ts_parser_reduce(TSParser *parser, TSSymbol symbol, size_t child_count) {
parser->next_lookahead = parser->lookahead;
parser->lookahead = ts_stack_reduce(
&parser->stack,
symbol,
child_count,
parser->config.hidden_symbol_flags, 1);
}
int ts_parser_reduce_extra(TSParser *parser, TSSymbol symbol) {
TSTree *top_node = ts_stack_top_node(&parser->stack);
if (top_node->symbol == symbol && !top_node->is_extra) {
ts_parser_reduce(parser, symbol, 1);
parser->lookahead->is_extra = 1;
return 1;
} else {
return 0;
}
}
int ts_parser_handle_error(TSParser *parser) {
size_t count = 0;
const TSSymbol *inputs = expected_symbols(parser, &count);
TSTree *error = ts_tree_make_error(ts_lexer_lookahead_char(&parser->lexer),
count,
inputs,
0,
0);
for (;;) {
ts_tree_release(parser->lookahead);
size_t position = ts_lexer_position(&parser->lexer);
parser->lookahead = parser->config.lex_fn(parser, ts_lex_state_error);
int at_end = 0;
if (ts_lexer_position(&parser->lexer) == position)
at_end = !ts_lexer_advance(&parser->lexer);
if (at_end || ts_tree_symbol(parser->lookahead) == ts_builtin_sym_end) {
ts_stack_push(&parser->stack, 0, error);
return 0;
}
/*
* Unwind the stack, looking for a state in which this token
* may appear after an error.
*/
for (size_t j = 0; j < parser->stack.size; j++) {
size_t i = parser->stack.size - 1 - j;
TSStateId stack_state = parser->stack.entries[i].state;
TSParseAction action_on_error = actions_for_state(parser->config, stack_state)[ts_builtin_sym_error];
if (action_on_error.type == TSParseActionTypeShift) {
TSStateId state_after_error = action_on_error.data.to_state;
if (actions_for_state(parser->config, state_after_error)[ts_tree_symbol(parser->lookahead)].type != TSParseActionTypeError) {
ts_stack_shrink(&parser->stack, i + 1);
ts_stack_push(&parser->stack, state_after_error, error);
return 1;
}
}
}
}
}
TSTree * ts_parser_tree_root(TSParser *parser) {
TSStack *stack = &parser->stack;
TSTree *top_node = ts_stack_top_node(stack);
if (stack->size <= 1)
return top_node;
if (ts_tree_symbol(top_node) == ts_builtin_sym_error)
return top_node;
size_t immediate_child_count;
TSTree **immedate_children = ts_tree_immediate_children(top_node, &immediate_child_count);
stack->size--;
for (size_t i = 0; i < immediate_child_count; i++) {
TSTree *child = immedate_children[i];
child->is_extra = 0;
ts_tree_retain(child);
TSStateId state = ts_stack_top_state(stack);
TSStateId next_state = actions_for_state(parser->config, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child);
}
TSTree *new_node = ts_stack_reduce(stack,
top_node->symbol,
stack->size,
parser->config.hidden_symbol_flags,
0);
ts_tree_release(top_node);
return new_node;
}
TSParseAction ts_parser_next_action(TSParser *parser) {
TSStateId state = ts_stack_top_state(&parser->stack);
if (!parser->lookahead)
parser->lookahead = parser->config.lex_fn(parser, parser->config.lex_states[state]);
return actions_for_state(parser->config, state)[ts_tree_symbol(parser->lookahead)];
}
#define DEBUG_PARSE(...) \
if (parser->debug) { fprintf(stderr, "\n" __VA_ARGS__); }
TSTree * ts_parser_step(TSParser *parser) {
TSParseAction action = ts_parser_next_action(parser);
DEBUG_PARSE("LOOKAHEAD %s", parser->config.symbol_names[ts_tree_symbol(parser->lookahead)]);
switch (action.type) {
case TSParseActionTypeShift:
DEBUG_PARSE("SHIFT %d", action.data.to_state);
ts_parser_shift(parser, action.data.to_state);
return NULL;
case TSParseActionTypeShiftExtra:
DEBUG_PARSE("SHIFT EXTRA");
ts_parser_shift_extra(parser);
return NULL;
case TSParseActionTypeReduce:
DEBUG_PARSE("REDUCE %s %d", parser->config.symbol_names[action.data.symbol], action.data.child_count);
ts_parser_reduce(parser, action.data.symbol, action.data.child_count);
return NULL;
case TSParseActionTypeReduceExtra:
if (!ts_parser_reduce_extra(parser, action.data.symbol))
goto error;
DEBUG_PARSE("REDUCE EXTRA");
return NULL;
case TSParseActionTypeAccept:
DEBUG_PARSE("ACCEPT");
return ts_parser_tree_root(parser);
case TSParseActionTypeError:
goto error;
default:
return NULL;
}
error:
DEBUG_PARSE("ERROR");
if (!ts_parser_handle_error(parser))
return ts_parser_tree_root(parser);
else
return NULL;
}
const TSTree * ts_parser_parse(TSParser *parser, TSInput input, TSInputEdit *edit) {
ts_parser_start(parser, input, edit);
for (;;) {
const TSTree *tree = ts_parser_step(parser);
if (tree) return tree;
}
}
TSParserConfig ts_parser_config(TSParser *parser) {
return parser->config;
}

2
src/runtime/stack.c
View file

@ -1,5 +1,5 @@
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser/stack.h"
#include "tree_sitter/parser.h"
#include "runtime/tree.h"
#include <string.h>

263
src/runtime/state_machine.c
View file

@ -1,263 +0,0 @@
#include "tree_sitter/parser/state_machine.h"
#include "runtime/tree.h"
/*
* Private
*/
static const TSParseAction * actions_for_state(TSStateMachine *machine, TSStateId state) {
return machine->config.parse_table + (state * machine->config.symbol_count);
}
void shift(TSStateMachine *machine, TSStateId parse_state) {
if (machine->lookahead->is_extra)
parse_state = ts_stack_top_state(&machine->stack);
ts_stack_push(&machine->stack, parse_state, machine->lookahead);
machine->lookahead = machine->next_lookahead;
machine->next_lookahead = NULL;
}
void shift_extra(TSStateMachine *machine) {
machine->lookahead->is_extra = 1;
shift(machine, 0);
}
void reduce(TSStateMachine *machine, TSSymbol symbol, size_t child_count) {
machine->next_lookahead = machine->lookahead;
machine->lookahead = ts_stack_reduce(
&machine->stack,
symbol,
child_count,
machine->config.hidden_symbol_flags, 1);
}
int reduce_extra(TSStateMachine *machine, TSSymbol symbol) {
TSTree *top_node = ts_stack_top_node(&machine->stack);
if (top_node->symbol == symbol && !top_node->is_extra) {
reduce(machine, symbol, 1);
machine->lookahead->is_extra = 1;
return 1;
} else {
return 0;
}
}
static size_t breakdown_stack(TSStateMachine *machine, TSInputEdit *edit) {
if (!edit) return 0;
TSStack *stack = &machine->stack;
size_t position = 0;
for (;;) {
TSTree *node = ts_stack_top_node(stack);
if (!node) break;
position = ts_stack_right_position(stack);
size_t child_count;
TSTree **children = ts_tree_immediate_children(node, &child_count);
if (position <= edit->position && !children) break;
stack->size--;
position -= ts_tree_total_size(node);
for (size_t i = 0; i < child_count && position < edit->position; i++) {
TSTree *child = children[i];
TSStateId state = ts_stack_top_state(stack);
TSStateId next_state = actions_for_state(machine, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child);
ts_tree_retain(child);
position += ts_tree_total_size(child);
}
ts_tree_release(node);
}
return position;
}
TSSymbol * expected_symbols(TSStateMachine *machine, size_t *count) {
*count = 0;
const TSParseAction *actions = actions_for_state(machine, ts_stack_top_state(&machine->stack));
for (size_t i = 0; i < machine->config.symbol_count; i++)
if (actions[i].type != TSParseActionTypeError)
++(*count);
size_t n = 0;
TSSymbol *result = malloc(*count * sizeof(*result));
for (TSSymbol i = 0; i < machine->config.symbol_count; i++)
if (actions[i].type != TSParseActionTypeError)
result[n++] = i;
return result;
}
int handle_error(TSStateMachine *machine) {
size_t count = 0;
const TSSymbol *inputs = expected_symbols(machine, &count);
TSTree *error = ts_tree_make_error(ts_lexer_lookahead_char(&machine->lexer),
count,
inputs,
0,
0);
for (;;) {
ts_tree_release(machine->lookahead);
size_t position = ts_lexer_position(&machine->lexer);
machine->lookahead = machine->config.lex_fn(&machine->lexer, ts_lex_state_error);
int at_end = 0;
if (ts_lexer_position(&machine->lexer) == position)
at_end = !ts_lexer_advance(&machine->lexer);
if (at_end || ts_tree_symbol(machine->lookahead) == ts_builtin_sym_end) {
ts_stack_push(&machine->stack, 0, error);
return 0;
}
/*
* Unwind the stack, looking for a state in which this token
* may appear after an error.
*/
for (size_t j = 0; j < machine->stack.size; j++) {
size_t i = machine->stack.size - 1 - j;
TSStateId stack_state = machine->stack.entries[i].state;
TSParseAction action_on_error = actions_for_state(machine, stack_state)[ts_builtin_sym_error];
if (action_on_error.type == TSParseActionTypeShift) {
TSStateId state_after_error = action_on_error.data.to_state;
if (actions_for_state(machine, state_after_error)[ts_tree_symbol(machine->lookahead)].type != TSParseActionTypeError) {
ts_stack_shrink(&machine->stack, i + 1);
ts_stack_push(&machine->stack, state_after_error, error);
return 1;
}
}
}
}
}
TSTree * get_tree_root(TSStateMachine *machine) {
TSStack *stack = &machine->stack;
TSTree *top_node = ts_stack_top_node(stack);
if (stack->size <= 1)
return top_node;
if (ts_tree_symbol(top_node) == ts_builtin_sym_error)
return top_node;
size_t immediate_child_count;
TSTree **immedate_children = ts_tree_immediate_children(top_node, &immediate_child_count);
stack->size--;
for (size_t i = 0; i < immediate_child_count; i++) {
TSTree *child = immedate_children[i];
child->is_extra = 0;
ts_tree_retain(child);
TSStateId state = ts_stack_top_state(stack);
TSStateId next_state = actions_for_state(machine, state)[ts_tree_symbol(child)].data.to_state;
ts_stack_push(stack, next_state, child);
}
TSTree *new_node = ts_stack_reduce(stack,
top_node->symbol,
stack->size,
machine->config.hidden_symbol_flags,
0);
ts_tree_release(top_node);
return new_node;
}
TSParseAction get_next_action(TSStateMachine *machine) {
TSStateId state = ts_stack_top_state(&machine->stack);
if (!machine->lookahead)
machine->lookahead = machine->config.lex_fn(&machine->lexer, machine->config.lex_states[state]);
return actions_for_state(machine, state)[ts_tree_symbol(machine->lookahead)];
}
/*
* Public API
*/
TSStateMachine * ts_state_machine_make(size_t symbol_count,
const TSParseAction *parse_table,
const TSStateId *lex_states,
TSTree * (* lex_fn)(TSLexer *, TSStateId),
const int *hidden_symbol_flags) {
TSStateMachine *result = malloc(sizeof(TSStateMachine));
*result = (TSStateMachine) {
.lexer = ts_lexer_make(),
.stack = ts_stack_make(),
.config = {
.symbol_count = symbol_count,
.parse_table = parse_table,
.lex_states = lex_states,
.lex_fn = lex_fn,
.hidden_symbol_flags = hidden_symbol_flags,
},
};
return result;
}
void ts_state_machine_free(void *data) {
TSStateMachine *machine = (TSStateMachine *)data;
if (machine->lookahead) ts_tree_release(machine->lookahead);
if (machine->next_lookahead) ts_tree_release(machine->next_lookahead);
ts_stack_delete(&machine->stack);
free(machine);
}
void ts_state_machine_initialize(TSStateMachine *machine, TSInput input, TSInputEdit *edit) {
if (!edit) ts_stack_shrink(&machine->stack, 0);
machine->lookahead = NULL;
machine->next_lookahead = NULL;
size_t position = breakdown_stack(machine, edit);
input.seek_fn(input.data, position);
machine->lexer = ts_lexer_make();
machine->lexer.input = input;
ts_lexer_advance(&machine->lexer);
}
// #define TS_DEBUG_PARSE
#ifdef TS_DEBUG_PARSE
#include <stdio.h>
#define DEBUG_PARSE(...) fprintf(stderr, "\n" __VA_ARGS__)
#else
#define DEBUG_PARSE(...)
#endif
TSTree * ts_state_machine_parse(TSStateMachine *machine, const char **symbol_names) {
TSParseAction action = get_next_action(machine);
DEBUG_PARSE("LOOKAHEAD %s", symbol_names[ts_tree_symbol(machine->lookahead)]);
switch (action.type) {
case TSParseActionTypeShift:
DEBUG_PARSE("SHIFT %d", action.data.to_state);
shift(machine, action.data.to_state);
return NULL;
case TSParseActionTypeShiftExtra:
DEBUG_PARSE("SHIFT EXTRA");
shift_extra(machine);
return NULL;
case TSParseActionTypeReduce:
DEBUG_PARSE("REDUCE %s %d", symbol_names[action.data.symbol], action.data.child_count);
reduce(machine, action.data.symbol, action.data.child_count);
return NULL;
case TSParseActionTypeReduceExtra:
if (!reduce_extra(machine, action.data.symbol))
goto error;
DEBUG_PARSE("REDUCE EXTRA");
return NULL;
case TSParseActionTypeAccept:
DEBUG_PARSE("ACCEPT");
return get_tree_root(machine);
case TSParseActionTypeError:
goto error;
default:
return NULL;
}
error:
DEBUG_PARSE("ERROR");
if (handle_error(machine))
return NULL;
else
return get_tree_root(machine);
}