diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index e6eaa796..a636e5af 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -5,18 +5,109 @@ extern "C" { #endif -// #define TS_DEBUG_LEX - +#include #include "tree_sitter/runtime.h" -#include "tree_sitter/parser/lexer.h" -#include "tree_sitter/parser/stack.h" -#include "tree_sitter/parser/state_machine.h" + +#define ts_lex_state_error 0 + +typedef struct { + TSInput input; + int debug; + const char *chunk; + size_t chunk_start; + size_t chunk_size; + size_t position_in_chunk; + size_t token_end_position; + size_t token_start_position; + int reached_end; +} TSLexer; + +TSLexer ts_lexer_make(); +int ts_lexer_advance(TSLexer *lexer); +TSTree * ts_lexer_build_node(TSLexer *lexer, TSSymbol symbol); + +static inline size_t ts_lexer_position(const TSLexer *lexer) { + return lexer->chunk_start + lexer->position_in_chunk; +} + +static inline char ts_lexer_lookahead_char(const TSLexer *lexer) { + return lexer->chunk[lexer->position_in_chunk]; +} + +static inline void ts_lexer_start_token(TSLexer *lexer) { + lexer->token_start_position = ts_lexer_position(lexer); +} + +typedef unsigned short TSStateId; + +typedef struct { + size_t size; + struct { + TSTree *node; + TSStateId state; + int is_extra; + } *entries; +} TSStack; + +TSStack ts_stack_make(); +void ts_stack_delete(TSStack *); +TSTree * ts_stack_reduce(TSStack *stack, TSSymbol symbol, size_t immediate_child_count, const int *hidden_symbol_flags, int gather_extras); +void ts_stack_shrink(TSStack *stack, size_t new_size); +void ts_stack_push(TSStack *stack, TSStateId state, TSTree *node); +TSStateId ts_stack_top_state(const TSStack *stack); +TSTree * ts_stack_top_node(const TSStack *stack); +size_t ts_stack_right_position(const TSStack *stack); + +typedef enum { + TSParseActionTypeError, + TSParseActionTypeShift, + TSParseActionTypeShiftExtra, + TSParseActionTypeReduce, + TSParseActionTypeReduceExtra, + TSParseActionTypeAccept, +} TSParseActionType; + +typedef struct { + TSParseActionType type; + union { + TSStateId to_state; + struct { + TSSymbol symbol; + unsigned short child_count; + }; + } data; +} TSParseAction; + +typedef struct { + size_t symbol_count; + const char **symbol_names; + const int *hidden_symbol_flags; + const TSParseAction *parse_table; + const TSStateId *lex_states; + TSTree * (* lex_fn)(TSParser *, TSStateId); +} TSParserConfig; + +struct TSParser { + TSLexer lexer; + TSStack stack; + int debug; + TSTree *lookahead; + TSTree *next_lookahead; + TSParserConfig config; +}; + +TSParser * ts_parser_make(TSParserConfig); +void ts_parser_free(TSParser *); +TSParserConfig ts_parser_config(TSParser *); +const TSTree * ts_parser_parse(TSParser *parser, TSInput input, TSInputEdit *edit); +void ts_parser_start(TSParser *parser, TSInput input, TSInputEdit *edit); +TSTree * ts_parser_step(TSParser *parser); #define SYMBOL_NAMES \ static const char *ts_symbol_names[] #define HIDDEN_SYMBOLS \ -static const int hidden_symbol_flags[SYMBOL_COUNT] +static const int ts_hidden_symbol_flags[SYMBOL_COUNT] #define LEX_STATES \ static TSStateId ts_lex_states[STATE_COUNT] @@ -25,42 +116,38 @@ static TSStateId ts_lex_states[STATE_COUNT] static const TSParseAction ts_parse_actions[STATE_COUNT][SYMBOL_COUNT] #define LEX_FN() \ -static TSTree * ts_lex(TSLexer *lexer, TSStateId lex_state) +static TSTree * ts_lex(TSParser *parser, TSStateId lex_state) -#ifdef TS_DEBUG_LEX -#include -#define DEBUG_LEX(...) fprintf(stderr, "\n" __VA_ARGS__) -#else -#define DEBUG_LEX(...) -#endif +#define DEBUG_LEX(...) \ +if (parser->lexer.debug) { fprintf(stderr, "\n" __VA_ARGS__); } #define START_LEXER() \ DEBUG_LEX("LEX %d", lex_state); \ char lookahead; \ next_state: \ -lookahead = ts_lexer_lookahead_char(lexer); \ +lookahead = ts_lexer_lookahead_char(&parser->lexer); \ DEBUG_LEX("CHAR '%c'", lookahead); #define START_TOKEN() \ -ts_lexer_start_token(lexer); +ts_lexer_start_token(&parser->lexer); #define ADVANCE(state_index) \ { \ DEBUG_LEX("ADVANCE %d", state_index); \ - if (!ts_lexer_advance(lexer)) ACCEPT_TOKEN(ts_builtin_sym_end); \ + if (!ts_lexer_advance(&parser->lexer)) ACCEPT_TOKEN(ts_builtin_sym_end); \ lex_state = state_index; goto next_state; \ } #define ACCEPT_TOKEN(symbol) \ { \ DEBUG_LEX("TOKEN %s", ts_symbol_names[symbol]); \ - return ts_lexer_build_node(lexer, symbol); \ + return ts_lexer_build_node(&parser->lexer, symbol); \ } #define LEX_ERROR() \ { \ DEBUG_LEX("ERROR"); \ - return ts_lexer_build_node(lexer, ts_builtin_sym_error); \ + return ts_lexer_build_node(&parser->lexer, ts_builtin_sym_error); \ } #define LEX_PANIC() \ @@ -69,31 +156,32 @@ ts_lexer_start_token(lexer); return NULL; \ } -SYMBOL_NAMES; +#define SHIFT(to_state_value) \ +{ .type = TSParseActionTypeShift, .data = { .to_state = to_state_value } } + +#define SHIFT_EXTRA() \ +{ .type = TSParseActionTypeShiftExtra } + +#define REDUCE_EXTRA(symbol_val) \ +{ .type = TSParseActionTypeReduceExtra, .data = { .symbol = symbol_val } } + +#define REDUCE(symbol_val, child_count_val) \ +{ .type = TSParseActionTypeReduce, .data = { .symbol = symbol_val, .child_count = child_count_val } } + +#define ACCEPT_INPUT() \ +{ .type = TSParseActionTypeAccept } -static const TSTree * ts_parse(void *data, TSInput input, TSInputEdit *edit) { - TSStateMachine *parser = (TSStateMachine *)data; - ts_state_machine_initialize(parser, input, edit); - for (;;) { - const TSTree *tree = ts_state_machine_parse(parser, ts_symbol_names); - if (tree) return tree; - } -} #define EXPORT_PARSER(constructor_name) \ -TSParser constructor_name() { \ - return (TSParser) { \ - .parse_fn = ts_parse, \ - .free_fn = ts_state_machine_free, \ - .symbol_names = ts_symbol_names, \ - .data = ts_state_machine_make( \ - SYMBOL_COUNT, \ - (const TSParseAction *)ts_parse_actions, \ - ts_lex_states, \ - ts_lex, \ - hidden_symbol_flags \ - ), \ - }; \ +TSParser * constructor_name() { \ + return ts_parser_make((TSParserConfig) { \ + .symbol_count = SYMBOL_COUNT, \ + .hidden_symbol_flags = ts_hidden_symbol_flags, \ + .parse_table = (const TSParseAction *)ts_parse_actions, \ + .lex_states = ts_lex_states, \ + .symbol_names = ts_symbol_names, \ + .lex_fn = ts_lex, \ + }); \ } #ifdef __cplusplus diff --git a/include/tree_sitter/parser/lexer.h b/include/tree_sitter/parser/lexer.h deleted file mode 100644 index 88e93ddc..00000000 --- a/include/tree_sitter/parser/lexer.h +++ /dev/null @@ -1,70 +0,0 @@ -#ifndef TREE_SITTER_PARSER_LEXER_H_ -#define TREE_SITTER_PARSER_LEXER_H_ - -#include "tree_sitter/runtime.h" - -#define ts_lex_state_error 0 - -typedef struct { - TSInput input; - const char *chunk; - size_t chunk_start; - size_t chunk_size; - size_t position_in_chunk; - size_t token_end_position; - size_t token_start_position; - int reached_end; -} TSLexer; - -static inline TSLexer ts_lexer_make() { - TSLexer result; - result.chunk = NULL; - result.chunk_start = 0; - result.chunk_size = 0; - result.position_in_chunk = 0; - result.token_start_position = 0; - result.token_end_position = 0; - result.reached_end = 0; - return result; -} - -static inline size_t ts_lexer_position(const TSLexer *lexer) { - return lexer->chunk_start + lexer->position_in_chunk; -} - -static inline char ts_lexer_lookahead_char(const TSLexer *lexer) { - return lexer->chunk[lexer->position_in_chunk]; -} - -static inline int ts_lexer_advance(TSLexer *lexer) { - static const char *empty_chunk = ""; - if (lexer->position_in_chunk + 1 < lexer->chunk_size) { - lexer->position_in_chunk++; - } else { - lexer->chunk_start += lexer->chunk_size; - lexer->chunk = lexer->input.read_fn(lexer->input.data, &lexer->chunk_size); - lexer->position_in_chunk = 0; - if (lexer->reached_end) { - return 0; - } - if (lexer->chunk_size == 0) { - lexer->reached_end = 1; - lexer->chunk = empty_chunk; - } - } - return 1; -} - -static inline void ts_lexer_start_token(TSLexer *lexer) { - lexer->token_start_position = ts_lexer_position(lexer); -} - -static inline TSTree * ts_lexer_build_node(TSLexer *lexer, TSSymbol symbol) { - size_t current_position = ts_lexer_position(lexer); - size_t size = current_position - lexer->token_start_position; - size_t offset = lexer->token_start_position - lexer->token_end_position; - lexer->token_end_position = current_position; - return ts_tree_make_leaf(symbol, size, offset); -} - -#endif diff --git a/include/tree_sitter/parser/stack.h b/include/tree_sitter/parser/stack.h deleted file mode 100644 index a0b9384d..00000000 --- a/include/tree_sitter/parser/stack.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef TREE_SITTER_PARSER_STACK_H_ -#define TREE_SITTER_PARSER_STACK_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tree_sitter/runtime.h" - -typedef unsigned short TSStateId; - -typedef struct { - size_t size; - struct { - TSTree *node; - TSStateId state; - int is_extra; - } *entries; -} TSStack; - -TSStack ts_stack_make(); -void ts_stack_delete(TSStack *); -TSTree * ts_stack_reduce(TSStack *stack, TSSymbol symbol, size_t immediate_child_count, const int *hidden_symbol_flags, int gather_extras); -void ts_stack_shrink(TSStack *stack, size_t new_size); -void ts_stack_push(TSStack *stack, TSStateId state, TSTree *node); -TSStateId ts_stack_top_state(const TSStack *stack); -TSTree * ts_stack_top_node(const TSStack *stack); -size_t ts_stack_right_position(const TSStack *stack); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/include/tree_sitter/parser/state_machine.h b/include/tree_sitter/parser/state_machine.h deleted file mode 100644 index c5d90877..00000000 --- a/include/tree_sitter/parser/state_machine.h +++ /dev/null @@ -1,74 +0,0 @@ -#ifndef TREE_SITTER_PARSER_LR_PARSER_H_ -#define TREE_SITTER_PARSER_LR_PARSER_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tree_sitter/parser/stack.h" -#include "tree_sitter/parser/lexer.h" - -typedef enum { - TSParseActionTypeError, - TSParseActionTypeShift, - TSParseActionTypeShiftExtra, - TSParseActionTypeReduce, - TSParseActionTypeReduceExtra, - TSParseActionTypeAccept, -} TSParseActionType; - -typedef struct { - TSParseActionType type; - union { - TSStateId to_state; - struct { - TSSymbol symbol; - unsigned short child_count; - }; - } data; -} TSParseAction; - -#define SHIFT(to_state_value) \ -{ .type = TSParseActionTypeShift, .data = { .to_state = to_state_value } } - -#define SHIFT_EXTRA() \ -{ .type = TSParseActionTypeShiftExtra } - -#define REDUCE_EXTRA(symbol_val) \ -{ .type = TSParseActionTypeReduceExtra, .data = { .symbol = symbol_val } } - -#define REDUCE(symbol_val, child_count_val) \ -{ .type = TSParseActionTypeReduce, .data = { .symbol = symbol_val, .child_count = child_count_val } } - -#define ACCEPT_INPUT() \ -{ .type = TSParseActionTypeAccept } - -typedef struct { - TSLexer lexer; - TSStack stack; - TSTree *lookahead; - TSTree *next_lookahead; - struct { - size_t symbol_count; - const int *hidden_symbol_flags; - const TSParseAction *parse_table; - const TSStateId *lex_states; - TSTree * (* lex_fn)(TSLexer *, TSStateId); - } config; -} TSStateMachine; - -TSStateMachine * ts_state_machine_make( - size_t symbol_count, - const TSParseAction *parse_table, - const TSStateId *lex_states, - TSTree * (* lex_fn)(TSLexer *, TSStateId), - const int *hidden_symbol_flags); -void ts_state_machine_free(void *data); -void ts_state_machine_initialize(TSStateMachine *, TSInput, TSInputEdit *); -TSTree * ts_state_machine_parse(TSStateMachine *, const char **symbol_names); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index e81b5108..65b4bad8 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -41,17 +41,12 @@ typedef struct { size_t bytes_removed; } TSInputEdit; -typedef struct { - const TSTree * (* parse_fn)(void *data, TSInput input, TSInputEdit *edit); - void (* free_fn)(void *data); - const char **symbol_names; - void *data; -} TSParser; +typedef struct TSParser TSParser; typedef struct TSDocument TSDocument; TSDocument * ts_document_make(); void ts_document_free(TSDocument *doc); -void ts_document_set_parser(TSDocument *doc, TSParser parser); +void ts_document_set_parser(TSDocument *doc, TSParser *parser); void ts_document_set_input(TSDocument *doc, TSInput input); void ts_document_set_input_string(TSDocument *doc, const char *text); void ts_document_edit(TSDocument *doc, TSInputEdit edit); diff --git a/spec/runtime/document_spec.cc b/spec/runtime/document_spec.cc new file mode 100644 index 00000000..2f8307a5 --- /dev/null +++ b/spec/runtime/document_spec.cc @@ -0,0 +1,78 @@ +#include "runtime/runtime_spec_helper.h" +#include "runtime/helpers/spy_reader.h" + +extern "C" TSParser * ts_parser_json(); + +START_TEST + +describe("incremental parsing", [&]() { + TSDocument *doc; + SpyReader *reader; + + before_each([&]() { + doc = ts_document_make(); + ts_document_set_parser(doc, ts_parser_json()); + + reader = new SpyReader("{ \"key\": [1, 2] }", 5); + ts_document_set_input(doc, reader->input); + }); + + after_each([&]() { + ts_document_free(doc); + delete reader; + }); + + it("parses the input", [&]() { + AssertThat(string(ts_document_string(doc)), Equals( + "(value (object (string) (array (number) (number))))")); + }); + + it("reads the entire input", [&]() { + AssertThat(reader->strings_read, Equals(vector({ + "{ \"key\": [1, 2] }" + }))); + }); + + describe("modifying the end of the input", [&]() { + before_each([&]() { + size_t position(string("{ \"key\": [1, 2]").length()); + string inserted_text(", \"key2\": 4"); + + reader->content.insert(position, inserted_text); + ts_document_edit(doc, { position, 0, inserted_text.length() }); + }); + + it("updates the parse tree", [&]() { + AssertThat(string(ts_document_string(doc)), Equals( + "(value (object (string) (array (number) (number)) (string) (number)))")); + }); + + it("re-reads only the changed portion of the input", [&]() { + AssertThat(reader->strings_read.size(), Equals(2)); + AssertThat(reader->strings_read[1], Equals(", \"key2\": 4 }")); + }); + }); + + describe("modifying the beginning of the input", [&]() { + before_each([&]() { + size_t position(string("{ ").length()); + string inserted_text("\"key2\": 4, "); + + reader->content.insert(position, inserted_text); + ts_document_edit(doc, { position, 0, inserted_text.length() }); + }); + + it("2 updates the parse tree", [&]() { + AssertThat(string(ts_document_string(doc)), Equals( + "(value (object (string) (number) (string) (array (number) (number))))")); + }); + + it_skip("re-reads only the changed portion of the input", [&]() { + AssertThat(reader->strings_read.size(), Equals(2)); + AssertThat(reader->strings_read[1], Equals("\"key2\": 4, ")); + }); + }); + +}); + +END_TEST diff --git a/spec/runtime/helpers/dummy_parser.c b/spec/runtime/helpers/dummy_parser.c index 569ccef1..50143dec 100644 --- a/spec/runtime/helpers/dummy_parser.c +++ b/spec/runtime/helpers/dummy_parser.c @@ -1,4 +1,5 @@ #include "runtime/helpers/dummy_parser.h" +#include "tree_sitter/parser.h" const TSParseAction parse_table[3][5] = { [0] = { @@ -29,10 +30,9 @@ const int hidden_symbols[5] = { [dummy_sym3] = 1, }; -struct test_parser dummy_parser = { - .state_count = 3, +TSParserConfig dummy_parser = { .symbol_count = 5, - .parse_table = (const TSParseAction **)parse_table, + .parse_table = (const TSParseAction *)parse_table, .lex_states = lex_states, - .hidden_symbols = hidden_symbols, + .hidden_symbol_flags = hidden_symbols, }; diff --git a/spec/runtime/helpers/dummy_parser.h b/spec/runtime/helpers/dummy_parser.h index 4d51ee2f..9b2d7cd0 100644 --- a/spec/runtime/helpers/dummy_parser.h +++ b/spec/runtime/helpers/dummy_parser.h @@ -5,7 +5,8 @@ extern "C" { #endif -#include "tree_sitter/parser/state_machine.h" +#include "tree_sitter/runtime.h" +#include "tree_sitter/parser.h" enum { dummy_sym1 = 2, @@ -13,15 +14,7 @@ enum { dummy_sym3 = 4, }; -struct test_parser { - size_t state_count; - size_t symbol_count; - const TSParseAction **parse_table; - const TSStateId *lex_states; - const int *hidden_symbols; -}; - -extern struct test_parser dummy_parser; +extern TSParserConfig dummy_parser; #ifdef __cplusplus } diff --git a/spec/runtime/language_specs.cc b/spec/runtime/language_specs.cc index 2faa1366..62ccf978 100644 --- a/spec/runtime/language_specs.cc +++ b/spec/runtime/language_specs.cc @@ -1,10 +1,10 @@ #include "runtime/runtime_spec_helper.h" #include "runtime/helpers/read_test_entries.h" -extern "C" TSParser ts_parser_javascript(); -extern "C" TSParser ts_parser_json(); -extern "C" TSParser ts_parser_arithmetic(); -extern "C" TSParser ts_parser_golang(); +extern "C" TSParser * ts_parser_javascript(); +extern "C" TSParser * ts_parser_json(); +extern "C" TSParser * ts_parser_arithmetic(); +extern "C" TSParser * ts_parser_golang(); START_TEST @@ -19,7 +19,7 @@ describe("Languages", [&]() { ts_document_free(doc); }); - auto run_tests_for_language = [&](string language, TSParser (parser_constructor)()) { + auto run_tests_for_language = [&](string language, TSParser * (parser_constructor)()) { describe(language.c_str(), [&]() { before_each([&]() { ts_document_set_parser(doc, parser_constructor()); diff --git a/spec/runtime/node_position_spec.cc b/spec/runtime/node_position_spec.cc index ebdaf7a1..9e593893 100644 --- a/spec/runtime/node_position_spec.cc +++ b/spec/runtime/node_position_spec.cc @@ -1,6 +1,6 @@ #include "runtime/runtime_spec_helper.h" -extern "C" TSParser ts_parser_json(); +extern "C" TSParser * ts_parser_json(); START_TEST diff --git a/spec/runtime/parser_spec.cc b/spec/runtime/parser_spec.cc index d0577094..f3c29784 100644 --- a/spec/runtime/parser_spec.cc +++ b/spec/runtime/parser_spec.cc @@ -1,78 +1,76 @@ #include "runtime/runtime_spec_helper.h" #include "runtime/helpers/spy_reader.h" +#include "runtime/helpers/dummy_parser.h" +#include "tree_sitter/parser.h" -extern "C" TSParser ts_parser_json(); +TSTree *lex_fn_node_to_return; +TSStateId lex_fn_state_received; +TSParser *lex_fn_parser_received; + +TSTree * fake_lex(TSParser *parser, TSStateId state_id) { + lex_fn_parser_received = parser; + lex_fn_state_received = state_id; + return lex_fn_node_to_return; +} START_TEST -describe("incremental parsing", [&]() { - TSDocument *doc; +describe("LR Parsers", [&]() { + TSParser *parser; SpyReader *reader; before_each([&]() { - doc = ts_document_make(); - ts_document_set_parser(doc, ts_parser_json()); + TSParserConfig config = dummy_parser; + config.lex_fn = fake_lex; - reader = new SpyReader("{ \"key\": [1, 2] }", 5); - ts_document_set_input(doc, reader->input); + parser = ts_parser_make(config); + + reader = new SpyReader("some structured text", 5); }); after_each([&]() { - ts_document_free(doc); + ts_parser_free(parser); delete reader; }); - it("parses the input", [&]() { - AssertThat(string(ts_document_string(doc)), Equals( - "(value (object (string) (array (number) (number))))")); - }); - - it("reads the entire input", [&]() { - AssertThat(reader->strings_read, Equals(vector({ - "{ \"key\": [1, 2] }" - }))); - }); - - describe("modifying the end of the input", [&]() { + describe("when starting at the beginning of the input (edit is NULL)", [&]() { before_each([&]() { - size_t position(string("{ \"key\": [1, 2]").length()); - string inserted_text(", \"key2\": 4"); - - reader->content.insert(position, inserted_text); - ts_document_edit(doc, { position, 0, inserted_text.length() }); + ts_parser_start(parser, reader->input, nullptr); }); - it("updates the parse tree", [&]() { - AssertThat(string(ts_document_string(doc)), Equals( - "(value (object (string) (array (number) (number)) (string) (number)))")); + it("runs the lexer with the lex state corresponding to the initial state", [&]() { + lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1); + ts_parser_step(parser); + AssertThat(lex_fn_state_received, Equals(100)); }); - it("re-reads only the changed portion of the input", [&]() { - AssertThat(reader->strings_read.size(), Equals(2)); - AssertThat(reader->strings_read[1], Equals(", \"key2\": 4 }")); + describe("when the returned symbol indicates a shift action", [&]() { + before_each([&]() { + lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1); + }); + + it("advances to the state specified in the action", [&]() { + ts_parser_step(parser); + AssertThat(ts_stack_top_state(&parser->stack), Equals(12)); + }); + + it("continues parsing (returns NULL)", [&]() { + auto result = ts_parser_step(parser); + AssertThat(result, Equals((TSTree *)nullptr)); + }); + }); + + describe("when the returned symbol indicates an error", [&]() { + before_each([&]() { + lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym1, 5, 1); + }); + + it("ends the parse, returning an error tree", [&]() { + auto result = ts_parser_step(parser); + AssertThat(ts_tree_symbol(result), Equals(ts_builtin_sym_error)); + }); }); }); - - describe("modifying the beginning of the input", [&]() { - before_each([&]() { - size_t position(string("{ ").length()); - string inserted_text("\"key2\": 4, "); - - reader->content.insert(position, inserted_text); - ts_document_edit(doc, { position, 0, inserted_text.length() }); - }); - - it("2 updates the parse tree", [&]() { - AssertThat(string(ts_document_string(doc)), Equals( - "(value (object (string) (number) (string) (array (number) (number))))")); - }); - - it_skip("re-reads only the changed portion of the input", [&]() { - AssertThat(reader->strings_read.size(), Equals(2)); - AssertThat(reader->strings_read[1], Equals("\"key2\": 4, ")); - }); - }); - }); END_TEST diff --git a/spec/runtime/stack_spec.cc b/spec/runtime/stack_spec.cc index 63b7e675..d4ab8df5 100644 --- a/spec/runtime/stack_spec.cc +++ b/spec/runtime/stack_spec.cc @@ -1,5 +1,5 @@ #include "runtime/runtime_spec_helper.h" -#include "tree_sitter/parser/stack.h" +#include "tree_sitter/parser.h" START_TEST diff --git a/spec/runtime/state_machine_spec.cc b/spec/runtime/state_machine_spec.cc deleted file mode 100644 index d90b3148..00000000 --- a/spec/runtime/state_machine_spec.cc +++ /dev/null @@ -1,75 +0,0 @@ -#include "runtime/runtime_spec_helper.h" -#include "runtime/helpers/spy_reader.h" -#include "runtime/helpers/dummy_parser.h" -#include "tree_sitter/parser/state_machine.h" - -TSTree *lex_fn_node_to_return; -TSStateId lex_fn_state_received; -TSLexer *lex_fn_lexer_received; - -TSTree * fake_lex(TSLexer *lexer, TSStateId state_id) { - lex_fn_lexer_received = lexer; - lex_fn_state_received = state_id; - return lex_fn_node_to_return; -} - -START_TEST - -describe("LR Parsers", [&]() { - TSStateMachine *parser; - SpyReader *reader; - - before_each([&]() { - reader = new SpyReader("some structured text", 5); - parser = ts_state_machine_make(dummy_parser.symbol_count, - (const TSParseAction *)dummy_parser.parse_table, - dummy_parser.lex_states, - fake_lex, - dummy_parser.hidden_symbols); - }); - - after_each([&]() { - delete reader; - }); - - describe("when starting at the beginning of the input (edit is NULL)", [&]() { - before_each([&]() { - ts_state_machine_initialize(parser, reader->input, nullptr); - }); - - it("runs the lexer with the lex state corresponding to the initial state", [&]() { - lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1); - ts_state_machine_parse(parser, nullptr); - AssertThat(lex_fn_state_received, Equals(100)); - }); - - describe("when the returned symbol indicates a shift action", [&]() { - before_each([&]() { - lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1); - }); - - it("advances to the state specified in the action", [&]() { - ts_state_machine_parse(parser, nullptr); - AssertThat(ts_stack_top_state(&parser->stack), Equals(12)); - }); - - it("continues parsing (returns NULL)", [&]() { - auto result = ts_state_machine_parse(parser, nullptr); - AssertThat(result, Equals((TSTree *)nullptr)); - }); - }); - - describe("when the returned symbol indicates an error", [&]() { - before_each([&]() { - lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym1, 5, 1); - }); - - it("ends the parse, returning an error tree", [&]() { - auto result = ts_state_machine_parse(parser, nullptr); - AssertThat(ts_tree_symbol(result), Equals(ts_builtin_sym_error)); - }); - }); - }); -}); - -END_TEST diff --git a/src/runtime/document.c b/src/runtime/document.c index 24c76261..73579733 100644 --- a/src/runtime/document.c +++ b/src/runtime/document.c @@ -1,8 +1,9 @@ #include "tree_sitter/runtime.h" +#include "tree_sitter/parser.h" #include struct TSDocument { - TSParser parser; + TSParser *parser; const TSTree *tree; TSInput input; size_t error_count; @@ -13,14 +14,13 @@ TSDocument * ts_document_make() { } void ts_document_free(TSDocument *document) { - if (document->parser.free_fn) - document->parser.free_fn(document->parser.data); + ts_parser_free(document->parser); if (document->input.release_fn) document->input.release_fn(document->input.data); free(document); } -void ts_document_set_parser(TSDocument *document, TSParser parser) { +void ts_document_set_parser(TSDocument *document, TSParser *parser) { document->parser = parser; } @@ -29,20 +29,20 @@ const TSTree * ts_document_tree(const TSDocument *document) { } const char * ts_document_string(const TSDocument *document) { - return ts_tree_string(document->tree, document->parser.symbol_names); + return ts_tree_string(document->tree, ts_parser_config(document->parser).symbol_names); } void ts_document_set_input(TSDocument *document, TSInput input) { document->input = input; - document->tree = document->parser.parse_fn(document->parser.data, input, NULL); + document->tree = ts_parser_parse(document->parser, document->input, NULL); } void ts_document_edit(TSDocument *document, TSInputEdit edit) { - document->tree = document->parser.parse_fn(document->parser.data, document->input, &edit); + document->tree = ts_parser_parse(document->parser, document->input, &edit); } const char * ts_document_symbol_name(const TSDocument *document, const TSTree *tree) { - return document->parser.symbol_names[ts_tree_symbol(tree)]; + return ts_parser_config(document->parser).symbol_names[ts_tree_symbol(tree)]; } typedef struct { diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c new file mode 100644 index 00000000..0cf3e331 --- /dev/null +++ b/src/runtime/lexer.c @@ -0,0 +1,43 @@ +#include "tree_sitter/runtime.h" +#include "tree_sitter/parser.h" + +TSLexer ts_lexer_make() { + return (TSLexer) { + .chunk = NULL, + .debug = 0, + .chunk_start = 0, + .chunk_size = 0, + .position_in_chunk = 0, + .token_start_position = 0, + .token_end_position = 0, + .reached_end = 0 + }; +} + +int ts_lexer_advance(TSLexer *lexer) { + static const char *empty_chunk = ""; + if (lexer->position_in_chunk + 1 < lexer->chunk_size) { + lexer->position_in_chunk++; + } else { + lexer->chunk_start += lexer->chunk_size; + lexer->chunk = lexer->input.read_fn(lexer->input.data, &lexer->chunk_size); + lexer->position_in_chunk = 0; + if (lexer->reached_end) { + return 0; + } + if (lexer->chunk_size == 0) { + lexer->reached_end = 1; + lexer->chunk = empty_chunk; + } + } + return 1; +} + +TSTree * ts_lexer_build_node(TSLexer *lexer, TSSymbol symbol) { + size_t current_position = ts_lexer_position(lexer); + size_t size = current_position - lexer->token_start_position; + size_t offset = lexer->token_start_position - lexer->token_end_position; + lexer->token_end_position = current_position; + return ts_tree_make_leaf(symbol, size, offset); +} + diff --git a/src/runtime/parser.c b/src/runtime/parser.c new file mode 100644 index 00000000..0cbd1678 --- /dev/null +++ b/src/runtime/parser.c @@ -0,0 +1,262 @@ +#include +#include "tree_sitter/runtime.h" +#include "tree_sitter/parser.h" +#include "runtime/tree.h" + +/* + * Private + */ + +static const TSParseAction * actions_for_state(TSParserConfig config, TSStateId state) { + return config.parse_table + (state * config.symbol_count); +} + +static size_t breakdown_stack(TSParser *parser, TSInputEdit *edit) { + if (!edit) return 0; + + TSStack *stack = &parser->stack; + size_t position = 0; + + for (;;) { + TSTree *node = ts_stack_top_node(stack); + if (!node) break; + + position = ts_stack_right_position(stack); + size_t child_count; + TSTree **children = ts_tree_immediate_children(node, &child_count); + if (position <= edit->position && !children) break; + + stack->size--; + position -= ts_tree_total_size(node); + + for (size_t i = 0; i < child_count && position < edit->position; i++) { + TSTree *child = children[i]; + TSStateId state = ts_stack_top_state(stack); + TSStateId next_state = actions_for_state(parser->config, state)[ts_tree_symbol(child)].data.to_state; + ts_stack_push(stack, next_state, child); + ts_tree_retain(child); + position += ts_tree_total_size(child); + } + + ts_tree_release(node); + } + + return position; +} + +static TSSymbol * expected_symbols(TSParser *parser, size_t *count) { + *count = 0; + const TSParseAction *actions = actions_for_state(parser->config, ts_stack_top_state(&parser->stack)); + for (size_t i = 0; i < parser->config.symbol_count; i++) + if (actions[i].type != TSParseActionTypeError) + (*count)++; + + size_t n = 0; + TSSymbol *result = malloc(*count * sizeof(*result)); + for (TSSymbol i = 0; i < parser->config.symbol_count; i++) + if (actions[i].type != TSParseActionTypeError) + result[n++] = i; + + return result; +} + +/* + * Public + */ + +TSParser * ts_parser_make(TSParserConfig config) { + TSParser *result = malloc(sizeof(*result)); + *result = (TSParser) { + .lexer = ts_lexer_make(), + .stack = ts_stack_make(), + .debug = 0, + .config = config, + }; + return result; +} + +void ts_parser_free(TSParser *parser) { + if (parser->lookahead) ts_tree_release(parser->lookahead); + if (parser->next_lookahead) ts_tree_release(parser->next_lookahead); + ts_stack_delete(&parser->stack); + free(parser); +} + +void ts_parser_start(TSParser *parser, TSInput input, TSInputEdit *edit) { + if (!edit) ts_stack_shrink(&parser->stack, 0); + parser->lookahead = NULL; + parser->next_lookahead = NULL; + + size_t position = breakdown_stack(parser, edit); + input.seek_fn(input.data, position); + + parser->lexer = ts_lexer_make(); + parser->lexer.input = input; + ts_lexer_advance(&parser->lexer); +} + +void ts_parser_shift(TSParser *parser, TSStateId parse_state) { + if (parser->lookahead->is_extra) + parse_state = ts_stack_top_state(&parser->stack); + ts_stack_push(&parser->stack, parse_state, parser->lookahead); + parser->lookahead = parser->next_lookahead; + parser->next_lookahead = NULL; +} + +void ts_parser_shift_extra(TSParser *parser) { + parser->lookahead->is_extra = 1; + ts_parser_shift(parser, 0); +} + +void ts_parser_reduce(TSParser *parser, TSSymbol symbol, size_t child_count) { + parser->next_lookahead = parser->lookahead; + parser->lookahead = ts_stack_reduce( + &parser->stack, + symbol, + child_count, + parser->config.hidden_symbol_flags, 1); +} + +int ts_parser_reduce_extra(TSParser *parser, TSSymbol symbol) { + TSTree *top_node = ts_stack_top_node(&parser->stack); + if (top_node->symbol == symbol && !top_node->is_extra) { + ts_parser_reduce(parser, symbol, 1); + parser->lookahead->is_extra = 1; + return 1; + } else { + return 0; + } +} + +int ts_parser_handle_error(TSParser *parser) { + size_t count = 0; + const TSSymbol *inputs = expected_symbols(parser, &count); + TSTree *error = ts_tree_make_error(ts_lexer_lookahead_char(&parser->lexer), + count, + inputs, + 0, + 0); + + for (;;) { + ts_tree_release(parser->lookahead); + size_t position = ts_lexer_position(&parser->lexer); + parser->lookahead = parser->config.lex_fn(parser, ts_lex_state_error); + + int at_end = 0; + if (ts_lexer_position(&parser->lexer) == position) + at_end = !ts_lexer_advance(&parser->lexer); + + if (at_end || ts_tree_symbol(parser->lookahead) == ts_builtin_sym_end) { + ts_stack_push(&parser->stack, 0, error); + return 0; + } + + /* + * Unwind the stack, looking for a state in which this token + * may appear after an error. + */ + for (size_t j = 0; j < parser->stack.size; j++) { + size_t i = parser->stack.size - 1 - j; + TSStateId stack_state = parser->stack.entries[i].state; + TSParseAction action_on_error = actions_for_state(parser->config, stack_state)[ts_builtin_sym_error]; + if (action_on_error.type == TSParseActionTypeShift) { + TSStateId state_after_error = action_on_error.data.to_state; + if (actions_for_state(parser->config, state_after_error)[ts_tree_symbol(parser->lookahead)].type != TSParseActionTypeError) { + ts_stack_shrink(&parser->stack, i + 1); + ts_stack_push(&parser->stack, state_after_error, error); + return 1; + } + } + } + } +} + +TSTree * ts_parser_tree_root(TSParser *parser) { + TSStack *stack = &parser->stack; + TSTree *top_node = ts_stack_top_node(stack); + if (stack->size <= 1) + return top_node; + if (ts_tree_symbol(top_node) == ts_builtin_sym_error) + return top_node; + + size_t immediate_child_count; + TSTree **immedate_children = ts_tree_immediate_children(top_node, &immediate_child_count); + + stack->size--; + for (size_t i = 0; i < immediate_child_count; i++) { + TSTree *child = immedate_children[i]; + child->is_extra = 0; + ts_tree_retain(child); + TSStateId state = ts_stack_top_state(stack); + TSStateId next_state = actions_for_state(parser->config, state)[ts_tree_symbol(child)].data.to_state; + ts_stack_push(stack, next_state, child); + } + + TSTree *new_node = ts_stack_reduce(stack, + top_node->symbol, + stack->size, + parser->config.hidden_symbol_flags, + 0); + ts_tree_release(top_node); + return new_node; +} + +TSParseAction ts_parser_next_action(TSParser *parser) { + TSStateId state = ts_stack_top_state(&parser->stack); + if (!parser->lookahead) + parser->lookahead = parser->config.lex_fn(parser, parser->config.lex_states[state]); + return actions_for_state(parser->config, state)[ts_tree_symbol(parser->lookahead)]; +} + +#define DEBUG_PARSE(...) \ +if (parser->debug) { fprintf(stderr, "\n" __VA_ARGS__); } + +TSTree * ts_parser_step(TSParser *parser) { + TSParseAction action = ts_parser_next_action(parser); + DEBUG_PARSE("LOOKAHEAD %s", parser->config.symbol_names[ts_tree_symbol(parser->lookahead)]); + switch (action.type) { + case TSParseActionTypeShift: + DEBUG_PARSE("SHIFT %d", action.data.to_state); + ts_parser_shift(parser, action.data.to_state); + return NULL; + case TSParseActionTypeShiftExtra: + DEBUG_PARSE("SHIFT EXTRA"); + ts_parser_shift_extra(parser); + return NULL; + case TSParseActionTypeReduce: + DEBUG_PARSE("REDUCE %s %d", parser->config.symbol_names[action.data.symbol], action.data.child_count); + ts_parser_reduce(parser, action.data.symbol, action.data.child_count); + return NULL; + case TSParseActionTypeReduceExtra: + if (!ts_parser_reduce_extra(parser, action.data.symbol)) + goto error; + DEBUG_PARSE("REDUCE EXTRA"); + return NULL; + case TSParseActionTypeAccept: + DEBUG_PARSE("ACCEPT"); + return ts_parser_tree_root(parser); + case TSParseActionTypeError: + goto error; + default: + return NULL; + } +error: + DEBUG_PARSE("ERROR"); + if (!ts_parser_handle_error(parser)) + return ts_parser_tree_root(parser); + else + return NULL; +} + +const TSTree * ts_parser_parse(TSParser *parser, TSInput input, TSInputEdit *edit) { + ts_parser_start(parser, input, edit); + + for (;;) { + const TSTree *tree = ts_parser_step(parser); + if (tree) return tree; + } +} + +TSParserConfig ts_parser_config(TSParser *parser) { + return parser->config; +} diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 6c4cbed0..f2e9e774 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -1,5 +1,5 @@ #include "tree_sitter/runtime.h" -#include "tree_sitter/parser/stack.h" +#include "tree_sitter/parser.h" #include "runtime/tree.h" #include diff --git a/src/runtime/state_machine.c b/src/runtime/state_machine.c deleted file mode 100644 index 1f9f7ed9..00000000 --- a/src/runtime/state_machine.c +++ /dev/null @@ -1,263 +0,0 @@ -#include "tree_sitter/parser/state_machine.h" -#include "runtime/tree.h" - -/* - * Private - */ - -static const TSParseAction * actions_for_state(TSStateMachine *machine, TSStateId state) { - return machine->config.parse_table + (state * machine->config.symbol_count); -} - -void shift(TSStateMachine *machine, TSStateId parse_state) { - if (machine->lookahead->is_extra) - parse_state = ts_stack_top_state(&machine->stack); - ts_stack_push(&machine->stack, parse_state, machine->lookahead); - machine->lookahead = machine->next_lookahead; - machine->next_lookahead = NULL; -} - -void shift_extra(TSStateMachine *machine) { - machine->lookahead->is_extra = 1; - shift(machine, 0); -} - -void reduce(TSStateMachine *machine, TSSymbol symbol, size_t child_count) { - machine->next_lookahead = machine->lookahead; - machine->lookahead = ts_stack_reduce( - &machine->stack, - symbol, - child_count, - machine->config.hidden_symbol_flags, 1); -} - -int reduce_extra(TSStateMachine *machine, TSSymbol symbol) { - TSTree *top_node = ts_stack_top_node(&machine->stack); - if (top_node->symbol == symbol && !top_node->is_extra) { - reduce(machine, symbol, 1); - machine->lookahead->is_extra = 1; - return 1; - } else { - return 0; - } -} - -static size_t breakdown_stack(TSStateMachine *machine, TSInputEdit *edit) { - if (!edit) return 0; - - TSStack *stack = &machine->stack; - size_t position = 0; - - for (;;) { - TSTree *node = ts_stack_top_node(stack); - if (!node) break; - - position = ts_stack_right_position(stack); - size_t child_count; - TSTree **children = ts_tree_immediate_children(node, &child_count); - if (position <= edit->position && !children) break; - - stack->size--; - position -= ts_tree_total_size(node); - - for (size_t i = 0; i < child_count && position < edit->position; i++) { - TSTree *child = children[i]; - TSStateId state = ts_stack_top_state(stack); - TSStateId next_state = actions_for_state(machine, state)[ts_tree_symbol(child)].data.to_state; - ts_stack_push(stack, next_state, child); - ts_tree_retain(child); - position += ts_tree_total_size(child); - } - - ts_tree_release(node); - } - - return position; -} - -TSSymbol * expected_symbols(TSStateMachine *machine, size_t *count) { - *count = 0; - const TSParseAction *actions = actions_for_state(machine, ts_stack_top_state(&machine->stack)); - for (size_t i = 0; i < machine->config.symbol_count; i++) - if (actions[i].type != TSParseActionTypeError) - ++(*count); - - size_t n = 0; - TSSymbol *result = malloc(*count * sizeof(*result)); - for (TSSymbol i = 0; i < machine->config.symbol_count; i++) - if (actions[i].type != TSParseActionTypeError) - result[n++] = i; - - return result; -} - -int handle_error(TSStateMachine *machine) { - size_t count = 0; - const TSSymbol *inputs = expected_symbols(machine, &count); - TSTree *error = ts_tree_make_error(ts_lexer_lookahead_char(&machine->lexer), - count, - inputs, - 0, - 0); - - for (;;) { - ts_tree_release(machine->lookahead); - size_t position = ts_lexer_position(&machine->lexer); - machine->lookahead = machine->config.lex_fn(&machine->lexer, ts_lex_state_error); - - int at_end = 0; - if (ts_lexer_position(&machine->lexer) == position) - at_end = !ts_lexer_advance(&machine->lexer); - - if (at_end || ts_tree_symbol(machine->lookahead) == ts_builtin_sym_end) { - ts_stack_push(&machine->stack, 0, error); - return 0; - } - - /* - * Unwind the stack, looking for a state in which this token - * may appear after an error. - */ - for (size_t j = 0; j < machine->stack.size; j++) { - size_t i = machine->stack.size - 1 - j; - TSStateId stack_state = machine->stack.entries[i].state; - TSParseAction action_on_error = actions_for_state(machine, stack_state)[ts_builtin_sym_error]; - if (action_on_error.type == TSParseActionTypeShift) { - TSStateId state_after_error = action_on_error.data.to_state; - if (actions_for_state(machine, state_after_error)[ts_tree_symbol(machine->lookahead)].type != TSParseActionTypeError) { - ts_stack_shrink(&machine->stack, i + 1); - ts_stack_push(&machine->stack, state_after_error, error); - return 1; - } - } - } - } -} - -TSTree * get_tree_root(TSStateMachine *machine) { - TSStack *stack = &machine->stack; - TSTree *top_node = ts_stack_top_node(stack); - if (stack->size <= 1) - return top_node; - if (ts_tree_symbol(top_node) == ts_builtin_sym_error) - return top_node; - - size_t immediate_child_count; - TSTree **immedate_children = ts_tree_immediate_children(top_node, &immediate_child_count); - - stack->size--; - for (size_t i = 0; i < immediate_child_count; i++) { - TSTree *child = immedate_children[i]; - child->is_extra = 0; - ts_tree_retain(child); - TSStateId state = ts_stack_top_state(stack); - TSStateId next_state = actions_for_state(machine, state)[ts_tree_symbol(child)].data.to_state; - ts_stack_push(stack, next_state, child); - } - - TSTree *new_node = ts_stack_reduce(stack, - top_node->symbol, - stack->size, - machine->config.hidden_symbol_flags, - 0); - ts_tree_release(top_node); - return new_node; -} - -TSParseAction get_next_action(TSStateMachine *machine) { - TSStateId state = ts_stack_top_state(&machine->stack); - if (!machine->lookahead) - machine->lookahead = machine->config.lex_fn(&machine->lexer, machine->config.lex_states[state]); - return actions_for_state(machine, state)[ts_tree_symbol(machine->lookahead)]; -} - -/* - * Public API - */ - -TSStateMachine * ts_state_machine_make(size_t symbol_count, - const TSParseAction *parse_table, - const TSStateId *lex_states, - TSTree * (* lex_fn)(TSLexer *, TSStateId), - const int *hidden_symbol_flags) { - TSStateMachine *result = malloc(sizeof(TSStateMachine)); - *result = (TSStateMachine) { - .lexer = ts_lexer_make(), - .stack = ts_stack_make(), - .config = { - .symbol_count = symbol_count, - .parse_table = parse_table, - .lex_states = lex_states, - .lex_fn = lex_fn, - .hidden_symbol_flags = hidden_symbol_flags, - }, - }; - return result; -} - -void ts_state_machine_free(void *data) { - TSStateMachine *machine = (TSStateMachine *)data; - if (machine->lookahead) ts_tree_release(machine->lookahead); - if (machine->next_lookahead) ts_tree_release(machine->next_lookahead); - ts_stack_delete(&machine->stack); - free(machine); -} - -void ts_state_machine_initialize(TSStateMachine *machine, TSInput input, TSInputEdit *edit) { - if (!edit) ts_stack_shrink(&machine->stack, 0); - machine->lookahead = NULL; - machine->next_lookahead = NULL; - - size_t position = breakdown_stack(machine, edit); - input.seek_fn(input.data, position); - - machine->lexer = ts_lexer_make(); - machine->lexer.input = input; - ts_lexer_advance(&machine->lexer); -} - -// #define TS_DEBUG_PARSE - -#ifdef TS_DEBUG_PARSE -#include -#define DEBUG_PARSE(...) fprintf(stderr, "\n" __VA_ARGS__) -#else -#define DEBUG_PARSE(...) -#endif - -TSTree * ts_state_machine_parse(TSStateMachine *machine, const char **symbol_names) { - TSParseAction action = get_next_action(machine); - DEBUG_PARSE("LOOKAHEAD %s", symbol_names[ts_tree_symbol(machine->lookahead)]); - switch (action.type) { - case TSParseActionTypeShift: - DEBUG_PARSE("SHIFT %d", action.data.to_state); - shift(machine, action.data.to_state); - return NULL; - case TSParseActionTypeShiftExtra: - DEBUG_PARSE("SHIFT EXTRA"); - shift_extra(machine); - return NULL; - case TSParseActionTypeReduce: - DEBUG_PARSE("REDUCE %s %d", symbol_names[action.data.symbol], action.data.child_count); - reduce(machine, action.data.symbol, action.data.child_count); - return NULL; - case TSParseActionTypeReduceExtra: - if (!reduce_extra(machine, action.data.symbol)) - goto error; - DEBUG_PARSE("REDUCE EXTRA"); - return NULL; - case TSParseActionTypeAccept: - DEBUG_PARSE("ACCEPT"); - return get_tree_root(machine); - case TSParseActionTypeError: - goto error; - default: - return NULL; - } -error: - DEBUG_PARSE("ERROR"); - if (handle_error(machine)) - return NULL; - else - return get_tree_root(machine); -}