traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Combine TSParser and TSStateMachine objects

Browse source 
My original thought was to decouple the runtime from
the LR parser generator by making TSParser a generic
interface that LR parsers implement.

I think this was more trouble than it was worth.
This commit is contained in:
Max Brunsfeld 2014-07-10 13:14:52 -07:00
parent 1c7d2d2d03
commit 9da7663e99
18 changed files with 586 additions and 645 deletions
Download patch file Download diff file Expand all files Collapse all files

78
spec/runtime/document_spec.cc Normal file
View file

@ -0,0 +1,78 @@
#include "runtime/runtime_spec_helper.h"
#include "runtime/helpers/spy_reader.h"
extern "C" TSParser * ts_parser_json();
START_TEST
describe("incremental parsing", [&]() {
TSDocument *doc;
SpyReader *reader;
before_each([&]() {
doc = ts_document_make();
ts_document_set_parser(doc, ts_parser_json());
reader = new SpyReader("{ \"key\": [1, 2] }", 5);
ts_document_set_input(doc, reader->input);
});
after_each([&]() {
ts_document_free(doc);
delete reader;
});
it("parses the input", [&]() {
AssertThat(string(ts_document_string(doc)), Equals(
"(value (object (string) (array (number) (number))))"));
});
it("reads the entire input", [&]() {
AssertThat(reader->strings_read, Equals(vector<string>({
"{ \"key\": [1, 2] }"
})));
});
describe("modifying the end of the input", [&]() {
before_each([&]() {
size_t position(string("{ \"key\": [1, 2]").length());
string inserted_text(", \"key2\": 4");
reader->content.insert(position, inserted_text);
ts_document_edit(doc, { position, 0, inserted_text.length() });
});
it("updates the parse tree", [&]() {
AssertThat(string(ts_document_string(doc)), Equals(
"(value (object (string) (array (number) (number)) (string) (number)))"));
});
it("re-reads only the changed portion of the input", [&]() {
AssertThat(reader->strings_read.size(), Equals<size_t>(2));
AssertThat(reader->strings_read[1], Equals(", \"key2\": 4 }"));
});
});
describe("modifying the beginning of the input", [&]() {
before_each([&]() {
size_t position(string("{ ").length());
string inserted_text("\"key2\": 4, ");
reader->content.insert(position, inserted_text);
ts_document_edit(doc, { position, 0, inserted_text.length() });
});
it("2 updates the parse tree", [&]() {
AssertThat(string(ts_document_string(doc)), Equals(
"(value (object (string) (number) (string) (array (number) (number))))"));
});
it_skip("re-reads only the changed portion of the input", [&]() {
AssertThat(reader->strings_read.size(), Equals<size_t>(2));
AssertThat(reader->strings_read[1], Equals("\"key2\": 4, "));
});
});
});
END_TEST

8
spec/runtime/helpers/dummy_parser.c
View file

@ -1,4 +1,5 @@
#include "runtime/helpers/dummy_parser.h"
#include "tree_sitter/parser.h"
const TSParseAction parse_table[3][5] = {
[0] = {
@ -29,10 +30,9 @@ const int hidden_symbols[5] = {
[dummy_sym3] = 1,
};
struct test_parser dummy_parser = {
.state_count = 3,
TSParserConfig dummy_parser = {
.symbol_count = 5,
.parse_table = (const TSParseAction **)parse_table,
.parse_table = (const TSParseAction *)parse_table,
.lex_states = lex_states,
.hidden_symbols = hidden_symbols,
.hidden_symbol_flags = hidden_symbols,
};

13
spec/runtime/helpers/dummy_parser.h
View file

@ -5,7 +5,8 @@
extern "C" {
#endif
#include "tree_sitter/parser/state_machine.h"
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser.h"
enum {
dummy_sym1 = 2,
@ -13,15 +14,7 @@ enum {
dummy_sym3 = 4,
};
struct test_parser {
size_t state_count;
size_t symbol_count;
const TSParseAction **parse_table;
const TSStateId *lex_states;
const int *hidden_symbols;
};
extern struct test_parser dummy_parser;
extern TSParserConfig dummy_parser;
#ifdef __cplusplus
}

10
spec/runtime/language_specs.cc
View file

@ -1,10 +1,10 @@
#include "runtime/runtime_spec_helper.h"
#include "runtime/helpers/read_test_entries.h"
extern "C" TSParser ts_parser_javascript();
extern "C" TSParser ts_parser_json();
extern "C" TSParser ts_parser_arithmetic();
extern "C" TSParser ts_parser_golang();
extern "C" TSParser * ts_parser_javascript();
extern "C" TSParser * ts_parser_json();
extern "C" TSParser * ts_parser_arithmetic();
extern "C" TSParser * ts_parser_golang();
START_TEST
@ -19,7 +19,7 @@ describe("Languages", [&]() {
ts_document_free(doc);
});
auto run_tests_for_language = [&](string language, TSParser (parser_constructor)()) {
auto run_tests_for_language = [&](string language, TSParser * (parser_constructor)()) {
describe(language.c_str(), [&]() {
before_each([&]() {
ts_document_set_parser(doc, parser_constructor());

2
spec/runtime/node_position_spec.cc
View file

@ -1,6 +1,6 @@
#include "runtime/runtime_spec_helper.h"
extern "C" TSParser ts_parser_json();
extern "C" TSParser * ts_parser_json();
START_TEST

102
spec/runtime/parser_spec.cc
View file

@ -1,78 +1,76 @@
#include "runtime/runtime_spec_helper.h"
#include "runtime/helpers/spy_reader.h"
#include "runtime/helpers/dummy_parser.h"
#include "tree_sitter/parser.h"
extern "C" TSParser ts_parser_json();
TSTree *lex_fn_node_to_return;
TSStateId lex_fn_state_received;
TSParser *lex_fn_parser_received;
TSTree * fake_lex(TSParser *parser, TSStateId state_id) {
lex_fn_parser_received = parser;
lex_fn_state_received = state_id;
return lex_fn_node_to_return;
}
START_TEST
describe("incremental parsing", [&]() {
TSDocument *doc;
describe("LR Parsers", [&]() {
TSParser *parser;
SpyReader *reader;
before_each([&]() {
doc = ts_document_make();
ts_document_set_parser(doc, ts_parser_json());
TSParserConfig config = dummy_parser;
config.lex_fn = fake_lex;
reader = new SpyReader("{ \"key\": [1, 2] }", 5);
ts_document_set_input(doc, reader->input);
parser = ts_parser_make(config);
reader = new SpyReader("some structured text", 5);
});
after_each([&]() {
ts_document_free(doc);
ts_parser_free(parser);
delete reader;
});
it("parses the input", [&]() {
AssertThat(string(ts_document_string(doc)), Equals(
"(value (object (string) (array (number) (number))))"));
});
it("reads the entire input", [&]() {
AssertThat(reader->strings_read, Equals(vector<string>({
"{ \"key\": [1, 2] }"
})));
});
describe("modifying the end of the input", [&]() {
describe("when starting at the beginning of the input (edit is NULL)", [&]() {
before_each([&]() {
size_t position(string("{ \"key\": [1, 2]").length());
string inserted_text(", \"key2\": 4");
reader->content.insert(position, inserted_text);
ts_document_edit(doc, { position, 0, inserted_text.length() });
ts_parser_start(parser, reader->input, nullptr);
});
it("updates the parse tree", [&]() {
AssertThat(string(ts_document_string(doc)), Equals(
"(value (object (string) (array (number) (number)) (string) (number)))"));
it("runs the lexer with the lex state corresponding to the initial state", [&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1);
ts_parser_step(parser);
AssertThat(lex_fn_state_received, Equals(100));
});
it("re-reads only the changed portion of the input", [&]() {
AssertThat(reader->strings_read.size(), Equals<size_t>(2));
AssertThat(reader->strings_read[1], Equals(", \"key2\": 4 }"));
describe("when the returned symbol indicates a shift action", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1);
});
it("advances to the state specified in the action", [&]() {
ts_parser_step(parser);
AssertThat(ts_stack_top_state(&parser->stack), Equals(12));
});
it("continues parsing (returns NULL)", [&]() {
auto result = ts_parser_step(parser);
AssertThat(result, Equals((TSTree *)nullptr));
});
});
describe("when the returned symbol indicates an error", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym1, 5, 1);
});
it("ends the parse, returning an error tree", [&]() {
auto result = ts_parser_step(parser);
AssertThat(ts_tree_symbol(result), Equals(ts_builtin_sym_error));
});
});
});
describe("modifying the beginning of the input", [&]() {
before_each([&]() {
size_t position(string("{ ").length());
string inserted_text("\"key2\": 4, ");
reader->content.insert(position, inserted_text);
ts_document_edit(doc, { position, 0, inserted_text.length() });
});
it("2 updates the parse tree", [&]() {
AssertThat(string(ts_document_string(doc)), Equals(
"(value (object (string) (number) (string) (array (number) (number))))"));
});
it_skip("re-reads only the changed portion of the input", [&]() {
AssertThat(reader->strings_read.size(), Equals<size_t>(2));
AssertThat(reader->strings_read[1], Equals("\"key2\": 4, "));
});
});
});
END_TEST

2
spec/runtime/stack_spec.cc
View file

@ -1,5 +1,5 @@
#include "runtime/runtime_spec_helper.h"
#include "tree_sitter/parser/stack.h"
#include "tree_sitter/parser.h"
START_TEST

75
spec/runtime/state_machine_spec.cc
View file

@ -1,75 +0,0 @@
#include "runtime/runtime_spec_helper.h"
#include "runtime/helpers/spy_reader.h"
#include "runtime/helpers/dummy_parser.h"
#include "tree_sitter/parser/state_machine.h"
TSTree *lex_fn_node_to_return;
TSStateId lex_fn_state_received;
TSLexer *lex_fn_lexer_received;
TSTree * fake_lex(TSLexer *lexer, TSStateId state_id) {
lex_fn_lexer_received = lexer;
lex_fn_state_received = state_id;
return lex_fn_node_to_return;
}
START_TEST
describe("LR Parsers", [&]() {
TSStateMachine *parser;
SpyReader *reader;
before_each([&]() {
reader = new SpyReader("some structured text", 5);
parser = ts_state_machine_make(dummy_parser.symbol_count,
(const TSParseAction *)dummy_parser.parse_table,
dummy_parser.lex_states,
fake_lex,
dummy_parser.hidden_symbols);
});
after_each([&]() {
delete reader;
});
describe("when starting at the beginning of the input (edit is NULL)", [&]() {
before_each([&]() {
ts_state_machine_initialize(parser, reader->input, nullptr);
});
it("runs the lexer with the lex state corresponding to the initial state", [&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1);
ts_state_machine_parse(parser, nullptr);
AssertThat(lex_fn_state_received, Equals(100));
});
describe("when the returned symbol indicates a shift action", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym2, 5, 1);
});
it("advances to the state specified in the action", [&]() {
ts_state_machine_parse(parser, nullptr);
AssertThat(ts_stack_top_state(&parser->stack), Equals(12));
});
it("continues parsing (returns NULL)", [&]() {
auto result = ts_state_machine_parse(parser, nullptr);
AssertThat(result, Equals((TSTree *)nullptr));
});
});
describe("when the returned symbol indicates an error", [&]() {
before_each([&]() {
lex_fn_node_to_return = ts_tree_make_leaf(dummy_sym1, 5, 1);
});
it("ends the parse, returning an error tree", [&]() {
auto result = ts_state_machine_parse(parser, nullptr);
AssertThat(ts_tree_symbol(result), Equals(ts_builtin_sym_error));
});
});
});
});
END_TEST