traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use table-driven tests for specific parsers

Browse source
This commit is contained in:
Karen Wang and Max Brunsfeld 2014-03-20 18:15:38 -07:00 • committed by Max Brunsfeld
parent 5869c1ea18
commit f248ece3aa
12 changed files with 355 additions and 167 deletions
Download patch file Download diff file Expand all files Collapse all files

2
include/tree_sitter/parser.h
View file

@ -277,7 +277,7 @@ static size_t ts_lr_parser_breakdown_stack(ts_lr_parser *parser, ts_input_edit *
ts_tree *node;
size_t position = 0;
size_t child_count = 0;
for (;;) {
node = ts_stack_top_node(stack);
if (!node) break;

65
spec/runtime/arithmetic_spec.cc
View file

@ -1,65 +0,0 @@
#include "runtime_spec_helper.h"
extern "C" ts_parser ts_parser_arithmetic();
START_TEST
describe("arithmetic", []() {
ts_document *doc;
before_each([&]() {
doc = ts_document_make();
ts_document_set_parser(doc, ts_parser_arithmetic());
});
after_each([&]() {
ts_document_free(doc);
});
it("parses variables", [&]() {
ts_document_set_input_string(doc, "x");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable))))"));
});
it("parses numbers", [&]() {
ts_document_set_input_string(doc, "5");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (number))))"));
});
it("parses products of variables", [&]() {
ts_document_set_input_string(doc, "x + y");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable))) (plus) (term (factor (variable))))"));
ts_document_set_input_string(doc, "x * y");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable)) (times) (factor (variable))))"));
});
it("parses complex trees", [&]() {
ts_document_set_input_string(doc, "x * y + z * a");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable)) (times) (factor (variable))) (plus) (term (factor (variable)) (times) (factor (variable))))"));
ts_document_set_input_string(doc, "x * (y + z)");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable)) (times) (factor (expression (term (factor (variable))) (plus) (term (factor (variable)))))))"));
});
describe("error recovery", [&]() {
it("recovers from errors at the top level", [&]() {
ts_document_set_input_string(doc, "x * * y");
AssertThat(string(ts_document_string(doc)), Equals("(ERROR)"));
});
it("recovers from errors in parenthesized expressions", [&]() {
ts_document_set_input_string(doc, "x + (y * + z) * 5");
AssertThat(string(ts_document_string(doc)), Equals(
"(expression (term (factor (variable))) (plus) (term (factor (ERROR)) (times) (factor (number))))"));
});
});
});
END_TEST

99
spec/runtime/helpers/read_test_entries.cc Normal file
View file

@ -0,0 +1,99 @@
#include "helpers/read_test_entries.h"
#include <string>
#include <fstream>
#include <streambuf>
#include <regex>
#include <dirent.h>
using std::string;
using std::vector;
using std::ifstream;
using std::istreambuf_iterator;
using std::regex;
using std::regex_search;
using std::regex_replace;
static string trim_output(const string &input) {
string result(input);
result = regex_replace(result, regex("^[\\s]+"), "");
result = regex_replace(result, regex("[\\s]+$"), "");
result = regex_replace(result, regex("\\)\\s+\\)"), "))");
result = regex_replace(result, regex("\\s+"), " ");
return result;
}
static vector<TestEntry> get_test_entries_from_string(string content) {
regex header_pattern("===+\n([\\w ]+)\n===+");
regex separator_pattern("---+");
vector<string> descriptions;
vector<string> bodies;
for (;;) {
std::smatch matches;
regex_search(content, matches, header_pattern);
if (matches.empty()) break;
string description = matches[1].str();
descriptions.push_back(description);
if (!bodies.empty())
bodies.back().erase(matches.position());
content.erase(0, matches.position() + matches[0].length());
bodies.push_back(content);
}
vector<TestEntry> result;
for (size_t i = 0; i < descriptions.size(); i++) {
string body = bodies[i];
std::smatch matches;
regex_search(body, matches, separator_pattern);
result.push_back({
.description = descriptions[i],
.input = body.substr(0, matches.position()),
.tree_string = trim_output(body.substr(matches.position() + matches[0].length()))
});
}
return result;
}
static vector<string> list_directory(string dir_name) {
vector<string> result;
DIR *dir = opendir(dir_name.c_str());
if (!dir) {
printf("\nTest error - no such directory '%s'", dir_name.c_str());
return result;
}
struct dirent *dir_entry;
while ((dir_entry = readdir(dir))) {
string name(dir_entry->d_name);
if (name != "." && name != "..")
result.push_back(dir_name + "/" + name);
}
return result;
}
static string src_dir() {
const char * dir = getenv("TREESITTER_DIR");
if (!dir) dir = getenv("PWD");
return dir;
}
vector<TestEntry> test_entries_for_language(string language) {
vector<TestEntry> result;
string language_dir = src_dir() + "/spec/runtime/languages/" + language;
vector<string> filenames = list_directory(language_dir);
for (string &filename : filenames) {
ifstream file(filename);
std::string content((istreambuf_iterator<char>(file)), istreambuf_iterator<char>());
for (TestEntry entry : get_test_entries_from_string(content)) {
result.push_back(entry);
}
}
return result;
}

15
spec/runtime/helpers/read_test_entries.h Normal file
View file

@ -0,0 +1,15 @@
#ifndef HELPERS_READ_TEST_ENTRIES_H_
#define HELPERS_READ_TEST_ENTRIES_H_
#include <string>
#include <vector>
struct TestEntry {
std::string description;
std::string input;
std::string tree_string;
};
std::vector<TestEntry> test_entries_for_language(std::string language);
#endif

101
spec/runtime/json_spec.cc
View file

@ -1,101 +0,0 @@
#include "runtime_spec_helper.h"
extern "C" ts_parser ts_parser_json();
START_TEST
describe("json", []() {
ts_document *doc;
before_each([&]() {
doc = ts_document_make();
ts_document_set_parser(doc, ts_parser_json());
});
after_each([&]() {
ts_document_free(doc);
});
it("parses strings", [&]() {
ts_document_set_input_string(doc, "\"\"");
AssertThat(string(ts_document_string(doc)), Equals("(value (string))"));
ts_document_set_input_string(doc, "\"simple-string\"");
AssertThat(string(ts_document_string(doc)), Equals("(value (string))"));
ts_document_set_input_string(doc, "\"this is a \\\"string\\\" within a string\"");
AssertThat(string(ts_document_string(doc)), Equals("(value (string))"));
});
it("parses objects", [&]() {
ts_document_set_input_string(doc, "{}");
AssertThat(string(ts_document_string(doc)), Equals("(value (object))"));
ts_document_set_input_string(doc, "{ \"key1\": 1 }");
AssertThat(string(ts_document_string(doc)), Equals("(value (object (string) (value (number))))"));
ts_document_set_input_string(doc, "{\"key1\": 1, \"key2\": 2 }");
AssertThat(string(ts_document_string(doc)), Equals("(value (object (string) (value (number)) (string) (value (number))))"));
});
it("parses arrays", [&]() {
ts_document_set_input_string(doc, "[]");
AssertThat(string(ts_document_string(doc)), Equals("(value (array))"));
ts_document_set_input_string(doc, "[5]");
AssertThat(string(ts_document_string(doc)), Equals("(value (array (value (number))))"));
ts_document_set_input_string(doc, "[1, 2, 3]");
AssertThat(string(ts_document_string(doc)), Equals("(value (array (value (number)) (value (number)) (value (number))))"));
});
describe("tracking the positions of AST nodes", [&]() {
it("records the widths and offsets of nodes", [&]() {
ts_document_set_input_string(doc, " [12, 5]");
const ts_tree *tree = ts_document_tree(doc);
// TODO - make this better
const ts_tree *array = ts_tree_children(tree, NULL)[0];
const ts_tree *number1 = ts_tree_children(ts_tree_children(array, NULL)[1], NULL)[0];
const ts_tree *number2 = ts_tree_children(ts_tree_children(ts_tree_children(array, NULL)[2], NULL)[1], NULL)[0];
AssertThat(ts_document_symbol_name(doc, array), Equals("array"));
AssertThat(ts_document_symbol_name(doc, number1), Equals("number"));
AssertThat(ts_document_symbol_name(doc, number2), Equals("number"));
AssertThat(number1->offset, Equals<size_t>(0));
AssertThat(number1->size, Equals<size_t>(2));
AssertThat(number2->offset, Equals<size_t>(1));
AssertThat(number2->size, Equals<size_t>(1));
AssertThat(array->offset, Equals<size_t>(2));
AssertThat(array->size, Equals<size_t>(7));
AssertThat(tree->offset, Equals<size_t>(2));
AssertThat(tree->size, Equals<size_t>(7));
});
});
describe("errors", [&]() {
it("reports errors in the top-level node", [&]() {
ts_document_set_input_string(doc, "[");
AssertThat(string(ts_document_string(doc)), Equals("(ERROR)"));
});
it("reports errors inside of arrays and objects", [&]() {
ts_document_set_input_string(doc, "{ \"key1\": 1, 5 }");
AssertThat(string(ts_document_string(doc)), Equals("(value (object (string) (value (number)) (ERROR)))"));
ts_document_set_input_string(doc, "[1,,2]");
AssertThat(string(ts_document_string(doc)), Equals("(value (array (value (number)) (ERROR) (value (number))))"));
});
it("reports errors in nested objects", [&]() {
ts_document_set_input_string(doc, "{ \"key1\": { \"key2\": 1, 2 }, [, \"key3\": 3 }");
AssertThat(string(ts_document_string(doc)), Equals("(value (object (string) (value (object (string) (value (number)) (ERROR))) (ERROR) (string) (value (number))))"));
});
});
});
END_TEST

46
spec/runtime/language_specs.cc Normal file
View file

@ -0,0 +1,46 @@
#include "runtime_spec_helper.h"
#include "helpers/read_test_entries.h"
extern "C" ts_parser ts_parser_json();
extern "C" ts_parser ts_parser_arithmetic();
START_TEST
describe("Languages", [&]() {
ts_document *doc;
before_each([&]() {
doc = ts_document_make();
});
after_each([&]() {
ts_document_free(doc);
});
auto run_tests_for_language = [&](string language) {
for (auto &entry : test_entries_for_language(language)) {
it(entry.description.c_str(), [&]() {
ts_document_set_input_string(doc, entry.input.c_str());
AssertThat(ts_document_string(doc), Equals(entry.tree_string.c_str()));
});
}
};
describe("json", [&]() {
before_each([&]() {
ts_document_set_parser(doc, ts_parser_json());
});
run_tests_for_language("json");
});
describe("arithmetic", [&]() {
before_each([&]() {
ts_document_set_parser(doc, ts_parser_arithmetic());
});
run_tests_for_language("arithmetic");
});
});
END_TEST

16
spec/runtime/languages/arithmetic/errors.txt Normal file
View file

@ -0,0 +1,16 @@
=====================================================
recovers from errors at the top level
=====================================================
x * * y
---
(ERROR)
=====================================================
recovers from errors inside parenthesized expressions
=====================================================
x + (y * + z) * 5
---
(expression
(term (factor (variable)))
(plus)
(term (factor (ERROR)) (times) (factor (number))))

54
spec/runtime/languages/arithmetic/main.txt Normal file
View file

@ -0,0 +1,54 @@
====================
parses numbers
===================
5
---
(expression (term (factor (number))))
===================
parses variables
===================
x
---
(expression (term (factor (variable))))
===================
parses products
===================
x * x
---
(expression (term
(factor (variable))
(times)
(factor (variable))))
===================
parses sums
===================
x + x
---
(expression
(term (factor (variable)))
(plus)
(term (factor (variable))))
====================
parses complex trees
====================
x * y + z * a
---
(expression
(term (factor (variable)) (times) (factor (variable)))
(plus)
(term (factor (variable)) (times) (factor (variable))))
=================================
handles parenthesized expressions
=================================
x * (y + z)
---
(expression
(term (factor (variable))
(times)
(factor (expression (term (factor (variable))) (plus) (term (factor (variable)))))))

33
spec/runtime/languages/json/errors.txt Normal file
View file

@ -0,0 +1,33 @@
==============================
recovers from top-level errors
==============================
[}
---
(ERROR)
==================================
recovers from errors inside arrays
==================================
[1,,2]
---
(value (array
(value (number))
(ERROR)
(value (number))))
==================================
recovers from errors inside objects
==================================
{ "key1": 1, 5 }
---
(value (object (string) (value (number)) (ERROR)))
==========================================
recovers from errors inside nested objects
==========================================
{ "key1": { "key2": 1, 2 }, [, "key3": 3 }
---
(value (object
(string) (value (object (string) (value (number)) (ERROR)))
(ERROR)
(string) (value (number))))

44
spec/runtime/languages/json/main.txt Normal file
View file

@ -0,0 +1,44 @@
====================
parses empty objects
====================
{}
---
(value (object))
===================
parses empty arrays
===================
[]
---
(value (array))
===================
parses arrays
===================
[
1, 2, 3,
{ "stuff": "good" }
]
---
(value (array
(value (number))
(value (number))
(value (number))
(value (object
(string) (value (string))
))
))
===================
parses long objects
===================
{
"key1": "value1",
"key2": 1
}
---
(value (object
(string) (value (string))
(string) (value (number))
))

46
spec/runtime/node_position_spec.cc Normal file
View file

@ -0,0 +1,46 @@
#include "runtime_spec_helper.h"
extern "C" ts_parser ts_parser_json();
START_TEST
describe("tracking the positions of AST nodes", []() {
ts_document *doc;
before_each([&]() {
doc = ts_document_make();
ts_document_set_parser(doc, ts_parser_json());
});
after_each([&]() {
ts_document_free(doc);
});
it("records the widths and offsets of nodes", [&]() {
ts_document_set_input_string(doc, " [12, 5]");
const ts_tree *tree = ts_document_tree(doc);
// TODO - make this better
const ts_tree *array = ts_tree_children(tree, NULL)[0];
const ts_tree *number1 = ts_tree_children(ts_tree_children(array, NULL)[1], NULL)[0];
const ts_tree *number2 = ts_tree_children(ts_tree_children(ts_tree_children(array, NULL)[2], NULL)[1], NULL)[0];
AssertThat(ts_document_symbol_name(doc, array), Equals("array"));
AssertThat(ts_document_symbol_name(doc, number1), Equals("number"));
AssertThat(ts_document_symbol_name(doc, number2), Equals("number"));
AssertThat(number1->offset, Equals<size_t>(0));
AssertThat(number1->size, Equals<size_t>(2));
AssertThat(number2->offset, Equals<size_t>(1));
AssertThat(number2->size, Equals<size_t>(1));
AssertThat(array->offset, Equals<size_t>(2));
AssertThat(array->size, Equals<size_t>(7));
AssertThat(tree->offset, Equals<size_t>(2));
AssertThat(tree->size, Equals<size_t>(7));
});
});
END_TEST

1
tree_sitter.gyp
View file

@ -67,6 +67,7 @@
'sources': [
'<!@(find spec/runtime -name "*.h" -or -name "*.cc")',
'<!@(find examples/parsers -name "*.c")',
'<!@(find spec/runtime/languages -name "*.txt")',
],
},
],