From 4a5deda071426647b3d0ffd3d381d48af57542b6 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 14 Jan 2016 09:25:03 -0800 Subject: [PATCH] Add tests that compile a grammar and use its parser --- script/test_all.sh | 1 + script/test_integration.sh | 6 + spec/compiler/compile_spec.cc | 47 ------- spec/integration/compile_grammar_spec.cc | 139 +++++++++++++++++++++ spec/integration/helpers/load_language.cc | 116 +++++++++++++++++ spec/integration/helpers/load_language.h | 10 ++ spec/integration/integration_spec_helper.h | 14 +++ spec/integration/integration_specs.cc | 16 +++ tests.gyp | 22 +++- 9 files changed, 323 insertions(+), 48 deletions(-) create mode 100755 script/test_integration.sh delete mode 100644 spec/compiler/compile_spec.cc create mode 100644 spec/integration/compile_grammar_spec.cc create mode 100644 spec/integration/helpers/load_language.cc create mode 100644 spec/integration/helpers/load_language.h create mode 100644 spec/integration/integration_spec_helper.h create mode 100644 spec/integration/integration_specs.cc diff --git a/script/test_all.sh b/script/test_all.sh index 8846b489..b83a530e 100755 --- a/script/test_all.sh +++ b/script/test_all.sh @@ -4,3 +4,4 @@ set -e script/test_compiler.sh script/test_runtime.sh +script/test_integration.sh diff --git a/script/test_integration.sh b/script/test_integration.sh new file mode 100755 index 00000000..247572eb --- /dev/null +++ b/script/test_integration.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e -u + +source `dirname $0`/util/run_tests.sh +run_tests integration_specs "$@" diff --git a/spec/compiler/compile_spec.cc b/spec/compiler/compile_spec.cc deleted file mode 100644 index c72c9449..00000000 --- a/spec/compiler/compile_spec.cc +++ /dev/null @@ -1,47 +0,0 @@ -#include "compiler/compiler_spec_helper.h" -#include "compiler/compile.h" - -using namespace rules; - -START_TEST - -describe("compile_grammar", []() { - describe("when the grammar's start symbol is a token", [&]() { - it("does not fail", [&]() { - TSCompileResult result = ts_compile_grammar(R"JSON( - { - "name": "the_grammar", - "rules": { - "rule1": { - "type": "STRING", - "value": "hello" - } - } - } - )JSON"); - - AssertThat(string(result.error_message), IsEmpty()); - AssertThat(string(result.code), !IsEmpty()); - }); - }); - - describe("when the grammar's start symbol is blank", [&]() { - it("does not fail", [&]() { - TSCompileResult result = ts_compile_grammar(R"JSON( - { - "name": "the_grammar", - "rules": { - "rule1": { - "type": "BLANK" - } - } - } - )JSON"); - - AssertThat(string(result.error_message), IsEmpty()); - AssertThat(string(result.code), !IsEmpty()); - }); - }); -}); - -END_TEST diff --git a/spec/integration/compile_grammar_spec.cc b/spec/integration/compile_grammar_spec.cc new file mode 100644 index 00000000..80fe0763 --- /dev/null +++ b/spec/integration/compile_grammar_spec.cc @@ -0,0 +1,139 @@ +#include "integration/integration_spec_helper.h" +#include "integration/helpers/load_language.h" + +START_TEST + +describe("compile_grammar", []() { + TSDocument *document; + + before_each([&]() { + document = ts_document_make(); + }); + + after_each([&]() { + ts_document_free(document); + }); + + describe("when the grammar's start symbol is a token", [&]() { + it("parses the token", [&]() { + TSCompileResult result = ts_compile_grammar(R"JSON( + { + "name": "test_language", + "rules": { + "first_rule": {"type": "STRING", "value": "the-value"} + } + } + )JSON"); + + const TSLanguage *language = load_language("test_language", result); + + ts_document_set_language(document, language); + ts_document_set_input_string(document, "the-value"); + ts_document_parse(document); + + TSNode root_node = ts_document_root_node(document); + AssertThat(ts_node_string(root_node, document), Equals("(first_rule)")); + }); + }); + + describe("when the grammar's start symbol is blank", [&]() { + it("parses the empty string", [&]() { + TSCompileResult result = ts_compile_grammar(R"JSON( + { + "name": "test_language", + "rules": { + "first_rule": {"type": "BLANK"} + } + } + )JSON"); + + const TSLanguage *language = load_language("test_language", result); + + ts_document_set_language(document, language); + ts_document_set_input_string(document, ""); + ts_document_parse(document); + + TSNode root_node = ts_document_root_node(document); + AssertThat(ts_node_string(root_node, document), Equals("(first_rule)")); + }); + }); + + describe("the grammar in the README", [&]() { + it("works", [&]() { + TSCompileResult result = ts_compile_grammar(R"JSON( + { + "name": "arithmetic", + + "extras": [ + {"type": "PATTERN", "value": "\\s"} + ], + + "rules": { + "expression": { + "type": "CHOICE", + "members": [ + {"type": "SYMBOL", "name": "sum"}, + {"type": "SYMBOL", "name": "product"}, + {"type": "SYMBOL", "name": "number"}, + {"type": "SYMBOL", "name": "variable"}, + { + "type": "SEQ", + "members": [ + {"type": "STRING", "value": "("}, + {"type": "SYMBOL", "name": "expression"}, + {"type": "STRING", "value": ")"} + ] + } + ] + }, + + "sum": { + "type": "PREC_LEFT", + "value": 1, + "content": { + "type": "SEQ", + "members": [ + {"type": "SYMBOL", "name": "expression"}, + {"type": "STRING", "value": "+"}, + {"type": "SYMBOL", "name": "expression"} + ] + } + }, + + "product": { + "type": "PREC_LEFT", + "value": 2, + "content": { + "type": "SEQ", + "members": [ + {"type": "SYMBOL", "name": "expression"}, + {"type": "STRING", "value": "*"}, + {"type": "SYMBOL", "name": "expression"} + ] + } + }, + + "number": {"type": "PATTERN", "value": "\\d+"}, + "variable": {"type": "PATTERN", "value": "[a-zA-Z]\\w*"} + } + } + )JSON"); + + const TSLanguage *language = load_language("arithmetic", result); + + ts_document_set_language(document, language); + ts_document_set_input_string(document, "a + b * c"); + ts_document_parse(document); + + TSNode root_node = ts_document_root_node(document); + AssertThat(ts_node_string(root_node, document), Equals( + "(expression (sum " + "(expression (variable)) " + "(expression (product " + "(expression (variable)) " + "(expression (variable))))))")); + }); + }); +}); + +END_TEST diff --git a/spec/integration/helpers/load_language.cc b/spec/integration/helpers/load_language.cc new file mode 100644 index 00000000..f47892d0 --- /dev/null +++ b/spec/integration/helpers/load_language.cc @@ -0,0 +1,116 @@ +#include "integration/helpers/load_language.h" +#include +#include +#include +#include +#include +#include +#include "tree_sitter/compiler.h" +#include "integration/integration_spec_helper.h" + +using std::string; +using std::ofstream; + +static std::string run_cmd(const char *cmd, const char *args[]) { + int child_pid = fork(); + if (child_pid < 0) + return "fork failed"; + + if (child_pid == 0) { + close(0); + dup2(1, 0); + dup2(2, 1); + dup2(1, 2); + execvp(cmd, (char * const * )args); + return ""; + } + + int status; + waitpid(child_pid, &status, 0); + if (!WIFEXITED(status)) + return "child process did not exit"; + + if (WEXITSTATUS(status) != 0) + return "command failed"; + + return ""; +} + +const TSLanguage *load_language(const string &name, const TSCompileResult &compile_result) { + if (compile_result.error_type != TSCompileErrorTypeNone) { + AssertThat(string(compile_result.error_message), IsEmpty()); + return nullptr; + } + + string language_function_name = "ts_language_" + name; + + static char source_file_template[256] = {}; + snprintf(source_file_template, 256, "/tmp/tree-sitter-test-%sXXXXXXXXX", name.c_str()); + + const char *temp_directory = mkdtemp(source_file_template); + if (!temp_directory) { + AssertThat(string("Failed to create temp directory"), IsEmpty()); + return nullptr; + } + + string source_filename = string(temp_directory) + "/parser.c"; + string obj_filename = string(source_filename) + ".o"; + string lib_filename = string(source_filename) + ".so"; + string header_dir = string(getenv("PWD")) + "/include"; + + ofstream source_file; + source_file.open(source_filename); + source_file << compile_result.code; + source_file.close(); + + const char *compiler_name = getenv("CC"); + if (!compiler_name) { + compiler_name = "gcc"; + } + + const char *compile_argv[] = { + compiler_name, + "-x", "c", + "-fPIC", + "-I", header_dir.c_str(), + "-c", source_filename.c_str(), + "-o", obj_filename.c_str(), + NULL + }; + string compile_error = run_cmd("gcc", compile_argv); + if (!compile_error.empty()) { + AssertThat(string(compile_error), IsEmpty()); + return nullptr; + } + + const char *link_argv[] = { + compiler_name, + "-shared", + "-Wl", obj_filename.c_str(), + "-o", lib_filename.c_str(), + NULL + }; + string link_error = run_cmd("gcc", link_argv); + if (!link_error.empty()) { + AssertThat(link_error, IsEmpty()); + return nullptr; + } + + void *parser_lib = dlopen(lib_filename.c_str(), RTLD_NOW); + if (!parser_lib) { + std::string message(dlerror()); + AssertThat(message, IsEmpty()); + return nullptr; + } + + void *symbol_value = dlsym(parser_lib, language_function_name.c_str()); + if (!symbol_value) { + std::string message(dlerror()); + AssertThat(message, IsEmpty()); + return nullptr; + } + + typedef TSLanguage * (* LanguageFunction)(); + LanguageFunction language_fn = reinterpret_cast(symbol_value); + return language_fn(); +} diff --git a/spec/integration/helpers/load_language.h b/spec/integration/helpers/load_language.h new file mode 100644 index 00000000..404cf822 --- /dev/null +++ b/spec/integration/helpers/load_language.h @@ -0,0 +1,10 @@ +#ifndef INTEGRATION_HELPERS_LOAD_LANGUAGE_H_ +#define INTEGRATION_HELPERS_LOAD_LANGUAGE_H_ + +#include "tree_sitter/compiler.h" +#include "tree_sitter/runtime.h" +#include + +const TSLanguage *load_language(const std::string &, const TSCompileResult &); + +#endif // INTEGRATION_HELPERS_LOAD_LANGUAGE_H_ diff --git a/spec/integration/integration_spec_helper.h b/spec/integration/integration_spec_helper.h new file mode 100644 index 00000000..d2a2fa8c --- /dev/null +++ b/spec/integration/integration_spec_helper.h @@ -0,0 +1,14 @@ +#ifndef INTEGRATION_SPEC_HELPER_ +#define INTEGRATION_SPEC_HELPER_ + +#include "bandit/bandit.h" +#include "tree_sitter/compiler.h" +#include "tree_sitter/runtime.h" + +using namespace std; +using namespace bandit; + +#define START_TEST go_bandit([]() { +#define END_TEST }); + +#endif // INTEGRATION_SPEC_HELPER_ diff --git a/spec/integration/integration_specs.cc b/spec/integration/integration_specs.cc new file mode 100644 index 00000000..60e1e433 --- /dev/null +++ b/spec/integration/integration_specs.cc @@ -0,0 +1,16 @@ +#include "integration/integration_spec_helper.h" + +int main(int argc, char *argv[]) { + int seed; + const char *seed_env = getenv("TREE_SITTER_SEED"); + if (seed_env) { + seed = atoi(seed_env); + } else { + seed = time(nullptr); + } + + printf("Random seed: %d\n", seed); + srandom(seed); + + return bandit::run(argc, argv); +} diff --git a/tests.gyp b/tests.gyp index 27f86b3c..a4d03aa4 100644 --- a/tests.gyp +++ b/tests.gyp @@ -25,7 +25,6 @@ ], 'include_dirs': [ 'src', - 'examples', 'spec', 'externals/bandit', 'externals/utf8proc', @@ -44,6 +43,27 @@ }] ], }, + + { + 'target_name': 'integration_specs', + 'type': 'executable', + 'dependencies': [ + 'project.gyp:compiler', + 'project.gyp:runtime' + ], + 'include_dirs': [ + 'src', + 'spec', + 'externals/bandit', + 'externals/utf8proc' + ], + 'libraries': [ + '-ldl' + ], + 'sources': [ + '