Add tests that compile a grammar and use its parser

script/test_all.sh

@@ -4,3 +4,4 @@ set -e
 
 script/test_compiler.sh
 script/test_runtime.sh
+script/test_integration.sh

script/test_integration.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e -u
+
+source `dirname $0`/util/run_tests.sh
+run_tests integration_specs "$@"

spec/compiler/compile_spec.cc

@@ -1,47 +0,0 @@
-#include "compiler/compiler_spec_helper.h"
-#include "compiler/compile.h"
-
-using namespace rules;
-
-START_TEST
-
-describe("compile_grammar", []() {
-  describe("when the grammar's start symbol is a token", [&]() {
-    it("does not fail", [&]() {
-      TSCompileResult result = ts_compile_grammar(R"JSON(
-        {
-          "name": "the_grammar",
-          "rules": {
-            "rule1": {
-              "type": "STRING",
-              "value": "hello"
-            }
-          }
-        }
-      )JSON");
-
-      AssertThat(string(result.error_message), IsEmpty());
-      AssertThat(string(result.code), !IsEmpty());
-    });
-  });
-
-  describe("when the grammar's start symbol is blank", [&]() {
-    it("does not fail", [&]() {
-      TSCompileResult result = ts_compile_grammar(R"JSON(
-        {
-          "name": "the_grammar",
-          "rules": {
-            "rule1": {
-              "type": "BLANK"
-            }
-          }
-        }
-      )JSON");
-
-      AssertThat(string(result.error_message), IsEmpty());
-      AssertThat(string(result.code), !IsEmpty());
-    });
-  });
-});
-
-END_TEST

spec/integration/compile_grammar_spec.cc

@@ -0,0 +1,139 @@
+#include "integration/integration_spec_helper.h"
+#include "integration/helpers/load_language.h"
+
+START_TEST
+
+describe("compile_grammar", []() {
+  TSDocument *document;
+
+  before_each([&]() {
+    document = ts_document_make();
+  });
+
+  after_each([&]() {
+    ts_document_free(document);
+  });
+
+  describe("when the grammar's start symbol is a token", [&]() {
+    it("parses the token", [&]() {
+      TSCompileResult result = ts_compile_grammar(R"JSON(
+        {
+          "name": "test_language",
+          "rules": {
+            "first_rule": {"type": "STRING", "value": "the-value"}
+          }
+        }
+      )JSON");
+
+      const TSLanguage *language = load_language("test_language", result);
+
+      ts_document_set_language(document, language);
+      ts_document_set_input_string(document, "the-value");
+      ts_document_parse(document);
+
+      TSNode root_node = ts_document_root_node(document);
+      AssertThat(ts_node_string(root_node, document), Equals("(first_rule)"));
+    });
+  });
+
+  describe("when the grammar's start symbol is blank", [&]() {
+    it("parses the empty string", [&]() {
+      TSCompileResult result = ts_compile_grammar(R"JSON(
+        {
+          "name": "test_language",
+          "rules": {
+            "first_rule": {"type": "BLANK"}
+          }
+        }
+      )JSON");
+
+      const TSLanguage *language = load_language("test_language", result);
+
+      ts_document_set_language(document, language);
+      ts_document_set_input_string(document, "");
+      ts_document_parse(document);
+
+      TSNode root_node = ts_document_root_node(document);
+      AssertThat(ts_node_string(root_node, document), Equals("(first_rule)"));
+    });
+  });
+
+  describe("the grammar in the README", [&]() {
+    it("works", [&]() {
+      TSCompileResult result = ts_compile_grammar(R"JSON(
+        {
+          "name": "arithmetic",
+
+          "extras": [
+            {"type": "PATTERN", "value": "\\s"}
+          ],
+
+          "rules": {
+            "expression": {
+              "type": "CHOICE",
+              "members": [
+                {"type": "SYMBOL", "name": "sum"},
+                {"type": "SYMBOL", "name": "product"},
+                {"type": "SYMBOL", "name": "number"},
+                {"type": "SYMBOL", "name": "variable"},
+                {
+                  "type": "SEQ",
+                  "members": [
+                    {"type": "STRING", "value": "("},
+                    {"type": "SYMBOL", "name": "expression"},
+                    {"type": "STRING", "value": ")"}
+                  ]
+                }
+              ]
+            },
+
+            "sum": {
+              "type": "PREC_LEFT",
+              "value": 1,
+              "content": {
+                "type": "SEQ",
+                "members": [
+                  {"type": "SYMBOL", "name": "expression"},
+                  {"type": "STRING", "value": "+"},
+                  {"type": "SYMBOL", "name": "expression"}
+                ]
+              }
+            },
+
+            "product": {
+              "type": "PREC_LEFT",
+              "value": 2,
+              "content": {
+                "type": "SEQ",
+                "members": [
+                  {"type": "SYMBOL", "name": "expression"},
+                  {"type": "STRING", "value": "*"},
+                  {"type": "SYMBOL", "name": "expression"}
+                ]
+              }
+            },
+
+            "number": {"type": "PATTERN", "value": "\\d+"},
+            "variable": {"type": "PATTERN", "value": "[a-zA-Z]\\w*"}
+          }
+        }
+      )JSON");
+
+      const TSLanguage *language = load_language("arithmetic", result);
+
+      ts_document_set_language(document, language);
+      ts_document_set_input_string(document, "a + b * c");
+      ts_document_parse(document);
+
+      TSNode root_node = ts_document_root_node(document);
+      AssertThat(ts_node_string(root_node, document), Equals(
+        "(expression (sum "
+          "(expression (variable)) "
+          "(expression (product "
+             "(expression (variable)) "
+             "(expression (variable))))))"));
+    });
+  });
+});
+
+END_TEST

spec/integration/helpers/load_language.cc

@@ -0,0 +1,116 @@
+#include "integration/helpers/load_language.h"
+#include <unistd.h>
+#include <dlfcn.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <string>
+#include <fstream>
+#include "tree_sitter/compiler.h"
+#include "integration/integration_spec_helper.h"
+
+using std::string;
+using std::ofstream;
+
+static std::string run_cmd(const char *cmd, const char *args[]) {
+  int child_pid = fork();
+  if (child_pid < 0)
+    return "fork failed";
+
+  if (child_pid == 0) {
+    close(0);
+    dup2(1, 0);
+    dup2(2, 1);
+    dup2(1, 2);
+    execvp(cmd, (char * const * )args);
+    return "";
+  }
+
+  int status;
+  waitpid(child_pid, &status, 0);
+  if (!WIFEXITED(status))
+    return "child process did not exit";
+
+  if (WEXITSTATUS(status) != 0)
+    return "command failed";
+
+  return "";
+}
+
+const TSLanguage *load_language(const string &name, const TSCompileResult &compile_result) {
+  if (compile_result.error_type != TSCompileErrorTypeNone) {
+    AssertThat(string(compile_result.error_message), IsEmpty());
+    return nullptr;
+  }
+
+  string language_function_name = "ts_language_" + name;
+
+  static char source_file_template[256] = {};
+  snprintf(source_file_template, 256, "/tmp/tree-sitter-test-%sXXXXXXXXX", name.c_str());
+
+  const char *temp_directory = mkdtemp(source_file_template);
+  if (!temp_directory) {
+    AssertThat(string("Failed to create temp directory"), IsEmpty());
+    return nullptr;
+  }
+
+  string source_filename = string(temp_directory) + "/parser.c";
+  string obj_filename = string(source_filename) + ".o";
+  string lib_filename = string(source_filename) + ".so";
+  string header_dir = string(getenv("PWD")) + "/include";
+
+  ofstream source_file;
+  source_file.open(source_filename);
+  source_file << compile_result.code;
+  source_file.close();
+
+  const char *compiler_name = getenv("CC");
+  if (!compiler_name) {
+    compiler_name = "gcc";
+  }
+
+  const char *compile_argv[] = {
+    compiler_name,
+    "-x", "c",
+    "-fPIC",
+    "-I", header_dir.c_str(),
+    "-c", source_filename.c_str(),
+    "-o", obj_filename.c_str(),
+    NULL
+  };
+  string compile_error = run_cmd("gcc", compile_argv);
+  if (!compile_error.empty()) {
+    AssertThat(string(compile_error), IsEmpty());
+    return nullptr;
+  }
+
+  const char *link_argv[] = {
+    compiler_name,
+    "-shared",
+    "-Wl", obj_filename.c_str(),
+    "-o", lib_filename.c_str(),
+    NULL
+  };
+  string link_error = run_cmd("gcc", link_argv);
+  if (!link_error.empty()) {
+    AssertThat(link_error, IsEmpty());
+    return nullptr;
+  }
+
+  void *parser_lib = dlopen(lib_filename.c_str(), RTLD_NOW);
+  if (!parser_lib) {
+    std::string message(dlerror());
+    AssertThat(message, IsEmpty());
+    return nullptr;
+  }
+
+  void *symbol_value = dlsym(parser_lib, language_function_name.c_str());
+  if (!symbol_value) {
+    std::string message(dlerror());
+    AssertThat(message, IsEmpty());
+    return nullptr;
+  }
+
+  typedef TSLanguage * (* LanguageFunction)();
+  LanguageFunction language_fn = reinterpret_cast<LanguageFunction>(symbol_value);
+  return language_fn();
+}

spec/integration/helpers/load_language.h

@@ -0,0 +1,10 @@
+#ifndef INTEGRATION_HELPERS_LOAD_LANGUAGE_H_
+#define INTEGRATION_HELPERS_LOAD_LANGUAGE_H_
+
+#include "tree_sitter/compiler.h"
+#include "tree_sitter/runtime.h"
+#include <string>
+
+const TSLanguage *load_language(const std::string &, const TSCompileResult &);
+
+#endif  // INTEGRATION_HELPERS_LOAD_LANGUAGE_H_

spec/integration/integration_spec_helper.h

@@ -0,0 +1,14 @@
+#ifndef INTEGRATION_SPEC_HELPER_
+#define INTEGRATION_SPEC_HELPER_
+
+#include "bandit/bandit.h"
+#include "tree_sitter/compiler.h"
+#include "tree_sitter/runtime.h"
+
+using namespace std;
+using namespace bandit;
+
+#define START_TEST go_bandit([]() {
+#define END_TEST });
+
+#endif  // INTEGRATION_SPEC_HELPER_

spec/integration/integration_specs.cc

@@ -0,0 +1,16 @@
+#include "integration/integration_spec_helper.h"
+
+int main(int argc, char *argv[]) {
+  int seed;
+  const char *seed_env = getenv("TREE_SITTER_SEED");
+  if (seed_env) {
+    seed = atoi(seed_env);
+  } else {
+    seed = time(nullptr);
+  }
+
+  printf("Random seed: %d\n", seed);
+  srandom(seed);
+
+  return bandit::run(argc, argv);
+}

tests.gyp

@@ -25,7 +25,6 @@
       ],
       'include_dirs': [
         'src',
-        'examples',
         'spec',
         'externals/bandit',
         'externals/utf8proc',
@@ -44,6 +43,27 @@
         }]
       ],
     },
+
+    {
+      'target_name': 'integration_specs',
+      'type': 'executable',
+      'dependencies': [
+        'project.gyp:compiler',
+        'project.gyp:runtime'
+      ],
+      'include_dirs': [
+        'src',
+        'spec',
+        'externals/bandit',
+        'externals/utf8proc'
+      ],
+      'libraries': [
+        '-ldl'
+      ],
+      'sources': [
+        '<!@(find spec/integration -name "*.cc")'
+      ],
+    },
   ],
 
   'target_defaults': {