Avoid recompiling test languages when possible

spec/helpers/load_language.cc

@@ -4,12 +4,27 @@
 #include <dlfcn.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <map>
 #include <string>
+#include <sys/stat.h>
 #include <fstream>
 #include "tree_sitter/compiler.h"
 
+using std::map;
 using std::string;
+using std::ifstream;
 using std::ofstream;
+using std::istreambuf_iterator;
+
+map<string, const TSLanguage *> loaded_languages;
+int libcompiler_mtime = -1;
+
+const char *libcompiler_path =
+#if defined(__linux)
+  "out/Test/obj.target/libcompiler.a";
+#else
+  "out/Test/libcompiler.a";
+#endif
 
 static std::string run_cmd(const char *cmd, const char *args[]) {
   int child_pid = fork();
@@ -38,71 +53,66 @@ static std::string run_cmd(const char *cmd, const char *args[]) {
   return "";
 }
 
-const TSLanguage *load_language(const string &name, const TSCompileResult &compile_result) {
+static int get_modified_time(const string &path) {
-  if (compile_result.error_type != TSCompileErrorTypeNone) {
+  struct stat file_stat;
-    Assert::Failure(string("Compilation failed ") + compile_result.error_message);
+  if (stat(path.c_str(), &file_stat) != 0) {
-    return nullptr;
+    if (errno != ENOENT)
+      fprintf(stderr, "Error in stat() for path: %s\n", + path.c_str());
+    return 0;
   }
-
+  return file_stat.st_mtime;
-  const TSLanguage *language = load_language(name, compile_result.code);
-  free(compile_result.code);
-  return language;
 }
 
-const TSLanguage *load_language(const string &name, const string &code) {
+const TSLanguage *load_language(const string &name, const string &code, int timestamp) {
+  mkdir("out/tmp", 0777);
+
+  string pwd(getenv("PWD"));
   string language_function_name = "ts_language_" + name;
+  string header_dir = pwd + "/include";
+  string source_filename = pwd + "/out/tmp/" + name + ".c";
+  string obj_filename = source_filename + ".o";
+  string lib_filename = source_filename + ".so";
 
-  static char source_file_template[256] = {};
+  int mtime = get_modified_time(lib_filename);
-  snprintf(source_file_template, 256, "/tmp/tree-sitter-test-%sXXXXXXXXX", name.c_str());
+  if (!mtime || !timestamp || mtime < timestamp) {
+    ofstream source_file;
+    source_file.open(source_filename);
+    source_file << code;
+    source_file.close();
 
-  const char *temp_directory = mkdtemp(source_file_template);
+    const char *compiler_name = getenv("CC");
-  if (!temp_directory) {
+    if (!compiler_name) {
-    AssertThat(string("Failed to create temp directory"), IsEmpty());
+      compiler_name = "gcc";
-    return nullptr;
+    }
-  }
 
-  string source_filename = string(temp_directory) + "/generated-parser.c";
+    const char *compile_argv[] = {
-  string obj_filename = string(source_filename) + ".o";
+      compiler_name,
-  string lib_filename = string(source_filename) + ".so";
+      "-x", "c",
-  string header_dir = string(getenv("PWD")) + "/include";
+      "-fPIC",
+      "-g",
+      "-I", header_dir.c_str(),
+      "-c", source_filename.c_str(),
+      "-o", obj_filename.c_str(),
+      NULL
+    };
+    string compile_error = run_cmd("gcc", compile_argv);
+    if (!compile_error.empty()) {
+      AssertThat(string(compile_error), IsEmpty());
+      return nullptr;
+    }
 
-  ofstream source_file;
+    const char *link_argv[] = {
-  source_file.open(source_filename);
+      compiler_name,
-  source_file << code;
+      "-shared",
-  source_file.close();
+      "-Wl", obj_filename.c_str(),
-
+      "-o", lib_filename.c_str(),
-  const char *compiler_name = getenv("CC");
+      NULL
-  if (!compiler_name) {
+    };
-    compiler_name = "gcc";
+    string link_error = run_cmd("gcc", link_argv);
-  }
+    if (!link_error.empty()) {
-
+      AssertThat(link_error, IsEmpty());
-  const char *compile_argv[] = {
+      return nullptr;
-    compiler_name,
+    }
-    "-x", "c",
-    "-fPIC",
-    "-g",
-    "-I", header_dir.c_str(),
-    "-c", source_filename.c_str(),
-    "-o", obj_filename.c_str(),
-    NULL
-  };
-  string compile_error = run_cmd("gcc", compile_argv);
-  if (!compile_error.empty()) {
-    AssertThat(string(compile_error), IsEmpty());
-    return nullptr;
-  }
-
-  const char *link_argv[] = {
-    compiler_name,
-    "-shared",
-    "-Wl", obj_filename.c_str(),
-    "-o", lib_filename.c_str(),
-    NULL
-  };
-  string link_error = run_cmd("gcc", link_argv);
-  if (!link_error.empty()) {
-    AssertThat(link_error, IsEmpty());
-    return nullptr;
   }
 
   void *parser_lib = dlopen(lib_filename.c_str(), RTLD_NOW);
@@ -123,3 +133,69 @@ const TSLanguage *load_language(const string &name, const string &code) {
   LanguageFunction language_fn = reinterpret_cast<LanguageFunction>(symbol_value);
   return language_fn();
 }
+
+const TSLanguage *load_language(const string &name, const TSCompileResult &compile_result) {
+  if (compile_result.error_type != TSCompileErrorTypeNone) {
+    Assert::Failure(string("Compilation failed ") + compile_result.error_message);
+    return nullptr;
+  }
+
+  const TSLanguage *language = load_language(name, compile_result.code, 0);
+  free(compile_result.code);
+  return language;
+}
+
+const TSLanguage *get_test_language(const string &language_name) {
+  if (loaded_languages[language_name])
+    return loaded_languages[language_name];
+
+  if (libcompiler_mtime == -1) {
+    libcompiler_mtime = get_modified_time(libcompiler_path);
+    if (!libcompiler_mtime)
+      return nullptr;
+  }
+
+  string language_dir = string("spec/fixtures/") + language_name;
+  string grammar_filename = language_dir + "/src/grammar.json";
+  string parser_filename = language_dir + "/src/parser.c";
+
+  int grammar_mtime = get_modified_time(grammar_filename);
+  if (!grammar_mtime)
+    return nullptr;
+
+  int parser_mtime = get_modified_time(parser_filename);
+
+  int input_mtime = (grammar_mtime < libcompiler_mtime) ?
+    grammar_mtime :
+    libcompiler_mtime;
+
+  string parser_code;
+  if (!parser_mtime || parser_mtime < input_mtime) {
+    printf("\n" "Regenerating the %s parser...\n", language_name.c_str());
+
+    ifstream grammar_file(grammar_filename);
+    istreambuf_iterator<char> grammar_file_iterator(grammar_file), end_iterator;
+    std::string grammar_json(grammar_file_iterator, end_iterator);
+
+    TSCompileResult result = ts_compile_grammar(grammar_json.c_str());
+    if (result.error_type != TSCompileErrorTypeNone) {
+      fprintf(stderr, "Failed to compile %s grammar: %s\n", language_name.c_str(), result.error_message);
+      return nullptr;
+    }
+
+    ofstream parser_file(parser_filename);
+    parser_file << result.code;
+    parser_code = result.code;
+
+    grammar_file.close();
+    parser_file.close();
+  } else {
+    ifstream parser_file(parser_filename);
+    istreambuf_iterator<char> grammar_file_iterator(parser_file), end_iterator;
+    parser_code.assign(grammar_file_iterator, end_iterator);
+  }
+
+  const TSLanguage *language = load_language(language_name, parser_code, input_mtime);
+  loaded_languages[language_name] = language;
+  return language;
+};

spec/helpers/load_language.h

@@ -5,7 +5,7 @@
 #include "tree_sitter/runtime.h"
 #include <string>
 
-const TSLanguage *load_language(const std::string &, const std::string &);
 const TSLanguage *load_language(const std::string &, const TSCompileResult &);
+const TSLanguage *get_test_language(const std::string &language_name);
 
 #endif  // HELPERS_LOAD_LANGUAGE_H_

spec/helpers/test_languages.cc

@@ -1,91 +0,0 @@
-#include "helpers/test_languages.h"
-#include "helpers/load_language.h"
-#include <sys/stat.h>
-#include <fstream>
-#include <map>
-#include <string>
-
-using std::map;
-using std::string;
-using std::ifstream;
-using std::ofstream;
-using std::istreambuf_iterator;
-
-map<string, const TSLanguage *> loaded_languages;
-int libcompiler_mtime = -1;
-
-const char libcompiler_path[] =
-#if defined(__linux)
-  "out/Test/obj.target/libcompiler.a"
-#else
-  "out/Test/libcompiler.a"
-#endif
-;
-
-static int get_modified_time(const string &path) {
-  struct stat file_stat;
-  int error = stat(path.c_str(), &file_stat);
-  if (error != 0) {
-    fprintf(stderr, "Error in stat() for path: %s\n", + path.c_str());
-    return 0;
-  }
-
-  return file_stat.st_mtime;
-}
-
-const TSLanguage *get_test_language(const string &language_name) {
-  if (libcompiler_mtime == -1) {
-    libcompiler_mtime = get_modified_time(libcompiler_path);
-    if (!libcompiler_mtime) {
-      return nullptr;
-    }
-  }
-
-  if (loaded_languages[language_name]) {
-    return loaded_languages[language_name];
-  }
-
-  string language_dir = string("spec/fixtures/") + language_name;
-  string grammar_filename = language_dir + "/src/grammar.json";
-  string parser_filename = language_dir + "/src/parser.c";
-
-  int grammar_mtime = get_modified_time(grammar_filename);
-  if (!grammar_mtime) {
-    return nullptr;
-  }
-
-  int parser_mtime = get_modified_time(parser_filename);
-  if (!parser_mtime) {
-    return nullptr;
-  }
-
-  string parser_code;
-  if (parser_mtime <= grammar_mtime || parser_mtime <= libcompiler_mtime) {
-    printf("\n" "Regenerating the %s parser...\n", language_name.c_str());
-
-    ifstream grammar_file(grammar_filename);
-    istreambuf_iterator<char> grammar_file_iterator(grammar_file), end_iterator;
-    std::string grammar_json(grammar_file_iterator, end_iterator);
-
-    TSCompileResult result = ts_compile_grammar(grammar_json.c_str());
-    if (result.error_type != TSCompileErrorTypeNone) {
-      fprintf(stderr, "Failed to compile %s grammar: %s\n", language_name.c_str(), result.error_message);
-      return nullptr;
-    }
-
-    ofstream parser_file(parser_filename);
-    parser_file << result.code;
-    parser_code = result.code;
-
-    grammar_file.close();
-    parser_file.close();
-  } else {
-    ifstream parser_file(parser_filename);
-    istreambuf_iterator<char> grammar_file_iterator(parser_file), end_iterator;
-    parser_code.assign(grammar_file_iterator, end_iterator);
-  }
-
-  const TSLanguage *language = load_language(language_name, parser_code);
-  loaded_languages[language_name] = language;
-  return language;
-};

spec/helpers/test_languages.h

@@ -1,9 +0,0 @@
-#ifndef HELPERS_TEST_LANGUAGES_H_
-#define HELPERS_TEST_LANGUAGES_H_
-
-#include "tree_sitter/runtime.h"
-#include <string>
-
-const TSLanguage *get_test_language(const std::string &);
-
-#endif  // HELPERS_TEST_LANGUAGES_H_

spec/integration/compile_grammar_spec.cc

@@ -26,14 +26,14 @@ describe("compile_grammar", []() {
     it("parses the token", [&]() {
       TSCompileResult result = ts_compile_grammar(R"JSON(
         {
-          "name": "test_language",
+          "name": "one_token_language",
           "rules": {
             "first_rule": {"type": "STRING", "value": "the-value"}
           }
         }
       )JSON");
 
-      ts_document_set_language(document, load_language("test_language", result));
+      ts_document_set_language(document, load_language("one_token_language", result));
 
       ts_document_set_input_string(document, "the-value");
       ts_document_parse(document);
@@ -45,14 +45,14 @@ describe("compile_grammar", []() {
     it("parses the empty string", [&]() {
       TSCompileResult result = ts_compile_grammar(R"JSON(
         {
-          "name": "test_language",
+          "name": "blank_language",
           "rules": {
             "first_rule": {"type": "BLANK"}
           }
         }
       )JSON");
 
-      ts_document_set_language(document, load_language("test_language", result));
+      ts_document_set_language(document, load_language("blank_language", result));
 
       ts_document_set_input_string(document, "");
       ts_document_parse(document);
@@ -64,7 +64,7 @@ describe("compile_grammar", []() {
     it("escapes the escaped characters properly in the generated parser", [&]() {
       TSCompileResult result = ts_compile_grammar(R"JSON(
         {
-          "name": "test_language",
+          "name": "escaped_char_language",
           "rules": {
             "first_rule": {
               "type": "CHOICE",
@@ -79,7 +79,7 @@ describe("compile_grammar", []() {
         }
       )JSON");
 
-      ts_document_set_language(document, load_language("test_language", result));
+      ts_document_set_language(document, load_language("escaped_char_language", result));
 
       ts_document_set_input_string(document, "1234");
       ts_document_parse(document);
@@ -96,7 +96,7 @@ describe("compile_grammar", []() {
   });
 
   describe("the grammar in the README", [&]() {
-    it("works", [&]() {
+    it("parses the input in the README", [&]() {
       TSCompileResult result = ts_compile_grammar(R"JSON(
         {
           "name": "arithmetic",

spec/integration/corpus_specs.cc

@@ -1,6 +1,6 @@
 #include "spec_helper.h"
 #include "runtime/alloc.h"
-#include "helpers/test_languages.h"
+#include "helpers/load_language.h"
 #include "helpers/read_test_entries.h"
 #include "helpers/spy_input.h"
 #include "helpers/log_debugger.h"

spec/runtime/document_spec.cc

@@ -6,7 +6,7 @@
 #include "helpers/tree_helpers.h"
 #include "helpers/spy_debugger.h"
 #include "helpers/spy_input.h"
-#include "helpers/test_languages.h"
+#include "helpers/load_language.h"
 
 START_TEST
 

spec/runtime/node_spec.cc

@@ -2,7 +2,7 @@
 #include "runtime/alloc.h"
 #include "helpers/tree_helpers.h"
 #include "helpers/point_helpers.h"
-#include "helpers/test_languages.h"
+#include "helpers/load_language.h"
 #include "helpers/record_alloc.h"
 #include "helpers/stream_methods.h"
 

spec/runtime/parser_spec.cc

@@ -2,7 +2,7 @@
 #include "runtime/alloc.h"
 #include "helpers/record_alloc.h"
 #include "helpers/spy_input.h"
-#include "helpers/test_languages.h"
+#include "helpers/load_language.h"
 #include "helpers/log_debugger.h"
 #include "helpers/record_alloc.h"