traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Avoid recompiling test languages when possible

Browse source
This commit is contained in:
Max Brunsfeld 2016-03-03 12:05:04 -08:00
parent c0595c21c5
commit bc8df9f5c5
9 changed files with 145 additions and 169 deletions
Download patch file Download diff file Expand all files Collapse all files

190
spec/helpers/load_language.cc
View file

@ -4,12 +4,27 @@
#include <dlfcn.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <map>
#include <string>
#include <sys/stat.h>
#include <fstream>
#include "tree_sitter/compiler.h"
using std::map;
using std::string;
using std::ifstream;
using std::ofstream;
using std::istreambuf_iterator;
map<string, const TSLanguage *> loaded_languages;
int libcompiler_mtime = -1;
const char *libcompiler_path =
#if defined(__linux)
"out/Test/obj.target/libcompiler.a";
#else
"out/Test/libcompiler.a";
#endif
static std::string run_cmd(const char *cmd, const char *args[]) {
int child_pid = fork();
@ -38,71 +53,66 @@ static std::string run_cmd(const char *cmd, const char *args[]) {
return "";
}
const TSLanguage *load_language(const string &name, const TSCompileResult &compile_result) {
if (compile_result.error_type != TSCompileErrorTypeNone) {
Assert::Failure(string("Compilation failed ") + compile_result.error_message);
return nullptr;
static int get_modified_time(const string &path) {
struct stat file_stat;
if (stat(path.c_str(), &file_stat) != 0) {
if (errno != ENOENT)
fprintf(stderr, "Error in stat() for path: %s\n", + path.c_str());
return 0;
}
const TSLanguage *language = load_language(name, compile_result.code);
free(compile_result.code);
return language;
return file_stat.st_mtime;
}
const TSLanguage *load_language(const string &name, const string &code) {
const TSLanguage *load_language(const string &name, const string &code, int timestamp) {
mkdir("out/tmp", 0777);
string pwd(getenv("PWD"));
string language_function_name = "ts_language_" + name;
string header_dir = pwd + "/include";
string source_filename = pwd + "/out/tmp/" + name + ".c";
string obj_filename = source_filename + ".o";
string lib_filename = source_filename + ".so";
static char source_file_template[256] = {};
snprintf(source_file_template, 256, "/tmp/tree-sitter-test-%sXXXXXXXXX", name.c_str());
int mtime = get_modified_time(lib_filename);
if (!mtime || !timestamp || mtime < timestamp) {
ofstream source_file;
source_file.open(source_filename);
source_file << code;
source_file.close();
const char *temp_directory = mkdtemp(source_file_template);
if (!temp_directory) {
AssertThat(string("Failed to create temp directory"), IsEmpty());
return nullptr;
}
const char *compiler_name = getenv("CC");
if (!compiler_name) {
compiler_name = "gcc";
}
string source_filename = string(temp_directory) + "/generated-parser.c";
string obj_filename = string(source_filename) + ".o";
string lib_filename = string(source_filename) + ".so";
string header_dir = string(getenv("PWD")) + "/include";
const char *compile_argv[] = {
compiler_name,
"-x", "c",
"-fPIC",
"-g",
"-I", header_dir.c_str(),
"-c", source_filename.c_str(),
"-o", obj_filename.c_str(),
NULL
};
string compile_error = run_cmd("gcc", compile_argv);
if (!compile_error.empty()) {
AssertThat(string(compile_error), IsEmpty());
return nullptr;
}
ofstream source_file;
source_file.open(source_filename);
source_file << code;
source_file.close();
const char *compiler_name = getenv("CC");
if (!compiler_name) {
compiler_name = "gcc";
}
const char *compile_argv[] = {
compiler_name,
"-x", "c",
"-fPIC",
"-g",
"-I", header_dir.c_str(),
"-c", source_filename.c_str(),
"-o", obj_filename.c_str(),
NULL
};
string compile_error = run_cmd("gcc", compile_argv);
if (!compile_error.empty()) {
AssertThat(string(compile_error), IsEmpty());
return nullptr;
}
const char *link_argv[] = {
compiler_name,
"-shared",
"-Wl", obj_filename.c_str(),
"-o", lib_filename.c_str(),
NULL
};
string link_error = run_cmd("gcc", link_argv);
if (!link_error.empty()) {
AssertThat(link_error, IsEmpty());
return nullptr;
const char *link_argv[] = {
compiler_name,
"-shared",
"-Wl", obj_filename.c_str(),
"-o", lib_filename.c_str(),
NULL
};
string link_error = run_cmd("gcc", link_argv);
if (!link_error.empty()) {
AssertThat(link_error, IsEmpty());
return nullptr;
}
}
void *parser_lib = dlopen(lib_filename.c_str(), RTLD_NOW);
@ -123,3 +133,69 @@ const TSLanguage *load_language(const string &name, const string &code) {
LanguageFunction language_fn = reinterpret_cast<LanguageFunction>(symbol_value);
return language_fn();
}
const TSLanguage *load_language(const string &name, const TSCompileResult &compile_result) {
if (compile_result.error_type != TSCompileErrorTypeNone) {
Assert::Failure(string("Compilation failed ") + compile_result.error_message);
return nullptr;
}
const TSLanguage *language = load_language(name, compile_result.code, 0);
free(compile_result.code);
return language;
}
const TSLanguage *get_test_language(const string &language_name) {
if (loaded_languages[language_name])
return loaded_languages[language_name];
if (libcompiler_mtime == -1) {
libcompiler_mtime = get_modified_time(libcompiler_path);
if (!libcompiler_mtime)
return nullptr;
}
string language_dir = string("spec/fixtures/") + language_name;
string grammar_filename = language_dir + "/src/grammar.json";
string parser_filename = language_dir + "/src/parser.c";
int grammar_mtime = get_modified_time(grammar_filename);
if (!grammar_mtime)
return nullptr;
int parser_mtime = get_modified_time(parser_filename);
int input_mtime = (grammar_mtime < libcompiler_mtime) ?
grammar_mtime :
libcompiler_mtime;
string parser_code;
if (!parser_mtime || parser_mtime < input_mtime) {
printf("\n" "Regenerating the %s parser...\n", language_name.c_str());
ifstream grammar_file(grammar_filename);
istreambuf_iterator<char> grammar_file_iterator(grammar_file), end_iterator;
std::string grammar_json(grammar_file_iterator, end_iterator);
TSCompileResult result = ts_compile_grammar(grammar_json.c_str());
if (result.error_type != TSCompileErrorTypeNone) {
fprintf(stderr, "Failed to compile %s grammar: %s\n", language_name.c_str(), result.error_message);
return nullptr;
}
ofstream parser_file(parser_filename);
parser_file << result.code;
parser_code = result.code;
grammar_file.close();
parser_file.close();
} else {
ifstream parser_file(parser_filename);
istreambuf_iterator<char> grammar_file_iterator(parser_file), end_iterator;
parser_code.assign(grammar_file_iterator, end_iterator);
}
const TSLanguage *language = load_language(language_name, parser_code, input_mtime);
loaded_languages[language_name] = language;
return language;
};

2
spec/helpers/load_language.h
View file

@ -5,7 +5,7 @@
#include "tree_sitter/runtime.h"
#include <string>
const TSLanguage *load_language(const std::string &, const std::string &);
const TSLanguage *load_language(const std::string &, const TSCompileResult &);
const TSLanguage *get_test_language(const std::string &language_name);
#endif // HELPERS_LOAD_LANGUAGE_H_

91
spec/helpers/test_languages.cc
View file

@ -1,91 +0,0 @@
#include "helpers/test_languages.h"
#include "helpers/load_language.h"
#include <sys/stat.h>
#include <fstream>
#include <map>
#include <string>
using std::map;
using std::string;
using std::ifstream;
using std::ofstream;
using std::istreambuf_iterator;
map<string, const TSLanguage *> loaded_languages;
int libcompiler_mtime = -1;
const char libcompiler_path[] =
#if defined(__linux)
"out/Test/obj.target/libcompiler.a"
#else
"out/Test/libcompiler.a"
#endif
;
static int get_modified_time(const string &path) {
struct stat file_stat;
int error = stat(path.c_str(), &file_stat);
if (error != 0) {
fprintf(stderr, "Error in stat() for path: %s\n", + path.c_str());
return 0;
}
return file_stat.st_mtime;
}
const TSLanguage *get_test_language(const string &language_name) {
if (libcompiler_mtime == -1) {
libcompiler_mtime = get_modified_time(libcompiler_path);
if (!libcompiler_mtime) {
return nullptr;
}
}
if (loaded_languages[language_name]) {
return loaded_languages[language_name];
}
string language_dir = string("spec/fixtures/") + language_name;
string grammar_filename = language_dir + "/src/grammar.json";
string parser_filename = language_dir + "/src/parser.c";
int grammar_mtime = get_modified_time(grammar_filename);
if (!grammar_mtime) {
return nullptr;
}
int parser_mtime = get_modified_time(parser_filename);
if (!parser_mtime) {
return nullptr;
}
string parser_code;
if (parser_mtime <= grammar_mtime || parser_mtime <= libcompiler_mtime) {
printf("\n" "Regenerating the %s parser...\n", language_name.c_str());
ifstream grammar_file(grammar_filename);
istreambuf_iterator<char> grammar_file_iterator(grammar_file), end_iterator;
std::string grammar_json(grammar_file_iterator, end_iterator);
TSCompileResult result = ts_compile_grammar(grammar_json.c_str());
if (result.error_type != TSCompileErrorTypeNone) {
fprintf(stderr, "Failed to compile %s grammar: %s\n", language_name.c_str(), result.error_message);
return nullptr;
}
ofstream parser_file(parser_filename);
parser_file << result.code;
parser_code = result.code;
grammar_file.close();
parser_file.close();
} else {
ifstream parser_file(parser_filename);
istreambuf_iterator<char> grammar_file_iterator(parser_file), end_iterator;
parser_code.assign(grammar_file_iterator, end_iterator);
}
const TSLanguage *language = load_language(language_name, parser_code);
loaded_languages[language_name] = language;
return language;
};

9
spec/helpers/test_languages.h
View file

@ -1,9 +0,0 @@
#ifndef HELPERS_TEST_LANGUAGES_H_
#define HELPERS_TEST_LANGUAGES_H_
#include "tree_sitter/runtime.h"
#include <string>
const TSLanguage *get_test_language(const std::string &);
#endif // HELPERS_TEST_LANGUAGES_H_

14
spec/integration/compile_grammar_spec.cc
View file

@ -26,14 +26,14 @@ describe("compile_grammar", []() {
it("parses the token", [&]() {
TSCompileResult result = ts_compile_grammar(R"JSON(
{
"name": "test_language",
"name": "one_token_language",
"rules": {
"first_rule": {"type": "STRING", "value": "the-value"}
}
}
)JSON");
ts_document_set_language(document, load_language("test_language", result));
ts_document_set_language(document, load_language("one_token_language", result));
ts_document_set_input_string(document, "the-value");
ts_document_parse(document);
@ -45,14 +45,14 @@ describe("compile_grammar", []() {
it("parses the empty string", [&]() {
TSCompileResult result = ts_compile_grammar(R"JSON(
{
"name": "test_language",
"name": "blank_language",
"rules": {
"first_rule": {"type": "BLANK"}
}
}
)JSON");
ts_document_set_language(document, load_language("test_language", result));
ts_document_set_language(document, load_language("blank_language", result));
ts_document_set_input_string(document, "");
ts_document_parse(document);
@ -64,7 +64,7 @@ describe("compile_grammar", []() {
it("escapes the escaped characters properly in the generated parser", [&]() {
TSCompileResult result = ts_compile_grammar(R"JSON(
{
"name": "test_language",
"name": "escaped_char_language",
"rules": {
"first_rule": {
"type": "CHOICE",
@ -79,7 +79,7 @@ describe("compile_grammar", []() {
}
)JSON");
ts_document_set_language(document, load_language("test_language", result));
ts_document_set_language(document, load_language("escaped_char_language", result));
ts_document_set_input_string(document, "1234");
ts_document_parse(document);
@ -96,7 +96,7 @@ describe("compile_grammar", []() {
});
describe("the grammar in the README", [&]() {
it("works", [&]() {
it("parses the input in the README", [&]() {
TSCompileResult result = ts_compile_grammar(R"JSON(
{
"name": "arithmetic",

2
spec/integration/corpus_specs.cc
View file

@ -1,6 +1,6 @@
#include "spec_helper.h"
#include "runtime/alloc.h"
#include "helpers/test_languages.h"
#include "helpers/load_language.h"
#include "helpers/read_test_entries.h"
#include "helpers/spy_input.h"
#include "helpers/log_debugger.h"

2
spec/runtime/document_spec.cc
View file

@ -6,7 +6,7 @@
#include "helpers/tree_helpers.h"
#include "helpers/spy_debugger.h"
#include "helpers/spy_input.h"
#include "helpers/test_languages.h"
#include "helpers/load_language.h"
START_TEST

2
spec/runtime/node_spec.cc
View file

@ -2,7 +2,7 @@
#include "runtime/alloc.h"
#include "helpers/tree_helpers.h"
#include "helpers/point_helpers.h"
#include "helpers/test_languages.h"
#include "helpers/load_language.h"
#include "helpers/record_alloc.h"
#include "helpers/stream_methods.h"

2
spec/runtime/parser_spec.cc
View file

@ -2,7 +2,7 @@
#include "runtime/alloc.h"
#include "helpers/record_alloc.h"
#include "helpers/spy_input.h"
#include "helpers/test_languages.h"
#include "helpers/load_language.h"
#include "helpers/log_debugger.h"
#include "helpers/record_alloc.h"