diff --git a/include/tree_sitter/runtime.h b/include/tree_sitter/runtime.h index 638bc5bd..503387f6 100644 --- a/include/tree_sitter/runtime.h +++ b/include/tree_sitter/runtime.h @@ -88,6 +88,7 @@ char *ts_node_string(TSNode, const TSDocument *); bool ts_node_eq(TSNode, TSNode); bool ts_node_is_named(TSNode); bool ts_node_has_changes(TSNode); +bool ts_node_has_error(TSNode); TSNode ts_node_parent(TSNode); TSNode ts_node_child(TSNode, uint32_t); TSNode ts_node_named_child(TSNode, uint32_t); diff --git a/script/benchmark b/script/benchmark new file mode 100755 index 00000000..68d64889 --- /dev/null +++ b/script/benchmark @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -e + +mode=normal +make -j2 benchmarks +cmd=out/release/benchmarks + +while getopts "df:l:SL" option; do + case ${option} in + d) + mode=debug + ;; + f) + export TREE_SITTER_BENCHMARK_FILE_NAME=${OPTARG} + ;; + l) + export TREE_SITTER_BENCHMARK_LANGUAGE=${OPTARG} + ;; + L) + export TREE_SITTER_BENCHMARK_LOG=1 + ;; + S) + mode=SVG + export TREE_SITTER_BENCHMARK_SVG=1 + ;; + esac +done + +case $mode in + debug) + lldb $cmd + ;; + + SVG) + $cmd 2> >(grep -v 'Assertion failed' > test.dot) + ;; + + normal) + exec $cmd + ;; +esac diff --git a/script/ci b/script/ci index 530981ef..6ada80e9 100755 --- a/script/ci +++ b/script/ci @@ -7,3 +7,4 @@ set -e script/fetch-fixtures script/check-mallocs script/test -b +script/benchmark diff --git a/src/runtime/node.c b/src/runtime/node.c index 15e2d5cf..84d4e3a7 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -310,6 +310,10 @@ bool ts_node_has_changes(TSNode self) { return ts_node__tree(self)->has_changes; } +bool ts_node_has_error(TSNode self) { + return ts_node__tree(self)->error_cost > 0; +} + TSNode ts_node_parent(TSNode self) { TSNode result = self; uint32_t index; diff --git a/test/benchmarks.cc b/test/benchmarks.cc new file mode 100644 index 00000000..07326735 --- /dev/null +++ b/test/benchmarks.cc @@ -0,0 +1,82 @@ +#include +#include +#include +#include +#include +#include "tree_sitter/runtime.h" +#include "helpers/load_language.h" +#include "helpers/stderr_logger.h" +#include "helpers/read_test_entries.h" + +using std::map; +using std::vector; +using std::string; + +vector language_names({ + "c", + "cpp", + "javascript", + "python", +}); + +int main(int argc, char *arg[]) { + map> example_entries_by_language_name; + + auto document = ts_document_new(); + + if (getenv("TREE_SITTER_BENCHMARK_SVG")) { + ts_document_print_debugging_graphs(document, true); + } else if (getenv("TREE_SITTER_BENCHMARK_LOG")) { + ts_document_set_logger(document, stderr_logger_new(false)); + } + + auto language_filter = getenv("TREE_SITTER_BENCHMARK_LANGUAGE"); + auto file_name_filter = getenv("TREE_SITTER_BENCHMARK_FILE_NAME"); + + for (auto &language_name : language_names) { + example_entries_by_language_name[language_name] = examples_for_language(language_name); + } + + for (auto &language_name : language_names) { + if (language_filter && language_name != language_filter) continue; + + ts_document_set_language(document, load_real_language(language_name)); + + printf("%s\n", language_name.c_str()); + + for (auto &example : example_entries_by_language_name[language_name]) { + if (file_name_filter && example.file_name != file_name_filter) continue; + + ts_document_invalidate(document); + ts_document_set_input_string(document, example.input.c_str()); + + clock_t start_time = clock(); + ts_document_parse(document); + clock_t end_time = clock(); + unsigned duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC; + assert(!ts_node_has_error(ts_document_root_node(document))); + printf(" %-30s\t%u\n", example.file_name.c_str(), duration); + } + + for (auto &other_language_name : language_names) { + if (other_language_name == language_name) continue; + + for (auto &example : example_entries_by_language_name[other_language_name]) { + if (file_name_filter && example.file_name != file_name_filter) continue; + + ts_document_invalidate(document); + ts_document_set_input_string(document, example.input.c_str()); + + clock_t start_time = clock(); + ts_document_parse(document); + clock_t end_time = clock(); + unsigned duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC; + printf(" %-30s\t%u\n", example.file_name.c_str(), duration); + } + } + + puts(""); + } + + return 0; +} diff --git a/test/helpers/load_language.cc b/test/helpers/load_language.cc index d057e130..6e2a6141 100644 --- a/test/helpers/load_language.cc +++ b/test/helpers/load_language.cc @@ -1,6 +1,6 @@ -#include "test_helper.h" #include "helpers/load_language.h" #include "helpers/file_helpers.h" +#include #include #include #include @@ -16,6 +16,8 @@ using std::string; using std::ifstream; using std::ofstream; using std::istreambuf_iterator; +using std::vector; +using std::to_string; map loaded_languages; int libcompiler_mtime = -1; @@ -95,23 +97,21 @@ static const TSLanguage *load_language(const string &source_filename, string compile_error = run_command(compiler_name, compile_args.data()); if (!compile_error.empty()) { - AssertThat(string(compile_error), IsEmpty()); - return nullptr; + fputs(compile_error.c_str(), stderr); + abort(); } } void *parser_lib = dlopen(lib_filename.c_str(), RTLD_NOW); if (!parser_lib) { - std::string message(dlerror()); - AssertThat(message, IsEmpty()); - return nullptr; + fputs(dlerror(), stderr); + abort(); } void *language_function = dlsym(parser_lib, language_function_name.c_str()); if (!language_function) { - std::string message(dlerror()); - AssertThat(message, IsEmpty()); - return nullptr; + fputs(dlerror(), stderr); + abort(); } return reinterpret_cast(language_function)(); @@ -121,8 +121,8 @@ const TSLanguage *load_test_language(const string &name, const TSCompileResult &compile_result, string external_scanner_path) { if (compile_result.error_type != TSCompileErrorTypeNone) { - Assert::Failure(string("Compilation failed ") + compile_result.error_message); - return nullptr; + fputs((string("Compilation failed ") + compile_result.error_message).c_str(), stderr); + abort(); } mkdir("out/tmp", 0777); diff --git a/test/helpers/random_helpers.cc b/test/helpers/random_helpers.cc index 1ce26400..b60519db 100644 --- a/test/helpers/random_helpers.cc +++ b/test/helpers/random_helpers.cc @@ -1,7 +1,9 @@ #include +#include #include using std::string; +using std::vector; static string random_string(char min, char max) { string result; @@ -33,3 +35,7 @@ string random_words(size_t count) { } return result; } + +string select_random(const vector &list) { + return list[random() % list.size()]; +} diff --git a/test/helpers/random_helpers.h b/test/helpers/random_helpers.h index 84aa1a23..0c2cc290 100644 --- a/test/helpers/random_helpers.h +++ b/test/helpers/random_helpers.h @@ -2,7 +2,9 @@ #define HELPERS_RANDOM_HELPERS_H_ #include +#include std::string random_words(size_t count); +std::string select_random(const std::vector &); #endif // HELPERS_RANDOM_HELPERS_H_ diff --git a/test/helpers/read_test_entries.cc b/test/helpers/read_test_entries.cc index a80d4571..733522ad 100644 --- a/test/helpers/read_test_entries.cc +++ b/test/helpers/read_test_entries.cc @@ -92,3 +92,15 @@ vector read_test_language_corpus(string language_name) { return result; } + +vector examples_for_language(string language_name) { + vector result; + string examples_directory = fixtures_dir + "grammars/" + language_name + "/examples"; + for (string &filename : list_directory(examples_directory)) { + result.push_back({ + filename, + read_file(examples_directory + "/" + filename) + }); + } + return result; +} diff --git a/test/helpers/read_test_entries.h b/test/helpers/read_test_entries.h index 3de397f1..016b19b4 100644 --- a/test/helpers/read_test_entries.h +++ b/test/helpers/read_test_entries.h @@ -10,7 +10,13 @@ struct TestEntry { std::string tree_string; }; +struct ExampleEntry { + std::string file_name; + std::string input; +}; + std::vector read_real_language_corpus(std::string name); std::vector read_test_language_corpus(std::string name); +std::vector examples_for_language(std::string name); #endif diff --git a/tests.gyp b/tests.gyp index bede6dfe..fbf696f6 100644 --- a/tests.gyp +++ b/tests.gyp @@ -1,5 +1,29 @@ { 'targets': [ + { + 'target_name': 'benchmarks', + 'type': 'executable', + 'dependencies': [ + 'project.gyp:runtime', + 'project.gyp:compiler' + ], + 'include_dirs': [ + 'src', + 'test', + 'externals/utf8proc', + ], + 'sources': [ + 'test/benchmarks.cc', + 'test/helpers/file_helpers.cc', + 'test/helpers/load_language.cc', + 'test/helpers/read_test_entries.cc', + 'test/helpers/stderr_logger.cc', + ], + 'configurations': {'Test': {}, 'Release': {}}, + 'default_configuration': 'Release', + 'xcode_settings': {'CLANG_CXX_LANGUAGE_STANDARD': 'c++11'}, + 'cflags_cc': ['-std=c++11'], + }, { 'target_name': 'tests', 'type': 'executable',