Add a benchmark command

This command measures the speed of parsing each grammar's examples.
It also uses each grammar to parse all of the *other* grammars' examples
in order to measure error recovery performance with fairly large files.

include/tree_sitter/runtime.h

@@ -88,6 +88,7 @@ char *ts_node_string(TSNode, const TSDocument *);
 bool ts_node_eq(TSNode, TSNode);
 bool ts_node_is_named(TSNode);
 bool ts_node_has_changes(TSNode);
+bool ts_node_has_error(TSNode);
 TSNode ts_node_parent(TSNode);
 TSNode ts_node_child(TSNode, uint32_t);
 TSNode ts_node_named_child(TSNode, uint32_t);

script/benchmark

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -e
+
+mode=normal
+make -j2 benchmarks
+cmd=out/release/benchmarks
+
+while getopts "df:l:SL" option; do
+  case ${option} in
+    d)
+      mode=debug
+      ;;
+    f)
+      export TREE_SITTER_BENCHMARK_FILE_NAME=${OPTARG}
+      ;;
+    l)
+      export TREE_SITTER_BENCHMARK_LANGUAGE=${OPTARG}
+      ;;
+    L)
+      export TREE_SITTER_BENCHMARK_LOG=1
+      ;;
+    S)
+      mode=SVG
+      export TREE_SITTER_BENCHMARK_SVG=1
+      ;;
+  esac
+done
+
+case $mode in
+  debug)
+    lldb $cmd
+    ;;
+
+  SVG)
+    $cmd 2> >(grep -v 'Assertion failed' > test.dot)
+    ;;
+
+  normal)
+    exec $cmd
+    ;;
+esac

script/ci

@@ -7,3 +7,4 @@ set -e
 script/fetch-fixtures
 script/check-mallocs
 script/test -b
+script/benchmark

src/runtime/node.c

@@ -310,6 +310,10 @@ bool ts_node_has_changes(TSNode self) {
   return ts_node__tree(self)->has_changes;
 }
 
+bool ts_node_has_error(TSNode self) {
+  return ts_node__tree(self)->error_cost > 0;
+}
+
 TSNode ts_node_parent(TSNode self) {
   TSNode result = self;
   uint32_t index;

test/benchmarks.cc

@@ -0,0 +1,82 @@
+#include <cassert>
+#include <map>
+#include <vector>
+#include <ctime>
+#include <string>
+#include "tree_sitter/runtime.h"
+#include "helpers/load_language.h"
+#include "helpers/stderr_logger.h"
+#include "helpers/read_test_entries.h"
+
+using std::map;
+using std::vector;
+using std::string;
+
+vector<string> language_names({
+  "c",
+  "cpp",
+  "javascript",
+  "python",
+});
+
+int main(int argc, char *arg[]) {
+  map<string, vector<ExampleEntry>> example_entries_by_language_name;
+
+  auto document = ts_document_new();
+
+  if (getenv("TREE_SITTER_BENCHMARK_SVG")) {
+    ts_document_print_debugging_graphs(document, true);
+  } else if (getenv("TREE_SITTER_BENCHMARK_LOG")) {
+    ts_document_set_logger(document, stderr_logger_new(false));
+  }
+
+  auto language_filter = getenv("TREE_SITTER_BENCHMARK_LANGUAGE");
+  auto file_name_filter = getenv("TREE_SITTER_BENCHMARK_FILE_NAME");
+
+  for (auto &language_name : language_names) {
+    example_entries_by_language_name[language_name] = examples_for_language(language_name);
+  }
+
+  for (auto &language_name : language_names) {
+    if (language_filter && language_name != language_filter) continue;
+
+    ts_document_set_language(document, load_real_language(language_name));
+
+    printf("%s\n", language_name.c_str());
+
+    for (auto &example : example_entries_by_language_name[language_name]) {
+      if (file_name_filter && example.file_name != file_name_filter) continue;
+
+      ts_document_invalidate(document);
+      ts_document_set_input_string(document, example.input.c_str());
+
+      clock_t start_time = clock();
+      ts_document_parse(document);
+      clock_t end_time = clock();
+      unsigned duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC;
+      assert(!ts_node_has_error(ts_document_root_node(document)));
+      printf("  %-30s\t%u\n", example.file_name.c_str(), duration);
+    }
+
+    for (auto &other_language_name : language_names) {
+      if (other_language_name == language_name) continue;
+
+      for (auto &example : example_entries_by_language_name[other_language_name]) {
+        if (file_name_filter && example.file_name != file_name_filter) continue;
+
+        ts_document_invalidate(document);
+        ts_document_set_input_string(document, example.input.c_str());
+
+        clock_t start_time = clock();
+        ts_document_parse(document);
+        clock_t end_time = clock();
+        unsigned duration = (end_time - start_time) * 1000 / CLOCKS_PER_SEC;
+        printf("  %-30s\t%u\n", example.file_name.c_str(), duration);
+      }
+    }
+
+    puts("");
+  }
+
+  return 0;
+}

test/helpers/load_language.cc

@@ -1,6 +1,6 @@
-#include "test_helper.h"
 #include "helpers/load_language.h"
 #include "helpers/file_helpers.h"
+#include <cassert>
 #include <unistd.h>
 #include <dlfcn.h>
 #include <sys/types.h>
@@ -16,6 +16,8 @@ using std::string;
 using std::ifstream;
 using std::ofstream;
 using std::istreambuf_iterator;
+using std::vector;
+using std::to_string;
 
 map<string, const TSLanguage *> loaded_languages;
 int libcompiler_mtime = -1;
@@ -95,23 +97,21 @@ static const TSLanguage *load_language(const string &source_filename,
 
     string compile_error = run_command(compiler_name, compile_args.data());
     if (!compile_error.empty()) {
-      AssertThat(string(compile_error), IsEmpty());
-      return nullptr;
+      fputs(compile_error.c_str(), stderr);
+      abort();
     }
   }
 
   void *parser_lib = dlopen(lib_filename.c_str(), RTLD_NOW);
   if (!parser_lib) {
-    std::string message(dlerror());
-    AssertThat(message, IsEmpty());
-    return nullptr;
+    fputs(dlerror(), stderr);
+    abort();
   }
 
   void *language_function = dlsym(parser_lib, language_function_name.c_str());
   if (!language_function) {
-    std::string message(dlerror());
-    AssertThat(message, IsEmpty());
-    return nullptr;
+    fputs(dlerror(), stderr);
+    abort();
   }
 
   return reinterpret_cast<TSLanguage *(*)()>(language_function)();
@@ -121,8 +121,8 @@ const TSLanguage *load_test_language(const string &name,
                                      const TSCompileResult &compile_result,
                                      string external_scanner_path) {
   if (compile_result.error_type != TSCompileErrorTypeNone) {
-    Assert::Failure(string("Compilation failed ") + compile_result.error_message);
-    return nullptr;
+    fputs((string("Compilation failed ") + compile_result.error_message).c_str(), stderr);
+    abort();
   }
 
   mkdir("out/tmp", 0777);

test/helpers/random_helpers.cc

@@ -1,7 +1,9 @@
 #include <string>
+#include <vector>
 #include <stdlib.h>
 
 using std::string;
+using std::vector;
 
 static string random_string(char min, char max) {
   string result;
@@ -33,3 +35,7 @@ string random_words(size_t count) {
   }
   return result;
 }
+
+string select_random(const vector<string> &list) {
+  return list[random() % list.size()];
+}

test/helpers/random_helpers.h

@@ -2,7 +2,9 @@
 #define HELPERS_RANDOM_HELPERS_H_
 
 #include <string>
+#include <vector>
 
 std::string random_words(size_t count);
+std::string select_random(const std::vector<std::string> &);
 
 #endif  // HELPERS_RANDOM_HELPERS_H_

test/helpers/read_test_entries.cc

@@ -92,3 +92,15 @@ vector<TestEntry> read_test_language_corpus(string language_name) {
 
   return result;
 }
+
+vector<ExampleEntry> examples_for_language(string language_name) {
+  vector<ExampleEntry> result;
+  string examples_directory = fixtures_dir + "grammars/" + language_name + "/examples";
+  for (string &filename : list_directory(examples_directory)) {
+    result.push_back({
+      filename,
+      read_file(examples_directory + "/" + filename)
+    });
+  }
+  return result;
+}

test/helpers/read_test_entries.h

@@ -10,7 +10,13 @@ struct TestEntry {
 	std::string tree_string;
 };
 
+struct ExampleEntry {
+  std::string file_name;
+	std::string input;
+};
+
 std::vector<TestEntry> read_real_language_corpus(std::string name);
 std::vector<TestEntry> read_test_language_corpus(std::string name);
+std::vector<ExampleEntry> examples_for_language(std::string name);
 
 #endif

tests.gyp

@@ -1,5 +1,29 @@
 {
   'targets': [
+    {
+      'target_name': 'benchmarks',
+      'type': 'executable',
+      'dependencies': [
+        'project.gyp:runtime',
+        'project.gyp:compiler'
+      ],
+      'include_dirs': [
+        'src',
+        'test',
+        'externals/utf8proc',
+      ],
+      'sources': [
+        'test/benchmarks.cc',
+        'test/helpers/file_helpers.cc',
+        'test/helpers/load_language.cc',
+        'test/helpers/read_test_entries.cc',
+        'test/helpers/stderr_logger.cc',
+      ],
+      'configurations': {'Test': {}, 'Release': {}},
+      'default_configuration': 'Release',
+      'xcode_settings': {'CLANG_CXX_LANGUAGE_STANDARD': 'c++11'},
+      'cflags_cc': ['-std=c++11'],
+    },
     {
       'target_name': 'tests',
       'type': 'executable',