Import error corpus entries from grammar repos

Now that error recovery requires no input for the grammar author, it shouldn't
be tested in the individual grammar repos.

.gitignore

@@ -11,5 +11,5 @@ externals/cpplint.py
 out
 *.pyc
 
-spec/fixtures/*
-!spec/fixtures/.gitkeep
+spec/fixtures/grammars/*
+!spec/fixtures/grammars/.gitkeep

script/fetch-fixtures

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-GRAMMARS_DIR=$(dirname $0)/../spec/fixtures
+GRAMMARS_DIR=$(dirname $0)/../spec/fixtures/grammars
 
 GRAMMARS=(
   javascript

spec/fixtures/error_corpus/c_errors.txt

@@ -0,0 +1,94 @@
+========================================
+Errors inside ifdefs
+========================================
+
+#ifdef something
+int x // no semicolon
+#endif
+
+int a;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int b;
+
+#ifdef __cplusplus
+}
+#endif
+
+int c;
+
+---
+
+(translation_unit
+  (preproc_ifdef
+    (identifier)
+    (ERROR (identifier) (identifier) (comment)))
+
+  (declaration (identifier) (identifier))
+
+  (preproc_ifdef
+    (identifier)
+    (ERROR (storage_class_specifier) (string_literal)))
+
+  (declaration (identifier) (identifier))
+
+  (preproc_ifdef
+    (identifier)
+    (ERROR))
+
+  (declaration (identifier) (identifier)))
+
+========================================
+Errors inside blocks
+========================================
+
+int main() {
+  int x;
+  int %$#@
+}
+
+---
+
+(translation_unit
+  (function_definition
+    (identifier)
+    (function_declarator (identifier))
+    (compound_statement
+      (declaration (identifier) (identifier))
+      (ERROR (identifier) (UNEXPECTED '$')))))
+
+========================================
+Errors inside expressions
+========================================
+
+int main() {
+  int x = (123 123);
+}
+
+---
+
+(translation_unit
+  (function_definition
+    (identifier)
+    (function_declarator (identifier))
+    (compound_statement
+      (declaration (identifier) (init_declarator
+        (identifier)
+        (ERROR (number_literal))
+        (number_literal))))))
+
+========================================
+Errors in declarations
+========================================
+
+float x WTF;
+int y = 5;
+
+---
+
+(translation_unit
+  (declaration (identifier) (identifier) (ERROR (identifier)))
+  (declaration (identifier) (init_declarator (identifier) (number_literal))))

spec/fixtures/error_corpus/javascript_errors.txt

@@ -0,0 +1,127 @@
+============================================
+Invalid statements
+============================================
+
+what the heck !
+y();
+
+if (x) {
+  >>>
+  var y = right
+}
+
+---
+
+(program
+  (expression_statement
+    (identifier)
+    (ERROR (identifier) (identifier)))
+  (expression_statement (function_call (identifier)))
+  (if_statement (identifier) (statement_block
+    (ERROR)
+    (var_declaration (var_assignment (identifier) (identifier))))))
+
+============================================
+Invalid if conditions
+============================================
+
+if (uh oh)
+  hmm();
+
+ok();
+
+---
+
+(program
+  (if_statement (identifier) (ERROR (identifier))
+    (expression_statement (function_call (identifier))))
+  (expression_statement (function_call (identifier))))
+
+============================================
+Invalid for loops
+============================================
+
+ok1;
+
+for (a b c; d; e)
+  wat();
+
+ok2;
+
+for (a; b; c d ef)
+  wat();
+
+---
+
+(program
+  (expression_statement (identifier))
+
+  (for_statement
+    (ERROR (identifier) (identifier))
+    (identifier)
+    (identifier)
+    (identifier)
+    (expression_statement (function_call (identifier))))
+
+  (expression_statement (identifier))
+
+  (for_statement
+    (identifier)
+    (identifier)
+    (ERROR (identifier) (identifier))
+    (identifier)
+    (expression_statement (function_call (identifier)))))
+
+============================================
+Invalid statement blocks
+============================================
+
+function() { ^ & * }
+
+---
+
+(program
+  (expression_statement (function (statement_block (ERROR)))))
+
+============================================
+Invalid objects
+============================================
+
+x = {
+  key1: value1,
+
+  abc efg,
+
+  key2: value2
+};
+
+---
+
+(program
+  (expression_statement (assignment
+    (identifier)
+    (object
+      (pair (identifier) (identifier))
+      (ERROR (identifier) (identifier))
+      (pair (identifier) (identifier))))))
+
+============================================
+Invalid items in var declarations
+============================================
+
+var
+  a = 1,
+  -b,
+  c = 2,
+  d = = = = =,
+  e;
+
+---
+
+(program
+  (var_declaration
+    (var_assignment (identifier) (number))
+    (ERROR (UNEXPECTED '-') (identifier))
+    (var_assignment (identifier) (number))
+    (ERROR (identifier) (UNEXPECTED '='))
+    (identifier)))

spec/fixtures/error_corpus/json_errors.txt

@@ -0,0 +1,56 @@
+==========================================
+top-level errors
+==========================================
+
+[}
+
+---
+
+(ERROR)
+
+==========================================
+unexpected tokens
+==========================================
+
+barf
+
+---
+
+(ERROR (UNEXPECTED 'b'))
+
+==========================================
+errors inside arrays
+==========================================
+
+[1, , 2]
+
+---
+(array
+  (number)
+  (ERROR)
+  (number))
+
+==========================================
+errors inside objects
+==========================================
+
+{ "key1": 1, oops }
+
+---
+
+(object (pair (string) (number)) (ERROR (UNEXPECTED 'o')))
+
+==========================================
+errors inside nested objects
+==========================================
+
+{ "key1": { "key2": 1, 2 }, [, "key3": 3 }
+
+---
+
+(object
+  (pair (string) (object
+    (pair (string) (number))
+    (ERROR (number))))
+  (ERROR)
+  (pair (string) (number)))

spec/helpers/load_language.cc

@@ -156,7 +156,7 @@ const TSLanguage *get_test_language(const string &language_name) {
       return nullptr;
   }
 
-  string language_dir = string("spec/fixtures/") + language_name;
+  string language_dir = string("spec/fixtures/grammars/") + language_name;
   string grammar_filename = language_dir + "/src/grammar.json";
   string parser_filename = language_dir + "/src/parser.c";
 

spec/helpers/read_test_entries.cc

@@ -38,7 +38,7 @@ static string trim_output(const string &input) {
   return result;
 }
 
-static vector<TestEntry> get_test_entries_from_string(string content) {
+static vector<TestEntry> parse_test_entries(string content) {
   regex header_pattern("===+\n"  "([^=]+)\n"  "===+", extended);
   regex separator_pattern("---+", extended);
   vector<string> descriptions;
@@ -94,16 +94,25 @@ static vector<string> list_directory(string dir_name) {
 	return result;
 }
 
-vector<TestEntry> read_corpus_entries(string directory) {
-  vector<TestEntry> result;
-  vector<string> filenames = list_directory(directory);
+static string read_file(string filename) {
+  ifstream file(filename);
+  string result((istreambuf_iterator<char>(file)), istreambuf_iterator<char>());
+  return result;
+}
 
-  for (string &filename : filenames) {
-    ifstream file(filename);
-    std::string content((istreambuf_iterator<char>(file)), istreambuf_iterator<char>());
-    for (TestEntry &entry : get_test_entries_from_string(content))
+vector<TestEntry> read_corpus_entries(string language_name) {
+  vector<TestEntry> result;
+
+  string fixtures_dir = "spec/fixtures/";
+
+  string test_directory = fixtures_dir + "grammars/" + language_name + "/grammar_test";
+  for (string &test_filename : list_directory(test_directory))
+    for (TestEntry &entry : parse_test_entries(read_file(test_filename)))
       result.push_back(entry);
-  }
+
+  string error_test_filename = fixtures_dir + "/error_corpus/" + language_name + "_errors.txt";
+  for (TestEntry &entry : parse_test_entries(read_file(error_test_filename)))
+    result.push_back(entry);
 
   return result;
 }

spec/integration/corpus_specs.cc

@@ -102,8 +102,6 @@ describe("The Corpus", []() {
   });
 
   for (auto &language_name : test_languages) {
-    string language_dir = string("spec/fixtures/") + language_name;
-
     describe(("the " + language_name + " language").c_str(), [&]() {
       TSDocument *document;
 
@@ -120,7 +118,7 @@ describe("The Corpus", []() {
         AssertThat(record_alloc::outstanding_allocation_indices(), IsEmpty());
       });
 
-      for (auto &entry : read_corpus_entries(language_dir + "/grammar_test")) {
+      for (auto &entry : read_corpus_entries(language_name)) {
         SpyInput *input;
 
         auto it_handles_edit_sequence = [&](string name, std::function<void()> edit_sequence){