Add test for handling of precedence within tokens

test/fixtures/test_grammars/precedence_on_token/corpus.txt

@@ -0,0 +1,22 @@
+==========================================
+simple strings
+==========================================
+
+"hi"
+
+---
+
+(program (string))
+
+==========================================
+strings starting with double slashes
+==========================================
+
+// comment
+"//not \t a \t comment"
+
+---
+
+(program
+  (comment)
+  (string (escape_sequence) (escape_sequence)))

test/fixtures/test_grammars/precedence_on_token/grammar.json

@@ -0,0 +1,61 @@
+{
+  "name": "precedence_on_token",
+
+  "extras": [
+    {"type": "SYMBOL", "name": "comment"},
+    {"type": "PATTERN", "value": "\\s"},
+  ],
+
+  "rules": {
+    "program": {
+      "type": "REPEAT",
+      "content": {
+        "type": "SYMBOL",
+        "name": "string"
+      }
+    },
+
+    "comment": {
+      "type": "PATTERN",
+      "value": "//.*"
+    },
+
+    "string": {
+      "type": "SEQ",
+      "members": [
+        {"type": "STRING", "value": "\""},
+
+        {
+          "type": "REPEAT",
+          "content": {
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "TOKEN",
+                "content": {
+                  "type": "PREC",
+                  "value": 1,
+                  "content": {
+                    "type": "PATTERN",
+                    "value": "[^\"\n\\\\]+"
+                  }
+                }
+              },
+              {
+                "type": "SYMBOL",
+                "name": "escape_sequence"
+              }
+            ]
+          }
+        },
+
+        {"type": "STRING", "value": "\""}
+      ]
+    },
+
+    "escape_sequence": {
+      "type": "PATTERN",
+      "value": "\\\\."
+    }
+  }
+}

test/integration/test_grammars.cc

@@ -9,8 +9,6 @@
 
 START_TEST
 
-if (TREE_SITTER_SEED == -1) return;
-
 string grammars_dir_path = join_path({"test", "fixtures", "test_grammars"});
 vector<string> test_languages = list_directory(grammars_dir_path);