Allow anonymous tokens to be used in grammars' external token lists

test/fixtures/test_grammars/external_and_internal_anonymous_tokens/corpus.txt

@@ -0,0 +1,41 @@
+=========================================
+single-line statements - internal tokens
+=========================================
+
+a b
+
+---
+
+(statement (variable) (variable))
+
+=========================================
+multi-line statements - internal tokens
+=========================================
+
+a
+b
+
+---
+
+(statement (variable) (variable))
+
+=========================================
+single-line statements - external tokens
+=========================================
+
+'hello' 'world'
+
+---
+
+(statement (string) (string))
+
+=========================================
+multi-line statements - external tokens
+=========================================
+
+'hello'
+'world'
+
+---
+
+(statement (string) (string))

test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.json

@@ -0,0 +1,35 @@
+{
+  "name": "external_and_internal_anonymous_tokens",
+
+  "externals": [
+    {"type": "SYMBOL", "name": "string"},
+    {"type": "STRING", "value": "\n"}
+  ],
+
+  "extras": [
+    {"type": "PATTERN", "value": "\\s"}
+  ],
+
+  "rules": {
+    "statement": {
+      "type": "SEQ",
+      "members": [
+        {"type": "SYMBOL", "name": "_expression"},
+        {"type": "SYMBOL", "name": "_expression"},
+        {"type": "STRING", "value": "\n"}
+      ]
+    },
+
+    "_expression": {
+      "type": "CHOICE",
+      "members": [
+        {"type": "SYMBOL", "name": "string"},
+        {"type": "SYMBOL", "name": "variable"},
+        {"type": "SYMBOL", "name": "number"}
+      ]
+    },
+
+    "variable": {"type": "PATTERN", "value": "\\a+"},
+    "number": {"type": "PATTERN", "value": "\\d+"}
+  }
+}

test/fixtures/test_grammars/external_and_internal_anonymous_tokens/readme.md

@@ -0,0 +1 @@
+This grammar is just like the `external_and_internal_tokens` grammar, except that the shared external token is *anonymous*; it's specified as a string in the grammar.

test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c

@@ -0,0 +1,23 @@
+#include "../external_and_internal_tokens/scanner.c"
+
+void *tree_sitter_external_and_internal_anonymous_tokens_external_scanner_create() { return NULL; }
+
+void tree_sitter_external_and_internal_anonymous_tokens_external_scanner_destroy(void *payload) {}
+
+void tree_sitter_external_and_internal_anonymous_tokens_external_scanner_reset(void *payload) {}
+
+bool tree_sitter_external_and_internal_anonymous_tokens_external_scanner_serialize(void *payload, TSExternalTokenState state) { return true; }
+
+void tree_sitter_external_and_internal_anonymous_tokens_external_scanner_deserialize(void *payload, TSExternalTokenState state) {}
+
+bool tree_sitter_external_and_internal_anonymous_tokens_external_scanner_scan(
+  void *payload,
+  TSLexer *lexer,
+  const bool *whitelist
+) {
+  return tree_sitter_external_and_internal_tokens_external_scanner_scan(
+    payload,
+    lexer,
+    whitelist
+  );
+}

test/fixtures/test_grammars/external_and_internal_tokens/grammar.json

@@ -2,8 +2,8 @@
   "name": "external_and_internal_tokens",
 
   "externals": [
-    "string",
-    "line_break"
+    {"type": "SYMBOL", "name": "string"},
+    {"type": "SYMBOL", "name": "line_break"}
   ],
 
   "extras": [

test/fixtures/test_grammars/external_extra_tokens/grammar.json

@@ -2,7 +2,7 @@
   "name": "external_extra_tokens",
 
   "externals": [
-    "comment"
+    {"type": "SYMBOL", "name": "comment"}
   ],
 
   "extras": [

test/fixtures/test_grammars/external_tokens/grammar.json

@@ -2,9 +2,9 @@
   "name": "external_tokens",
 
   "externals": [
-    "_percent_string",
-    "_percent_string_start",
-    "_percent_string_end"
+    {"type": "SYMBOL", "name": "_percent_string"},
+    {"type": "SYMBOL", "name": "_percent_string_start"},
+    {"type": "SYMBOL", "name": "_percent_string_end"}
   ],
 
   "extras": [