Ignore external tokens that are zero-length and extra (#4213)

Co-authored-by: Anthony <anthony@zed.dev>

test/fixtures/test_grammars/epsilon_external_extra_tokens/corpus.txt

@@ -0,0 +1,9 @@
+==========================
+A document
+==========================
+
+a b
+
+---
+
+(document)

test/fixtures/test_grammars/epsilon_external_extra_tokens/grammar.js

@@ -0,0 +1,11 @@
+module.exports = grammar({
+    name: 'epsilon_external_extra_tokens',
+
+    extras: $ => [/\s/, $.comment],
+
+    externals: $ => [$.comment],
+
+    rules: {
+        document: $ => seq('a', 'b'),
+    }
+});

test/fixtures/test_grammars/epsilon_external_extra_tokens/scanner.c

@@ -0,0 +1,33 @@
+#include "tree_sitter/parser.h"
+
+enum TokenType {
+  COMMENT
+};
+
+void *tree_sitter_epsilon_external_extra_tokens_external_scanner_create(void) {
+  return NULL;
+}
+
+bool tree_sitter_epsilon_external_extra_tokens_external_scanner_scan(
+  void *payload,
+  TSLexer *lexer,
+  const bool *valid_symbols
+) {
+  lexer->result_symbol = COMMENT;
+  return true;
+}
+
+unsigned tree_sitter_epsilon_external_extra_tokens_external_scanner_serialize(
+  void *payload,
+  char *buffer
+) {
+  return 0;
+}
+
+void tree_sitter_epsilon_external_extra_tokens_external_scanner_deserialize(
+  void *payload,
+  const char *buffer,
+  unsigned length
+) {}
+
+void tree_sitter_epsilon_external_extra_tokens_external_scanner_destroy(void *payload) {}