Fix handling of zero-width external tokens at beginning of document

Fixes #316
This commit is contained in:
Max Brunsfeld 2019-04-08 15:01:52 -07:00
parent 701d63dab7
commit bc5e71bcc2
4 changed files with 64 additions and 0 deletions

View file

@ -0,0 +1,9 @@
==========================
A leading zero-width token
==========================
hello
---
(document (zero_width))

View file

@ -0,0 +1,21 @@
{
"name": "epsilon_external_tokens",
"externals": [
{"type": "SYMBOL", "name": "zero_width"}
],
"extras": [
{"type": "PATTERN", "value": "\\s"}
],
"rules": {
"document": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "zero_width"},
{"type": "STRING", "value": "hello"}
]
}
}
}

View file

@ -0,0 +1,33 @@
#include <tree_sitter/parser.h>
enum TokenType {
ZERO_WIDTH_TOKEN
};
void *tree_sitter_epsilon_external_tokens_external_scanner_create() {
return NULL;
}
bool tree_sitter_epsilon_external_tokens_external_scanner_scan(
void *payload,
TSLexer *lexer,
const bool *valid_symbols
) {
lexer->result_symbol = ZERO_WIDTH_TOKEN;
return true;
}
unsigned tree_sitter_epsilon_external_tokens_external_scanner_serialize(
void *payload,
char *buffer
) {
return 0;
}
void tree_sitter_epsilon_external_tokens_external_scanner_deserialize(
void *payload,
const char *buffer,
unsigned length
) {}
void tree_sitter_epsilon_external_tokens_external_scanner_destroy(void *payload) {}