Fix handling of zero-width external tokens at beginning of document

Fixes #316
This commit is contained in:
Max Brunsfeld 2019-04-08 15:01:52 -07:00
parent 701d63dab7
commit bc5e71bcc2
4 changed files with 64 additions and 0 deletions

View file

@ -583,6 +583,7 @@ int ts_stack_dynamic_precedence(Stack *self, StackVersion version) {
bool ts_stack_has_advanced_since_error(const Stack *self, StackVersion version) {
const StackHead *head = array_get(&self->heads, version);
const StackNode *node = head->node;
if (node->error_cost == 0) return true;
while (node) {
if (node->link_count > 0) {
Subtree subtree = node->links[0].subtree;

View file

@ -0,0 +1,9 @@
==========================
A leading zero-width token
==========================
hello
---
(document (zero_width))

View file

@ -0,0 +1,21 @@
{
"name": "epsilon_external_tokens",
"externals": [
{"type": "SYMBOL", "name": "zero_width"}
],
"extras": [
{"type": "PATTERN", "value": "\\s"}
],
"rules": {
"document": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "zero_width"},
{"type": "STRING", "value": "hello"}
]
}
}
}

View file

@ -0,0 +1,33 @@
#include <tree_sitter/parser.h>
enum TokenType {
ZERO_WIDTH_TOKEN
};
void *tree_sitter_epsilon_external_tokens_external_scanner_create() {
return NULL;
}
bool tree_sitter_epsilon_external_tokens_external_scanner_scan(
void *payload,
TSLexer *lexer,
const bool *valid_symbols
) {
lexer->result_symbol = ZERO_WIDTH_TOKEN;
return true;
}
unsigned tree_sitter_epsilon_external_tokens_external_scanner_serialize(
void *payload,
char *buffer
) {
return 0;
}
void tree_sitter_epsilon_external_tokens_external_scanner_deserialize(
void *payload,
const char *buffer,
unsigned length
) {}
void tree_sitter_epsilon_external_tokens_external_scanner_destroy(void *payload) {}