Fix handling of zero-width external tokens at beginning of document
Fixes #316
This commit is contained in:
parent
701d63dab7
commit
bc5e71bcc2
4 changed files with 64 additions and 0 deletions
|
|
@ -583,6 +583,7 @@ int ts_stack_dynamic_precedence(Stack *self, StackVersion version) {
|
|||
bool ts_stack_has_advanced_since_error(const Stack *self, StackVersion version) {
|
||||
const StackHead *head = array_get(&self->heads, version);
|
||||
const StackNode *node = head->node;
|
||||
if (node->error_cost == 0) return true;
|
||||
while (node) {
|
||||
if (node->link_count > 0) {
|
||||
Subtree subtree = node->links[0].subtree;
|
||||
|
|
|
|||
9
test/fixtures/test_grammars/epsilon_external_tokens/corpus.txt
vendored
Normal file
9
test/fixtures/test_grammars/epsilon_external_tokens/corpus.txt
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
==========================
|
||||
A leading zero-width token
|
||||
==========================
|
||||
|
||||
hello
|
||||
|
||||
---
|
||||
|
||||
(document (zero_width))
|
||||
21
test/fixtures/test_grammars/epsilon_external_tokens/grammar.json
vendored
Normal file
21
test/fixtures/test_grammars/epsilon_external_tokens/grammar.json
vendored
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"name": "epsilon_external_tokens",
|
||||
|
||||
"externals": [
|
||||
{"type": "SYMBOL", "name": "zero_width"}
|
||||
],
|
||||
|
||||
"extras": [
|
||||
{"type": "PATTERN", "value": "\\s"}
|
||||
],
|
||||
|
||||
"rules": {
|
||||
"document": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{"type": "SYMBOL", "name": "zero_width"},
|
||||
{"type": "STRING", "value": "hello"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
33
test/fixtures/test_grammars/epsilon_external_tokens/scanner.c
vendored
Normal file
33
test/fixtures/test_grammars/epsilon_external_tokens/scanner.c
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
#include <tree_sitter/parser.h>
|
||||
|
||||
enum TokenType {
|
||||
ZERO_WIDTH_TOKEN
|
||||
};
|
||||
|
||||
void *tree_sitter_epsilon_external_tokens_external_scanner_create() {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bool tree_sitter_epsilon_external_tokens_external_scanner_scan(
|
||||
void *payload,
|
||||
TSLexer *lexer,
|
||||
const bool *valid_symbols
|
||||
) {
|
||||
lexer->result_symbol = ZERO_WIDTH_TOKEN;
|
||||
return true;
|
||||
}
|
||||
|
||||
unsigned tree_sitter_epsilon_external_tokens_external_scanner_serialize(
|
||||
void *payload,
|
||||
char *buffer
|
||||
) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
void tree_sitter_epsilon_external_tokens_external_scanner_deserialize(
|
||||
void *payload,
|
||||
const char *buffer,
|
||||
unsigned length
|
||||
) {}
|
||||
|
||||
void tree_sitter_epsilon_external_tokens_external_scanner_destroy(void *payload) {}
|
||||
Loading…
Add table
Add a link
Reference in a new issue