Fix handling of tokens consisting of separator characters
The parser is no longer hard-coded to skip whitespace. Tokens such as newlines, whose characters overlap with the separator characters, can now be correctly recognized.
This commit is contained in:
parent
f39cb1890d
commit
1cc7e32e2d
32 changed files with 5401 additions and 4847 deletions
|
|
@ -112,7 +112,7 @@ namespace tree_sitter_examples {
|
|||
str("]") }) },
|
||||
|
||||
// Keywords
|
||||
{ "_terminator", choice({ str(";"), str("\n") }) },
|
||||
{ "_terminator", pattern("[;\n]") },
|
||||
{ "_var", str("var") },
|
||||
{ "_for", str("for") },
|
||||
{ "_if", str("if") },
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue