Fix handling of tokens consisting of separator characters

The parser is no longer hard-coded to skip whitespace. Tokens
such as newlines, whose characters overlap with the separator
characters, can now be correctly recognized.
This commit is contained in:
Max Brunsfeld 2014-04-03 19:10:09 -07:00
parent f39cb1890d
commit 1cc7e32e2d
32 changed files with 5401 additions and 4847 deletions

View file

@ -112,7 +112,7 @@ namespace tree_sitter_examples {
str("]") }) },
// Keywords
{ "_terminator", choice({ str(";"), str("\n") }) },
{ "_terminator", pattern("[;\n]") },
{ "_var", str("var") },
{ "_for", str("for") },
{ "_if", str("if") },