Allow lexer to accept tokens that ended at previous positions

* Track lookahead in each tree
* Add 'mark_end' API that external scanners can use
This commit is contained in:
Max Brunsfeld 2017-03-13 17:03:47 -07:00
parent 12d2a9d93f
commit d222dbb9fd
12 changed files with 96 additions and 71 deletions

View file

@ -34,6 +34,7 @@ typedef struct Tree {
Length padding;
Length size;
uint32_t bytes_scanned;
TSSymbol symbol;
TSStateId parse_state;