traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Allow lexer to accept tokens that ended at previous positions

Browse source 
* Track lookahead in each tree
* Add 'mark_end' API that external scanners can use
This commit is contained in:
Max Brunsfeld 2017-03-13 17:03:47 -07:00
parent 12d2a9d93f
commit d222dbb9fd
12 changed files with 96 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/compiler/lex_table.cc
View file

@ -42,12 +42,9 @@ bool AcceptTokenAction::operator==(const AcceptTokenAction &other) const {
(is_string == other.is_string);
}
LexState::LexState() : is_token_start(false) {}
bool LexState::operator==(const LexState &other) const {
return advance_actions == other.advance_actions &&
accept_action == other.accept_action &&
is_token_start == other.is_token_start;
accept_action == other.accept_action;
}
} // namespace tree_sitter