traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

In lexer, prefer tokens to skipped separator characters

Browse source 
This was causing newlines in go and javascript to be parsed as
meaningless separator characters instead of statement terminators
This commit is contained in:
Max Brunsfeld 2014-05-30 13:29:54 -07:00
parent 220e081c49
commit e93e254518
26 changed files with 5559 additions and 6650 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/compiler/lex_table.cc
View file

@ -12,24 +12,25 @@ namespace tree_sitter {
LexAction::LexAction() :
type(LexActionTypeError),
symbol(Symbol(-1)),
state_index(-1) {}
state_index(-1),
precedence_values({0}) {}
LexAction::LexAction(LexActionType type, size_t state_index, Symbol symbol, int precedence) :
LexAction::LexAction(LexActionType type, size_t state_index, Symbol symbol, set<int> precedence_values) :
type(type),
symbol(symbol),
state_index(state_index),
precedence(precedence) {}
precedence_values(precedence_values) {}
LexAction LexAction::Error() {
return LexAction(LexActionTypeError, -1, Symbol(-1), 0);
return LexAction(LexActionTypeError, -1, Symbol(-1), {0});
}
LexAction LexAction::Advance(size_t state_index) {
return LexAction(LexActionTypeAdvance, state_index, Symbol(-1), 0);
LexAction LexAction::Advance(size_t state_index, set<int> precedence_values) {
return LexAction(LexActionTypeAdvance, state_index, Symbol(-1), precedence_values);
}
LexAction LexAction::Accept(Symbol symbol, int precedence) {
return LexAction(LexActionTypeAccept, -1, symbol, precedence);
return LexAction(LexActionTypeAccept, -1, symbol, { precedence });
}
bool LexAction::operator==(const LexAction &other) const {