traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor generated lex function to use fewer instructions per state

Browse source
This commit is contained in:
Max Brunsfeld 2019-06-19 21:08:59 -07:00
parent 28011b1e60
commit e4873191d6
2 changed files with 20 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

11
cli/src/generate/render.rs
View file

@ -40,6 +40,7 @@ macro_rules! indent {
macro_rules! dedent {
($this: tt) => {
assert_ne!($this.indent_level, 0);
$this.indent_level -= 1;
};
}
@ -498,17 +499,16 @@ impl Generator {
add_whitespace!(self);
add!(self, "if (");
if self.add_character_set_condition(&characters, &ruled_out_characters) {
add!(self, ")\n");
indent!(self);
add!(self, ") ");
self.add_advance_action(index, &action);
if let CharacterSet::Include(chars) = characters {
ruled_out_characters.extend(chars.iter().map(|c| *c as u32));
}
dedent!(self);
} else {
self.buffer.truncate(previous_length);
self.add_advance_action(index, &action);
}
add!(self, "\n");
}
add_line!(self, "END_STATE();");
@ -620,10 +620,11 @@ impl Generator {
}
fn add_advance_action(&mut self, index: usize, action: &AdvanceAction) {
let state_id = action.state.unwrap_or(index);
if action.in_main_token {
add_line!(self, "ADVANCE({});", action.state.unwrap_or(index));
add!(self, "ADVANCE({});", state_id);
} else {
add_line!(self, "SKIP({});", action.state.unwrap_or(index));
add!(self, "SKIP({})", state_id);
}
}

24
lib/include/tree_sitter/parser.h
View file

@ -122,22 +122,26 @@ struct TSLanguage {
#define START_LEXER() \
bool result = false; \
bool skip = false; \
int32_t lookahead; \
goto start; \
next_state: \
lexer->advance(lexer, skip); \
start: \
skip = false; \
lookahead = lexer->lookahead;
#define ADVANCE(state_value) \
{ \
lexer->advance(lexer, false); \
state = state_value; \
goto next_state; \
#define ADVANCE(state_value) \
{ \
state = state_value; \
goto next_state; \
}
#define SKIP(state_value) \
{ \
lexer->advance(lexer, true); \
state = state_value; \
goto next_state; \
#define SKIP(state_value) \
{ \
skip = true; \
state = state_value; \
goto next_state; \
}
#define ACCEPT_TOKEN(symbol_value) \