traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make grammars' separator characters configurable

Browse source
This commit is contained in:
Max Brunsfeld 2014-06-26 07:31:08 -07:00
parent 8aea89750d
commit a9dff20658
5 changed files with 24 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/compiler/build_tables/build_lex_table.cc
View file

@ -93,11 +93,17 @@ namespace tree_sitter {
lex_table.state(state_id).is_token_start = true;
}
CharacterSet separator_set() const {
set<rules::CharacterRange> ranges;
for (char c : lex_grammar.separators)
ranges.insert(c);
return CharacterSet(ranges);
}
rules::rule_ptr after_separators(rules::rule_ptr rule) {
return rules::Seq::Build({
make_shared<rules::Metadata>(
make_shared<rules::Repeat>(
CharacterSet({ ' ', '\t', '\n', '\r' }).copy()),
make_shared<rules::Repeat>(separator_set().copy()),
map<rules::MetadataKey, int>({
{rules::START_TOKEN, 1},
{rules::PRECEDENCE, -1},

2
src/compiler/build_tables/build_parse_table.cc
View file

@ -34,8 +34,8 @@ namespace tree_sitter {
ParseStateId state_id = parse_table.add_state();
parse_state_ids[item_set] = state_id;
add_shift_actions(item_set, state_id);
add_ubiquitous_token_actions(item_set, state_id);
add_reduce_actions(item_set, state_id);
add_ubiquitous_token_actions(item_set, state_id);
return state_id;
} else {
return pair->second;