traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Handle ambiguities between extra and non-extra tokens using normal GLR splitting

Browse source
This commit is contained in:
Max Brunsfeld 2016-09-06 10:22:16 -07:00
parent d31934ac77
commit b76574e01c
8 changed files with 63 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

26
src/compiler/build_tables/build_parse_table.cc
View file

@ -81,9 +81,6 @@ class ParseTableBuilder {
process_part_state_queue();
allow_any_conflict = false;
for (ParseStateId state = 0; state < parse_table.states.size(); state++)
add_reduce_extra_actions(state);
mark_fragile_actions();
remove_duplicate_parse_states();
@ -198,31 +195,10 @@ class ParseTableBuilder {
ParseAction action = ParseAction::ShiftExtra();
ParseState &state = parse_table.states[state_id];
for (const Symbol &extra_symbol : grammar.extra_tokens)
if (!state.entries.count(extra_symbol) ||
(allow_any_conflict &&
state.entries[extra_symbol].actions.back().type ==
ParseActionTypeReduce))
if (!state.entries.count(extra_symbol) || state.has_shift_action() || allow_any_conflict)
parse_table.add_action(state_id, extra_symbol, action);
}
void add_reduce_extra_actions(ParseStateId state_id) {
const ParseState &state = parse_table.states[state_id];
for (const Symbol &extra_symbol : grammar.extra_tokens) {
const auto &entry_for_symbol = state.entries.find(extra_symbol);
if (entry_for_symbol == state.entries.end())
continue;
for (const ParseAction &action : entry_for_symbol->second.actions)
if (action.type == ParseActionTypeShift && !action.extra) {
size_t dest_state_id = action.state_index;
ParseAction reduce_extra = ParseAction::ReduceExtra(extra_symbol);
for (const auto &pair : state.entries)
add_action(dest_state_id, pair.first, reduce_extra, null_item_set);
}
}
}
void mark_fragile_actions() {
for (ParseState &state : parse_table.states) {
set<Symbol> symbols_with_multiple_actions;

4
src/compiler/generate_code/c_code.cc
View file

@ -340,9 +340,7 @@ class CCodeGenerator {
}
break;
case ParseActionTypeReduce:
if (action.extra) {
add("REDUCE_EXTRA(" + symbol_id(action.symbol) + ")");
} else if (action.fragile) {
if (action.fragile) {
add("REDUCE_FRAGILE(" + symbol_id(action.symbol) + ", " +
to_string(action.consumed_symbol_count) + ")");
} else {

16
src/compiler/parse_table.cc
View file

@ -65,15 +65,6 @@ ParseAction ParseAction::ShiftExtra() {
return action;
}
ParseAction ParseAction::ReduceExtra(Symbol symbol) {
ParseAction action;
action.type = ParseActionTypeReduce;
action.extra = true;
action.symbol = symbol;
action.consumed_symbol_count = 1;
return action;
}
ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count,
int precedence,
rules::Associativity associativity,
@ -133,6 +124,13 @@ bool ParseTableEntry::operator==(const ParseTableEntry &other) const {
ParseState::ParseState() : lex_state_id(-1) {}
bool ParseState::has_shift_action() const {
for (const auto &pair : entries)
if (pair.second.actions.size() > 0 && pair.second.actions.back().type == ParseActionTypeShift)
return true;
return false;
}
set<Symbol> ParseState::expected_inputs() const {
set<Symbol> result;
for (auto &entry : entries)

2
src/compiler/parse_table.h
View file

@ -38,7 +38,6 @@ class ParseAction {
int precedence, rules::Associativity,
const Production &);
static ParseAction ShiftExtra();
static ParseAction ReduceExtra(rules::Symbol symbol);
bool operator==(const ParseAction &) const;
bool operator<(const ParseAction &) const;
@ -74,6 +73,7 @@ class ParseState {
bool operator==(const ParseState &) const;
bool merge(const ParseState &);
void each_advance_action(std::function<void(ParseAction *)>);
bool has_shift_action() const;
std::map<rules::Symbol, ParseTableEntry> entries;
LexStateId lex_state_id;

61
src/runtime/parser.c
View file

@ -472,7 +472,7 @@ static bool parser__switch_children(Parser *self, TSTree *tree,
}
static Reduction parser__reduce(Parser *self, StackVersion version,
TSSymbol symbol, unsigned count, bool extra,
TSSymbol symbol, unsigned count,
bool fragile, bool allow_skipping) {
size_t initial_version_count = ts_stack_version_count(self->stack);
StackPopResult pop = ts_stack_pop_count(self->stack, version, count);
@ -531,41 +531,34 @@ static Reduction parser__reduce(Parser *self, StackVersion version,
parent->parse_state = state;
}
TSStateId new_state;
if (extra) {
parent->extra = true;
new_state = state;
} else {
const TSParseAction *action =
ts_language_last_action(language, state, symbol);
assert(action->type == TSParseActionTypeShift ||
action->type == TSParseActionTypeRecover);
new_state = action->to_state;
const TSParseAction *action =
ts_language_last_action(language, state, symbol);
assert(action->type == TSParseActionTypeShift ||
action->type == TSParseActionTypeRecover);
if (action->type == TSParseActionTypeRecover && child_count > 1 &&
allow_skipping) {
StackVersion other_version =
ts_stack_duplicate_version(self->stack, slice.version);
CHECK(other_version != STACK_VERSION_NONE);
if (action->type == TSParseActionTypeRecover && child_count > 1 &&
allow_skipping) {
StackVersion other_version =
ts_stack_duplicate_version(self->stack, slice.version);
CHECK(other_version != STACK_VERSION_NONE);
CHECK(ts_stack_push(self->stack, other_version, parent, false,
CHECK(ts_stack_push(self->stack, other_version, parent, false,
TS_STATE_ERROR));
for (size_t j = parent->child_count; j < slice.trees.size; j++) {
TSTree *tree = slice.trees.contents[j];
CHECK(ts_stack_push(self->stack, other_version, tree, false,
TS_STATE_ERROR));
for (size_t j = parent->child_count; j < slice.trees.size; j++) {
TSTree *tree = slice.trees.contents[j];
CHECK(ts_stack_push(self->stack, other_version, tree, false,
TS_STATE_ERROR));
}
ErrorStatus error_status = ts_stack_error_status(self->stack, other_version);
if (parser__better_version_exists(self, version, error_status))
ts_stack_remove_version(self->stack, other_version);
}
ErrorStatus error_status = ts_stack_error_status(self->stack, other_version);
if (parser__better_version_exists(self, version, error_status))
ts_stack_remove_version(self->stack, other_version);
}
CHECK(parser__push(self, slice.version, parent, new_state));
CHECK(parser__push(self, slice.version, parent, action->to_state));
for (size_t j = parent->child_count; j < slice.trees.size; j++) {
TSTree *tree = slice.trees.contents[j];
CHECK(parser__push(self, slice.version, tree, new_state));
CHECK(parser__push(self, slice.version, tree, action->to_state));
}
}
@ -899,7 +892,7 @@ static PotentialReductionStatus parser__do_potential_reductions(
for (size_t i = 0; i < self->reduce_actions.size; i++) {
ReduceAction action = self->reduce_actions.contents[i];
Reduction reduction = parser__reduce(self, version, action.symbol,
action.count, false, true, false);
action.count, true, false);
switch (reduction.status) {
case ReduceFailed:
goto error;
@ -1128,15 +1121,11 @@ static bool parser__advance(Parser *self, StackVersion version,
if (reduction_stopped_at_error)
continue;
if (action.extra) {
LOG("reduce_extra");
} else {
LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.symbol),
action.child_count);
}
LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.symbol),
action.child_count);
Reduction reduction = parser__reduce(
self, version, action.symbol, action.child_count, action.extra,
self, version, action.symbol, action.child_count,
(i < table_entry.action_count - 1), true);
switch (reduction.status) {