traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

In parse tables, store production ids as 16 bits

Browse source 
Also remove the use of bitfields from the parse table format.
In all cases, bitfields were not necessary to achieve the
current binary sizes. Avoiding them makes the binaries more
portable.

There was no way to make this change backward-compatible,
so we have finally dropped support for parsers generated
with an earlier version of Tree-sitter.

At some point, when Atom adopts this version of Tree-sitter,
this change will affect Atom users who have installed packages
using third-party Tree-sitter parsers. The packages will need
to be updated to use a regenerated version of the parsers.
This commit is contained in:
Max Brunsfeld 2021-02-25 16:12:29 -08:00
parent 63616289dd
commit 774ae5e3d5
7 changed files with 95 additions and 174 deletions
Download patch file Download diff file Expand all files Collapse all files

32
lib/src/parser.c
View file

@ -1012,15 +1012,15 @@ static bool ts_parser__do_all_potential_reductions(
switch (action.type) {
case TSParseActionTypeShift:
case TSParseActionTypeRecover:
if (!action.params.shift.extra && !action.params.shift.repetition) has_shift_action = true;
if (!action.shift.extra && !action.shift.repetition) has_shift_action = true;
break;
case TSParseActionTypeReduce:
if (action.params.reduce.child_count > 0)
if (action.reduce.child_count > 0)
ts_reduce_action_set_add(&self->reduce_actions, (ReduceAction){
.symbol = action.params.reduce.symbol,
.count = action.params.reduce.child_count,
.dynamic_precedence = action.params.reduce.dynamic_precedence,
.production_id = action.params.reduce.production_id,
.symbol = action.reduce.symbol,
.count = action.reduce.child_count,
.dynamic_precedence = action.reduce.dynamic_precedence,
.production_id = action.reduce.production_id,
});
break;
default:
@ -1311,7 +1311,7 @@ static void ts_parser__recover(
// be counted in error cost calculations.
unsigned n;
const TSParseAction *actions = ts_language_actions(self->language, 1, ts_subtree_symbol(lookahead), &n);
if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].params.shift.extra) {
if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].shift.extra) {
MutableSubtree mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead);
ts_subtree_set_extra(&mutable_lookahead);
lookahead = ts_subtree_from_mut(mutable_lookahead);
@ -1441,17 +1441,13 @@ static bool ts_parser__advance(
switch (action.type) {
case TSParseActionTypeShift: {
if (action.params.shift.repetition) break;
if (action.shift.repetition) break;
TSStateId next_state;
if (action.params.shift.extra) {
// TODO: remove when TREE_SITTER_LANGUAGE_VERSION 9 is out.
if (state == ERROR_STATE) continue;
if (action.shift.extra) {
next_state = state;
LOG("shift_extra");
} else {
next_state = action.params.shift.state;
next_state = action.shift.state;
LOG("shift state:%u", next_state);
}
@ -1460,7 +1456,7 @@ static bool ts_parser__advance(
next_state = ts_language_next_state(self->language, state, ts_subtree_symbol(lookahead));
}
ts_parser__shift(self, version, next_state, lookahead, action.params.shift.extra);
ts_parser__shift(self, version, next_state, lookahead, action.shift.extra);
if (did_reuse) reusable_node_advance(&self->reusable_node);
return true;
}
@ -1468,10 +1464,10 @@ static bool ts_parser__advance(
case TSParseActionTypeReduce: {
bool is_fragile = table_entry.action_count > 1;
bool end_of_non_terminal_extra = lookahead.ptr == NULL;
LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.params.reduce.symbol), action.params.reduce.child_count);
LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.reduce.symbol), action.reduce.child_count);
StackVersion reduction_version = ts_parser__reduce(
self, version, action.params.reduce.symbol, action.params.reduce.child_count,
action.params.reduce.dynamic_precedence, action.params.reduce.production_id,
self, version, action.reduce.symbol, action.reduce.child_count,
action.reduce.dynamic_precedence, action.reduce.production_id,
is_fragile, end_of_non_terminal_extra
);
if (reduction_version != STACK_VERSION_NONE) {