traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refine logic for deciding when tokens need to be re-lexed

Browse source 
* While generating the lex table, note which tokens can match the
  same string. A token needs to be relexed when it has possible
  homonyms in the current state.
* Also note which tokens can match substrings of each other tokens.
  A token needs to be relexed when there are viable tokens that
  could match longer strings in the current state and the next
  token has been edited.
* Remove the logic for marking tokens as fragile on creation.
* Store the reusability/non-reusability of symbols off of individual
  actions and onto the entire entry for the state & symbol.
This commit is contained in:
Max Brunsfeld 2016-06-21 07:28:04 -07:00
parent 45f7cee0c8
commit 38c144b4a3
19 changed files with 337 additions and 257 deletions
Download patch file Download diff file Expand all files Collapse all files

36
src/compiler/parse_table.h
View file

@ -15,12 +15,12 @@ namespace tree_sitter {
typedef uint64_t ParseStateId;
typedef enum {
enum ParseActionType {
ParseActionTypeError,
ParseActionTypeShift,
ParseActionTypeReduce,
ParseActionTypeAccept,
} ParseActionType;
};
class ParseAction {
ParseAction(ParseActionType type, ParseStateId state_index,
@ -43,7 +43,6 @@ class ParseAction {
ParseActionType type;
bool extra;
bool fragile;
bool can_hide_split;
rules::Symbol symbol;
ParseStateId state_index;
size_t consumed_symbol_count;
@ -52,30 +51,16 @@ class ParseAction {
const Production *production;
};
} // namespace tree_sitter
struct ParseTableEntry {
std::vector<ParseAction> actions;
bool reusable;
bool depends_on_lookahead;
namespace std {
template <>
struct hash<tree_sitter::ParseAction> {
size_t operator()(const tree_sitter::ParseAction &action) const {
return (hash<int>()(action.type) ^
hash<tree_sitter::rules::Symbol>()(action.symbol) ^
hash<size_t>()(action.state_index) ^
hash<size_t>()(action.consumed_symbol_count) ^
hash<bool>()(action.extra) ^ hash<bool>()(action.fragile) ^
hash<bool>()(action.can_hide_split) ^
hash<int>()(action.associativity) ^
hash<int>()(action.precedence_range.min) ^
hash<int>()(action.precedence_range.max) ^
hash<const void *>()(&action.production));
}
ParseTableEntry();
ParseTableEntry(const std::vector<ParseAction> &, bool, bool);
bool operator==(const ParseTableEntry &other) const;
};
} // namespace std
namespace tree_sitter {
class ParseState {
public:
ParseState();
@ -83,11 +68,12 @@ class ParseState {
bool operator==(const ParseState &) const;
void each_advance_action(std::function<void(ParseAction *)>);
std::map<rules::Symbol, std::vector<ParseAction>> actions;
std::map<rules::Symbol, ParseTableEntry> entries;
LexStateId lex_state_id;
};
struct ParseTableSymbolMetadata {
bool extra;
bool structural;
};