traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Replace NodeType enum with SymbolMetadata bitfield

Browse source 
This will allow storing other metadata about symbols, like if they
only appear as ubiquitous tokens
This commit is contained in:
Max Brunsfeld 2015-11-22 13:32:20 -08:00
parent 53424699e4
commit f08554e958
17 changed files with 541 additions and 512 deletions
Download patch file Download diff file Expand all files Collapse all files

22
include/tree_sitter/parser.h
View file

@ -16,20 +16,18 @@ typedef struct TSTree TSTree;
typedef unsigned short TSStateId;
typedef enum {
TSNodeTypeHidden,
TSNodeTypeAnonymous,
TSNodeTypeNamed,
} TSNodeType;
typedef struct {
bool visible : 1;
bool named : 1;
} TSSymbolMetadata;
typedef struct TSLexer {
// Public
void (*start_fn)(struct TSLexer *, TSStateId);
void (*start_token_fn)(struct TSLexer *);
bool (*advance_fn)(struct TSLexer *, TSStateId);
TSTree *(*accept_fn)(struct TSLexer *, TSSymbol, TSNodeType, const char *);
TSTree *(*accept_fn)(struct TSLexer *, TSSymbol, TSSymbolMetadata,
const char *);
// Private
const char *chunk;
size_t chunk_start;
size_t chunk_size;
@ -74,7 +72,7 @@ typedef struct {
struct TSLanguage {
size_t symbol_count;
const char **symbol_names;
const TSNodeType *node_types;
const TSSymbolMetadata *symbol_metadata;
const TSParseAction **parse_table;
const TSStateId *lex_states;
TSTree *(*lex_fn)(TSLexer *, TSStateId);
@ -105,8 +103,8 @@ struct TSLanguage {
GO_TO_STATE(state_index); \
}
#define ACCEPT_TOKEN(symbol) \
return lexer->accept_fn(lexer, symbol, ts_node_types[symbol], \
#define ACCEPT_TOKEN(symbol) \
return lexer->accept_fn(lexer, symbol, ts_symbol_metadata[symbol], \
ts_symbol_names[symbol]);
#define LEX_ERROR() \
@ -161,7 +159,7 @@ struct TSLanguage {
#define EXPORT_LANGUAGE(language_name) \
static TSLanguage language = { \
.symbol_count = SYMBOL_COUNT, \
.node_types = ts_node_types, \
.symbol_metadata = ts_symbol_metadata, \
.parse_table = (const TSParseAction **)ts_parse_actions, \
.lex_states = ts_lex_states, \
.symbol_names = ts_symbol_names, \