traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Allow ubiquitous tokens to also be used in grammar rules

Browse source
This commit is contained in:
Max Brunsfeld 2014-06-26 08:52:42 -07:00
parent a9dff20658
commit 9686c57e90
22 changed files with 49452 additions and 47887 deletions
Download patch file Download diff file Expand all files Collapse all files

10
include/tree_sitter/parser.h
View file

@ -6,7 +6,7 @@ extern "C" {
#endif
//#define TS_DEBUG_PARSE
//#define TS_DEBUG_LEX
// #define TS_DEBUG_LEX
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser/lexer.h"
@ -19,9 +19,6 @@ static const char *ts_symbol_names[]
#define HIDDEN_SYMBOLS \
static const int hidden_symbol_flags[SYMBOL_COUNT]
#define UBIQUITOUS_SYMBOLS \
static const int ubiquitous_symbol_flags[SYMBOL_COUNT]
#define LEX_STATES \
static ts_state_id ts_lex_states[STATE_COUNT]
@ -79,7 +76,7 @@ static const ts_tree * ts_parse(void *data, ts_input input, ts_input_edit *edit)
ts_lr_parser *parser = (ts_lr_parser *)data;
ts_lr_parser_initialize(parser, input, edit);
for (;;) {
ts_tree *tree = ts_lr_parser_parse(parser, ts_symbol_names);
const ts_tree *tree = ts_lr_parser_parse(parser, ts_symbol_names);
if (tree) return tree;
}
}
@ -95,8 +92,7 @@ ts_parser constructor_name() { \
(const ts_parse_action *)ts_parse_actions, \
ts_lex_states, \
ts_lex, \
hidden_symbol_flags, \
ubiquitous_symbol_flags \
hidden_symbol_flags \
), \
}; \
}

10
include/tree_sitter/parser/lr_parser.h
View file

@ -11,6 +11,7 @@ extern "C" {
typedef enum {
ts_parse_action_type_error,
ts_parse_action_type_shift,
ts_parse_action_type_shift_extra,
ts_parse_action_type_reduce,
ts_parse_action_type_accept,
} ts_parse_action_type;
@ -29,6 +30,9 @@ typedef struct {
#define SHIFT(to_state_value) \
{ .type = ts_parse_action_type_shift, .data = { .to_state = to_state_value } }
#define SHIFT_EXTRA() \
{ .type = ts_parse_action_type_shift_extra }
#define REDUCE(symbol_val, child_count_val) \
{ .type = ts_parse_action_type_reduce, .data = { .symbol = symbol_val, .child_count = child_count_val } }
@ -43,7 +47,6 @@ typedef struct {
struct {
size_t symbol_count;
const int *hidden_symbol_flags;
const int *ubiquitous_symbol_flags;
const ts_parse_action *parse_table;
const ts_state_id *lex_states;
ts_tree * (* lex_fn)(ts_lexer *, ts_state_id);
@ -54,8 +57,7 @@ ts_lr_parser * ts_lr_parser_make(size_t symbol_count,
const ts_parse_action *parse_table,
const ts_state_id *lex_states,
ts_tree * (* lex_fn)(ts_lexer *, ts_state_id),
const int *hidden_symbol_flags,
const int *ubiquitous_symbol_flags);
const int *hidden_symbol_flags);
void ts_lr_parser_free(void *data);
void ts_lr_parser_initialize(ts_lr_parser *parser, ts_input input, ts_input_edit *edit);
ts_tree * ts_lr_parser_parse(ts_lr_parser *parser, const char **symbol_names);
@ -64,4 +66,4 @@ ts_tree * ts_lr_parser_parse(ts_lr_parser *parser, const char **symbol_names);
}
#endif
#endif
#endif

5
include/tree_sitter/parser/stack.h
View file

@ -14,12 +14,13 @@ typedef struct {
struct {
ts_tree *node;
ts_state_id state;
int is_extra;
} *entries;
} ts_stack;
ts_stack ts_stack_make();
void ts_stack_delete(ts_stack *);
ts_tree * ts_stack_reduce(ts_stack *stack, ts_symbol symbol, size_t immediate_child_count, const int *hidden_symbol_flags, const int *ubiquitous_symbol_flags);
ts_tree * ts_stack_reduce(ts_stack *stack, ts_symbol symbol, size_t immediate_child_count, const int *hidden_symbol_flags, int gather_extras);
void ts_stack_shrink(ts_stack *stack, size_t new_size);
void ts_stack_push(ts_stack *stack, ts_state_id state, ts_tree *node);
ts_state_id ts_stack_top_state(const ts_stack *stack);
@ -30,4 +31,4 @@ size_t ts_stack_right_position(const ts_stack *stack);
}
#endif
#endif
#endif