traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Allow ubiquitous tokens to also be used in grammar rules

Browse source
This commit is contained in:
Max Brunsfeld 2014-06-26 08:52:42 -07:00
parent a9dff20658
commit 9686c57e90
22 changed files with 49452 additions and 47887 deletions
Download patch file Download diff file Expand all files Collapse all files

23
src/runtime/lr_parser.c
View file

@ -1,4 +1,5 @@
#include "tree_sitter/parser/lr_parser.h"
#include "runtime/tree.h"
/*
* Private
@ -8,7 +9,8 @@ static const ts_parse_action * actions_for_state(ts_lr_parser *parser, ts_state_
return parser->config.parse_table + (state * parser->config.symbol_count);
}
void shift(ts_lr_parser *parser, ts_state_id parse_state) {
void shift(ts_lr_parser *parser, ts_state_id parse_state, int is_extra) {
parser->lookahead->is_extra = is_extra;
ts_stack_push(&parser->stack, parse_state, parser->lookahead);
parser->lookahead = parser->next_lookahead;
parser->next_lookahead = NULL;
@ -20,7 +22,7 @@ void reduce(ts_lr_parser *parser, ts_symbol symbol, size_t child_count) {
symbol,
child_count,
parser->config.hidden_symbol_flags,
parser->config.ubiquitous_symbol_flags);
1);
}
static size_t breakdown_stack(ts_lr_parser *parser, ts_input_edit *edit) {
@ -129,6 +131,7 @@ ts_tree * get_tree_root(ts_lr_parser *parser) {
stack->size--;
for (size_t i = 0; i < immediate_child_count; i++) {
ts_tree *child = immedate_children[i];
child->is_extra = 0;
ts_tree_retain(child);
ts_state_id state = ts_stack_top_state(stack);
ts_state_id next_state = actions_for_state(parser, state)[ts_tree_symbol(child)].data.to_state;
@ -136,10 +139,10 @@ ts_tree * get_tree_root(ts_lr_parser *parser) {
}
ts_tree *new_node = ts_stack_reduce(stack,
ts_tree_symbol(top_node),
top_node->symbol,
stack->size,
parser->config.hidden_symbol_flags,
NULL);
0);
ts_tree_release(top_node);
return new_node;
}
@ -159,8 +162,7 @@ ts_lr_parser * ts_lr_parser_make(size_t symbol_count,
const ts_parse_action *parse_table,
const ts_state_id *lex_states,
ts_tree * (* lex_fn)(ts_lexer *, ts_state_id),
const int *hidden_symbol_flags,
const int *ubiquitous_symbol_flags) {
const int *hidden_symbol_flags) {
ts_lr_parser *result = malloc(sizeof(ts_lr_parser));
*result = (ts_lr_parser) {
.lexer = ts_lexer_make(),
@ -171,7 +173,6 @@ ts_lr_parser * ts_lr_parser_make(size_t symbol_count,
.lex_states = lex_states,
.lex_fn = lex_fn,
.hidden_symbol_flags = hidden_symbol_flags,
.ubiquitous_symbol_flags = ubiquitous_symbol_flags,
},
};
return result;
@ -198,6 +199,8 @@ void ts_lr_parser_initialize(ts_lr_parser *parser, ts_input input, ts_input_edit
ts_lexer_advance(&parser->lexer);
}
/* #define TS_DEBUG_PARSE */
#ifdef TS_DEBUG_PARSE
#include <stdio.h>
#define DEBUG_PARSE(...) fprintf(stderr, "\n" __VA_ARGS__)
@ -211,7 +214,11 @@ ts_tree * ts_lr_parser_parse(ts_lr_parser *parser, const char **symbol_names) {
switch (action.type) {
case ts_parse_action_type_shift:
DEBUG_PARSE("SHIFT %d", action.data.to_state);
shift(parser, action.data.to_state);
shift(parser, action.data.to_state, 0);
return NULL;
case ts_parse_action_type_shift_extra:
DEBUG_PARSE("SHIFT EXTRA");
shift(parser, ts_stack_top_state(&parser->stack), 1);
return NULL;
case ts_parse_action_type_reduce:
DEBUG_PARSE("REDUCE %s %d", symbol_names[action.data.symbol], action.data.child_count);

7
src/runtime/stack.c
View file

@ -1,5 +1,6 @@
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser/stack.h"
#include "runtime/tree.h"
#include <string.h>
static size_t INITIAL_STACK_SIZE = 100;
@ -54,12 +55,12 @@ ts_tree * ts_stack_reduce(ts_stack *stack,
ts_symbol symbol,
size_t immediate_child_count,
const int *hidden_symbol_flags,
const int *ubiquitous_symbol_flags) {
int gather_extra) {
// First, walk down the stack to determine which symbols will be reduced.
// The child node count is known ahead of time, but some of the
// nodes at the top of the stack might be hidden nodes, in which
// case we 'collapse' them. Some may also be ubiquitous tokens,
// case we 'collapse' them. Some may also be extra tokens,
// which don't count towards the child node count.
static int collapse_flags[100];
int child_count = 0;
@ -77,7 +78,7 @@ ts_tree * ts_stack_reduce(ts_stack *stack,
child_count += collapse_flags[i] ? grandchild_count : 1;
if (ubiquitous_symbol_flags && ubiquitous_symbol_flags[child_symbol])
if (gather_extra && child->is_extra)
immediate_child_count++;
}

20
src/runtime/tree.c
View file

@ -1,25 +1,7 @@
#include "tree_sitter/runtime.h"
#include <string.h>
#include <stdio.h>
struct ts_tree {
ts_symbol symbol;
size_t ref_count;
size_t offset;
size_t size;
union {
struct {
size_t count;
size_t immediate_count;
struct ts_tree **contents;
} children;
struct {
char lookahead_char;
size_t expected_input_count;
const ts_symbol *expected_inputs;
} error;
} data;
};
#include "runtime/tree.h"
static ts_tree * ts_tree_make(ts_symbol symbol, size_t size, size_t offset) {
ts_tree *result = malloc(sizeof(ts_tree));

26
src/runtime/tree.h Normal file
View file

@ -0,0 +1,26 @@
#ifndef RUNTIME_TREE_H_
#define RUNTIME_TREE_H_
#include "tree_sitter/runtime.h"
struct ts_tree {
ts_symbol symbol;
size_t ref_count;
size_t offset;
size_t size;
int is_extra;
union {
struct {
size_t count;
size_t immediate_count;
struct ts_tree **contents;
} children;
struct {
char lookahead_char;
size_t expected_input_count;
const ts_symbol *expected_inputs;
} error;
} data;
};
#endif // RUNTIME_TREE_H_