traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Allow ubiquitous tokens to also be used in grammar rules

Browse source
This commit is contained in:
Max Brunsfeld 2014-06-26 08:52:42 -07:00
parent a9dff20658
commit 9686c57e90
22 changed files with 49452 additions and 47887 deletions
Download patch file Download diff file Expand all files Collapse all files

9
examples/grammars/javascript.cc
View file

@ -6,9 +6,9 @@ namespace tree_sitter_examples {
using namespace tree_sitter::rules;
static rule_ptr terminated(rule_ptr rule) {
return seq({ rule, prec(-1, token(choice({
str("\n"),
str(";") }))) });
return seq({ rule, choice({
sym("_line_break"),
str(";") }) });
}
extern const Grammar javascript = Grammar({
@ -186,12 +186,13 @@ namespace tree_sitter_examples {
{ "string", token(choice({
delimited("\""),
delimited("'") })) },
{ "_line_break", str("\n") },
{ "identifier", pattern("[\\a_$][\\w_$]*") },
{ "number", pattern("\\d+(\\.\\d+)?") },
{ "null", keyword("null") },
{ "true", keyword("true") },
{ "false", keyword("false") },
})
.ubiquitous_tokens({ "comment" })
.ubiquitous_tokens({ "comment", "_line_break" })
.separators({ ' ', '\t', '\r' });
}

3
examples/parsers/arithmetic.c
View file

@ -43,9 +43,6 @@ SYMBOL_NAMES = {
[ts_aux_sym_7] = "')'",
};
UBIQUITOUS_SYMBOLS = {
};
HIDDEN_SYMBOLS = {
[ts_aux_sym_1] = 1,
[ts_aux_sym_2] = 1,

782
examples/parsers/golang.c
View file

File diff suppressed because it is too large Load diff

96389
examples/parsers/javascript.c
View file

File diff suppressed because it is too large Load diff

3
examples/parsers/json.c
View file

@ -43,9 +43,6 @@ SYMBOL_NAMES = {
[ts_aux_sym_6] = "']'",
};
UBIQUITOUS_SYMBOLS = {
};
HIDDEN_SYMBOLS = {
[ts_aux_sym_object_repeat0] = 1,
[ts_aux_sym_array_repeat0] = 1,

10
include/tree_sitter/parser.h
View file

@ -6,7 +6,7 @@ extern "C" {
#endif
//#define TS_DEBUG_PARSE
//#define TS_DEBUG_LEX
// #define TS_DEBUG_LEX
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser/lexer.h"
@ -19,9 +19,6 @@ static const char *ts_symbol_names[]
#define HIDDEN_SYMBOLS \
static const int hidden_symbol_flags[SYMBOL_COUNT]
#define UBIQUITOUS_SYMBOLS \
static const int ubiquitous_symbol_flags[SYMBOL_COUNT]
#define LEX_STATES \
static ts_state_id ts_lex_states[STATE_COUNT]
@ -79,7 +76,7 @@ static const ts_tree * ts_parse(void *data, ts_input input, ts_input_edit *edit)
ts_lr_parser *parser = (ts_lr_parser *)data;
ts_lr_parser_initialize(parser, input, edit);
for (;;) {
ts_tree *tree = ts_lr_parser_parse(parser, ts_symbol_names);
const ts_tree *tree = ts_lr_parser_parse(parser, ts_symbol_names);
if (tree) return tree;
}
}
@ -95,8 +92,7 @@ ts_parser constructor_name() { \
(const ts_parse_action *)ts_parse_actions, \
ts_lex_states, \
ts_lex, \
hidden_symbol_flags, \
ubiquitous_symbol_flags \
hidden_symbol_flags \
), \
}; \
}

10
include/tree_sitter/parser/lr_parser.h
View file

@ -11,6 +11,7 @@ extern "C" {
typedef enum {
ts_parse_action_type_error,
ts_parse_action_type_shift,
ts_parse_action_type_shift_extra,
ts_parse_action_type_reduce,
ts_parse_action_type_accept,
} ts_parse_action_type;
@ -29,6 +30,9 @@ typedef struct {
#define SHIFT(to_state_value) \
{ .type = ts_parse_action_type_shift, .data = { .to_state = to_state_value } }
#define SHIFT_EXTRA() \
{ .type = ts_parse_action_type_shift_extra }
#define REDUCE(symbol_val, child_count_val) \
{ .type = ts_parse_action_type_reduce, .data = { .symbol = symbol_val, .child_count = child_count_val } }
@ -43,7 +47,6 @@ typedef struct {
struct {
size_t symbol_count;
const int *hidden_symbol_flags;
const int *ubiquitous_symbol_flags;
const ts_parse_action *parse_table;
const ts_state_id *lex_states;
ts_tree * (* lex_fn)(ts_lexer *, ts_state_id);
@ -54,8 +57,7 @@ ts_lr_parser * ts_lr_parser_make(size_t symbol_count,
const ts_parse_action *parse_table,
const ts_state_id *lex_states,
ts_tree * (* lex_fn)(ts_lexer *, ts_state_id),
const int *hidden_symbol_flags,
const int *ubiquitous_symbol_flags);
const int *hidden_symbol_flags);
void ts_lr_parser_free(void *data);
void ts_lr_parser_initialize(ts_lr_parser *parser, ts_input input, ts_input_edit *edit);
ts_tree * ts_lr_parser_parse(ts_lr_parser *parser, const char **symbol_names);
@ -64,4 +66,4 @@ ts_tree * ts_lr_parser_parse(ts_lr_parser *parser, const char **symbol_names);
}
#endif
#endif
#endif

5
include/tree_sitter/parser/stack.h
View file

@ -14,12 +14,13 @@ typedef struct {
struct {
ts_tree *node;
ts_state_id state;
int is_extra;
} *entries;
} ts_stack;
ts_stack ts_stack_make();
void ts_stack_delete(ts_stack *);
ts_tree * ts_stack_reduce(ts_stack *stack, ts_symbol symbol, size_t immediate_child_count, const int *hidden_symbol_flags, const int *ubiquitous_symbol_flags);
ts_tree * ts_stack_reduce(ts_stack *stack, ts_symbol symbol, size_t immediate_child_count, const int *hidden_symbol_flags, int gather_extras);
void ts_stack_shrink(ts_stack *stack, size_t new_size);
void ts_stack_push(ts_stack *stack, ts_state_id state, ts_tree *node);
ts_state_id ts_stack_top_state(const ts_stack *stack);
@ -30,4 +31,4 @@ size_t ts_stack_right_position(const ts_stack *stack);
}
#endif
#endif
#endif

6
spec/compiler/build_tables/build_parse_table_spec.cc
View file

@ -35,7 +35,7 @@ describe("building parse tables", []() {
{ Symbol(1, SymbolOptionToken), ParseAction::Shift(4, { 0 }) },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::Shift(0, { 0 }) },
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
@ -46,7 +46,7 @@ describe("building parse tables", []() {
{ END_OF_INPUT(), ParseAction::Accept() },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::Shift(1, { 0 }) },
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
@ -57,7 +57,7 @@ describe("building parse tables", []() {
{ END_OF_INPUT(), ParseAction::Reduce(Symbol(0), 1, 0) },
// for the ubiquitous_token 'token2'
{ Symbol(2, SymbolOptionToken), ParseAction::Shift(2, { 0 }) },
{ Symbol(2, SymbolOptionToken), ParseAction::ShiftExtra() },
})));
});
});

2
spec/compiler/compiler_specs.cc
View file

@ -5,7 +5,7 @@ int main(int argc, char *argv[]) {
"",
"--no-color",
"--only="
""
// "compiles the javascript"
};
return bandit::run(4, const_cast<char **>(args));
}

2
spec/runtime/languages/javascript/control_flow.txt
View file

@ -1,5 +1,5 @@
==========================================
parses if statements
js parses if statements
==========================================
if (isReady()) {
console.log(theData)

5
spec/runtime/languages/javascript/operators.txt
View file

@ -62,6 +62,7 @@ print(isDone() ? stuff : otherStuff);
parses mathematical operators
==========================================
-a + b * c - d / +e
---
(program (expression_statement
(math_op
@ -74,6 +75,7 @@ parses mathematical operators
parses boolean operators
=========================================
!a || !(b && c)
---
(program (expression_statement
(bool_op
@ -85,6 +87,7 @@ parses boolean operators
parses the type operators
===========================================
print((x instanceof Array) || (typeof x == "string"))
---
(program (expression_statement (function_call (identifier)
(bool_op
@ -95,6 +98,7 @@ print((x instanceof Array) || (typeof x == "string"))
parses the 'in' operator
===========================================
print(x in y)
---
(program (expression_statement (function_call
(identifier)
@ -104,6 +108,7 @@ print(x in y)
parses property access and operators
============================================
print(x.y.z && a.b.c)
---
(program (expression_statement (function_call (identifier)
(bool_op

3
spec/runtime/lr_parser_spec.cc
View file

@ -25,8 +25,7 @@ describe("LR Parsers", [&]() {
(const ts_parse_action *)dummy_parser.parse_table,
dummy_parser.lex_states,
fake_lex,
dummy_parser.hidden_symbols,
nullptr);
dummy_parser.hidden_symbols);
});
after_each([&]() {

10
spec/runtime/stack_spec.cc
View file

@ -65,12 +65,12 @@ describe("stacks", [&]() {
it("pops the given number of nodes off the stack", [&]() {
AssertThat(stack.size, Equals<size_t>(4));
ts_stack_reduce(&stack, sym2, 3, hidden_symbols, nullptr);
ts_stack_reduce(&stack, sym2, 3, hidden_symbols, 0);
AssertThat(stack.size, Equals<size_t>(1));
});
it("returns a node with the given symbol", [&]() {
ts_tree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols, nullptr);
ts_tree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols, 0);
AssertThat(ts_tree_symbol(node), Equals(sym2));
});
@ -81,7 +81,7 @@ describe("stacks", [&]() {
stack.entries[3].node,
};
ts_tree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols, nullptr);
ts_tree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols, 0);
size_t immediate_child_count;
ts_tree **immediate_children = ts_tree_immediate_children(node, &immediate_child_count);
@ -96,7 +96,7 @@ describe("stacks", [&]() {
stack.entries[3].node,
};
ts_tree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols, nullptr);
ts_tree *node = ts_stack_reduce(&stack, sym2, 3, hidden_symbols, 0);
size_t child_count;
ts_tree **children = ts_tree_children(node, &child_count);
@ -127,7 +127,7 @@ describe("stacks", [&]() {
});
it("makes those child nodes children of the new node", [&]() {
ts_tree *node = ts_stack_reduce(&stack, sym2, 4, hidden_symbols, nullptr);
ts_tree *node = ts_stack_reduce(&stack, sym2, 4, hidden_symbols, 0);
ts_tree *expected_children[4] = {
stack.entries[1].node,

2
src/compiler/build_tables/build_parse_table.cc
View file

@ -62,7 +62,7 @@ namespace tree_sitter {
for (const Symbol &symbol : grammar.ubiquitous_tokens) {
auto &actions = parse_table.states[state_id].actions;
if (actions.find(symbol) == actions.end())
parse_table.add_action(state_id, symbol, ParseAction::Shift(state_id, { 0 }));
parse_table.add_action(state_id, symbol, ParseAction::ShiftExtra());
}
}

14
src/compiler/generate_code/c_code.cc
View file

@ -51,7 +51,6 @@ namespace tree_sitter {
state_and_symbol_counts();
symbol_enum();
symbol_names_list();
ubiquitous_symbols_list();
hidden_symbols_list();
lex_function();
lex_states_list();
@ -104,16 +103,6 @@ namespace tree_sitter {
line();
}
void ubiquitous_symbols_list() {
line("UBIQUITOUS_SYMBOLS = {");
indent([&]() {
for (auto &symbol : syntax_grammar.ubiquitous_tokens)
line("[" + symbol_id(symbol) + "] = 1,");
});
line("};");
line();
}
void hidden_symbols_list() {
line("HIDDEN_SYMBOLS = {");
indent([&]() {
@ -293,6 +282,9 @@ namespace tree_sitter {
case ParseActionTypeShift:
add("SHIFT(" + to_string(action.state_index) + ")");
break;
case ParseActionTypeShiftExtra:
add("SHIFT_EXTRA()");
break;
case ParseActionTypeReduce:
add("REDUCE(" +
symbol_id(action.symbol) + ", " +

6
src/compiler/parse_table.cc
View file

@ -38,6 +38,10 @@ namespace tree_sitter {
return ParseAction(ParseActionTypeShift, state_index, Symbol(-1), 0, precedence_values);
}
ParseAction ParseAction::ShiftExtra() {
return ParseAction(ParseActionTypeShiftExtra, -1, Symbol(-1), 0, set<int>({}));
}
ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count, int precedence) {
return ParseAction(ParseActionTypeReduce, -1, symbol, consumed_symbol_count, { precedence });
}
@ -58,6 +62,8 @@ namespace tree_sitter {
return stream << string("#<accept>");
case ParseActionTypeShift:
return stream << (string("#<shift ") + to_string(action.state_index) + ">");
case ParseActionTypeShiftExtra:
return stream << string("#<shift_extra");
case ParseActionTypeReduce:
return stream << (string("#<reduce sym") + to_string(action.symbol.index) + " " + to_string(action.consumed_symbol_count) + ">");
default:

2
src/compiler/parse_table.h
View file

@ -12,6 +12,7 @@ namespace tree_sitter {
typedef enum {
ParseActionTypeError,
ParseActionTypeShift,
ParseActionTypeShiftExtra,
ParseActionTypeReduce,
ParseActionTypeAccept,
} ParseActionType;
@ -27,6 +28,7 @@ namespace tree_sitter {
static ParseAction Accept();
static ParseAction Error();
static ParseAction Shift(size_t state_index, std::set<int> precedence_values);
static ParseAction ShiftExtra();
static ParseAction Reduce(rules::Symbol symbol, size_t consumed_symbol_count, int precedence);
bool operator==(const ParseAction &action) const;

23
src/runtime/lr_parser.c
View file

@ -1,4 +1,5 @@
#include "tree_sitter/parser/lr_parser.h"
#include "runtime/tree.h"
/*
* Private
@ -8,7 +9,8 @@ static const ts_parse_action * actions_for_state(ts_lr_parser *parser, ts_state_
return parser->config.parse_table + (state * parser->config.symbol_count);
}
void shift(ts_lr_parser *parser, ts_state_id parse_state) {
void shift(ts_lr_parser *parser, ts_state_id parse_state, int is_extra) {
parser->lookahead->is_extra = is_extra;
ts_stack_push(&parser->stack, parse_state, parser->lookahead);
parser->lookahead = parser->next_lookahead;
parser->next_lookahead = NULL;
@ -20,7 +22,7 @@ void reduce(ts_lr_parser *parser, ts_symbol symbol, size_t child_count) {
symbol,
child_count,
parser->config.hidden_symbol_flags,
parser->config.ubiquitous_symbol_flags);
1);
}
static size_t breakdown_stack(ts_lr_parser *parser, ts_input_edit *edit) {
@ -129,6 +131,7 @@ ts_tree * get_tree_root(ts_lr_parser *parser) {
stack->size--;
for (size_t i = 0; i < immediate_child_count; i++) {
ts_tree *child = immedate_children[i];
child->is_extra = 0;
ts_tree_retain(child);
ts_state_id state = ts_stack_top_state(stack);
ts_state_id next_state = actions_for_state(parser, state)[ts_tree_symbol(child)].data.to_state;
@ -136,10 +139,10 @@ ts_tree * get_tree_root(ts_lr_parser *parser) {
}
ts_tree *new_node = ts_stack_reduce(stack,
ts_tree_symbol(top_node),
top_node->symbol,
stack->size,
parser->config.hidden_symbol_flags,
NULL);
0);
ts_tree_release(top_node);
return new_node;
}
@ -159,8 +162,7 @@ ts_lr_parser * ts_lr_parser_make(size_t symbol_count,
const ts_parse_action *parse_table,
const ts_state_id *lex_states,
ts_tree * (* lex_fn)(ts_lexer *, ts_state_id),
const int *hidden_symbol_flags,
const int *ubiquitous_symbol_flags) {
const int *hidden_symbol_flags) {
ts_lr_parser *result = malloc(sizeof(ts_lr_parser));
*result = (ts_lr_parser) {
.lexer = ts_lexer_make(),
@ -171,7 +173,6 @@ ts_lr_parser * ts_lr_parser_make(size_t symbol_count,
.lex_states = lex_states,
.lex_fn = lex_fn,
.hidden_symbol_flags = hidden_symbol_flags,
.ubiquitous_symbol_flags = ubiquitous_symbol_flags,
},
};
return result;
@ -198,6 +199,8 @@ void ts_lr_parser_initialize(ts_lr_parser *parser, ts_input input, ts_input_edit
ts_lexer_advance(&parser->lexer);
}
/* #define TS_DEBUG_PARSE */
#ifdef TS_DEBUG_PARSE
#include <stdio.h>
#define DEBUG_PARSE(...) fprintf(stderr, "\n" __VA_ARGS__)
@ -211,7 +214,11 @@ ts_tree * ts_lr_parser_parse(ts_lr_parser *parser, const char **symbol_names) {
switch (action.type) {
case ts_parse_action_type_shift:
DEBUG_PARSE("SHIFT %d", action.data.to_state);
shift(parser, action.data.to_state);
shift(parser, action.data.to_state, 0);
return NULL;
case ts_parse_action_type_shift_extra:
DEBUG_PARSE("SHIFT EXTRA");
shift(parser, ts_stack_top_state(&parser->stack), 1);
return NULL;
case ts_parse_action_type_reduce:
DEBUG_PARSE("REDUCE %s %d", symbol_names[action.data.symbol], action.data.child_count);

7
src/runtime/stack.c
View file

@ -1,5 +1,6 @@
#include "tree_sitter/runtime.h"
#include "tree_sitter/parser/stack.h"
#include "runtime/tree.h"
#include <string.h>
static size_t INITIAL_STACK_SIZE = 100;
@ -54,12 +55,12 @@ ts_tree * ts_stack_reduce(ts_stack *stack,
ts_symbol symbol,
size_t immediate_child_count,
const int *hidden_symbol_flags,
const int *ubiquitous_symbol_flags) {
int gather_extra) {
// First, walk down the stack to determine which symbols will be reduced.
// The child node count is known ahead of time, but some of the
// nodes at the top of the stack might be hidden nodes, in which
// case we 'collapse' them. Some may also be ubiquitous tokens,
// case we 'collapse' them. Some may also be extra tokens,
// which don't count towards the child node count.
static int collapse_flags[100];
int child_count = 0;
@ -77,7 +78,7 @@ ts_tree * ts_stack_reduce(ts_stack *stack,
child_count += collapse_flags[i] ? grandchild_count : 1;
if (ubiquitous_symbol_flags && ubiquitous_symbol_flags[child_symbol])
if (gather_extra && child->is_extra)
immediate_child_count++;
}

20
src/runtime/tree.c
View file

@ -1,25 +1,7 @@
#include "tree_sitter/runtime.h"
#include <string.h>
#include <stdio.h>
struct ts_tree {
ts_symbol symbol;
size_t ref_count;
size_t offset;
size_t size;
union {
struct {
size_t count;
size_t immediate_count;
struct ts_tree **contents;
} children;
struct {
char lookahead_char;
size_t expected_input_count;
const ts_symbol *expected_inputs;
} error;
} data;
};
#include "runtime/tree.h"
static ts_tree * ts_tree_make(ts_symbol symbol, size_t size, size_t offset) {
ts_tree *result = malloc(sizeof(ts_tree));

26
src/runtime/tree.h Normal file
View file

@ -0,0 +1,26 @@
#ifndef RUNTIME_TREE_H_
#define RUNTIME_TREE_H_
#include "tree_sitter/runtime.h"
struct ts_tree {
ts_symbol symbol;
size_t ref_count;
size_t offset;
size_t size;
int is_extra;
union {
struct {
size_t count;
size_t immediate_count;
struct ts_tree **contents;
} children;
struct {
char lookahead_char;
size_t expected_input_count;
const ts_symbol *expected_inputs;
} error;
} data;
};
#endif // RUNTIME_TREE_H_