traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix handling of ubiquitous tokens used in grammar rules

Browse source
This commit is contained in:
Max Brunsfeld 2014-07-01 20:47:35 -07:00
parent 59cc65c2e3
commit 83a1b9439e
30 changed files with 39086 additions and 32890 deletions
Download patch file Download diff file Expand all files Collapse all files

13
include/tree_sitter/compiler.h
View file

@ -2,6 +2,7 @@
#define TREE_SITTER_COMPILER_H_
#include <vector>
#include <set>
#include <string>
#include <memory>
@ -29,8 +30,8 @@ namespace tree_sitter {
class Grammar {
protected:
const std::vector<std::pair<std::string, rules::rule_ptr>> rules_;
std::vector<std::string> ubiquitous_tokens_;
std::vector<char> separators_;
std::set<std::string> ubiquitous_tokens_;
std::set<char> separators_;
public:
Grammar(const std::vector<std::pair<std::string, rules::rule_ptr>> &rules);
@ -39,10 +40,10 @@ namespace tree_sitter {
const rules::rule_ptr rule(const std::string &name) const;
const std::vector<std::pair<std::string, rules::rule_ptr>> & rules() const;
const std::vector<std::string> & ubiquitous_tokens() const;
Grammar & ubiquitous_tokens(const std::vector<std::string> &ubiquitous_tokens);
const std::vector<char> & separators() const;
Grammar & separators(const std::vector<char> &separators);
const std::set<std::string> & ubiquitous_tokens() const;
Grammar & ubiquitous_tokens(const std::set<std::string> &ubiquitous_tokens);
const std::set<char> & separators() const;
Grammar & separators(const std::set<char> &separators);
};
struct Conflict {

1
include/tree_sitter/parser.h
View file

@ -5,7 +5,6 @@
extern "C" {
#endif
//#define TS_DEBUG_PARSE
// #define TS_DEBUG_LEX
#include "tree_sitter/runtime.h"

4
include/tree_sitter/parser/state_machine.h
View file

@ -13,6 +13,7 @@ typedef enum {
TSParseActionTypeShift,
TSParseActionTypeShiftExtra,
TSParseActionTypeReduce,
TSParseActionTypeReduceExtra,
TSParseActionTypeAccept,
} TSParseActionType;
@ -33,6 +34,9 @@ typedef struct {
#define SHIFT_EXTRA() \
{ .type = TSParseActionTypeShiftExtra }
#define REDUCE_EXTRA(symbol_val) \
{ .type = TSParseActionTypeReduceExtra, .data = { .symbol = symbol_val } }
#define REDUCE(symbol_val, child_count_val) \
{ .type = TSParseActionTypeReduce, .data = { .symbol = symbol_val, .child_count = child_count_val } }