Put public headers in tree_sitter directory
This commit is contained in:
Max Brunsfeld
parent
84b8addb63
commit
d09fa910ef
25 changed files with 45 additions and 35 deletions
74
include/tree_sitter/compiler.h
Normal file
74
include/tree_sitter/compiler.h
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
#ifndef tree_sitter_compiler_h
|
||||
#define tree_sitter_compiler_h
|
||||
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <string>
|
||||
|
||||
namespace tree_sitter {
|
||||
namespace rules {
|
||||
class Rule;
|
||||
class Symbol;
|
||||
|
||||
struct CharacterRange {
|
||||
char min;
|
||||
char max;
|
||||
CharacterRange(char);
|
||||
CharacterRange(char, char);
|
||||
bool operator==(const CharacterRange &) const;
|
||||
bool operator<(const CharacterRange &) const;
|
||||
std::string to_string() const;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
namespace std {
|
||||
template<>
|
||||
struct hash<tree_sitter::rules::CharacterRange> {
|
||||
size_t operator()(const tree_sitter::rules::CharacterRange &range) const {
|
||||
return (hash<char>()(range.min) ^ hash<char>()(range.max));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
namespace tree_sitter {
|
||||
namespace rules {
|
||||
typedef std::shared_ptr<Rule> rule_ptr;
|
||||
std::ostream& operator<<(std::ostream& stream, const rule_ptr &rule);
|
||||
|
||||
rule_ptr blank();
|
||||
rule_ptr character(const std::set<CharacterRange> &matches);
|
||||
rule_ptr character(const std::set<CharacterRange> &matches, bool);
|
||||
rule_ptr choice(const std::vector<rule_ptr> &rules);
|
||||
rule_ptr pattern(const std::string &value);
|
||||
rule_ptr repeat(const rule_ptr content);
|
||||
rule_ptr seq(const std::vector<rule_ptr> &rules);
|
||||
rule_ptr str(const std::string &value);
|
||||
rule_ptr sym(const std::string &name);
|
||||
rule_ptr aux_sym(const std::string &name);
|
||||
}
|
||||
}
|
||||
|
||||
namespace tree_sitter {
|
||||
class Grammar {
|
||||
typedef std::map<const std::string, const rules::rule_ptr> rule_map;
|
||||
public:
|
||||
Grammar(std::string start_rule_name, const rule_map &rules);
|
||||
Grammar(std::string start_rule_name, const rule_map &rules, const rule_map &aux_rules);
|
||||
|
||||
bool operator==(const Grammar &other) const;
|
||||
bool has_definition(const rules::Symbol &symbol) const;
|
||||
const rules::rule_ptr rule(const rules::Symbol &symbol) const;
|
||||
|
||||
const std::string start_rule_name;
|
||||
const rule_map rules;
|
||||
const rule_map aux_rules;
|
||||
};
|
||||
|
||||
std::ostream& operator<<(std::ostream &stream, const Grammar &grammar);
|
||||
|
||||
std::string compile(const Grammar &grammar, std::string name);
|
||||
}
|
||||
|
||||
#endif
|
||||
224
include/tree_sitter/parser.h
Normal file
224
include/tree_sitter/parser.h
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
#ifndef tree_sitter_parser_h
|
||||
#define tree_sitter_parser_h
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "./runtime.h"
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
//#define TS_DEBUG_PARSE
|
||||
//#define TS_DEBUG_LEX
|
||||
|
||||
#ifdef TS_DEBUG_LEX
|
||||
#define DEBUG_LEX(...) fprintf(stderr, __VA_ARGS__)
|
||||
#else
|
||||
#define DEBUG_LEX(...)
|
||||
#endif
|
||||
|
||||
#ifdef TS_DEBUG_PARSE
|
||||
#define DEBUG_PARSE(...) fprintf(stderr, __VA_ARGS__)
|
||||
#else
|
||||
#define DEBUG_PARSE(...)
|
||||
#endif
|
||||
|
||||
static int INITIAL_STACK_SIZE = 100;
|
||||
static const char *ts_symbol_names[];
|
||||
|
||||
typedef int TSState;
|
||||
|
||||
typedef struct {
|
||||
TSState state;
|
||||
TSTree *node;
|
||||
} TSStackEntry;
|
||||
|
||||
typedef struct {
|
||||
const char *input;
|
||||
size_t position;
|
||||
TSTree *lookahead_node;
|
||||
TSTree *prev_lookahead_node;
|
||||
TSState lex_state;
|
||||
TSStackEntry *stack;
|
||||
size_t stack_size;
|
||||
TSParseResult result;
|
||||
} TSParser;
|
||||
|
||||
static TSParser TSParserMake(const char *input) {
|
||||
TSParser result = {
|
||||
.input = input,
|
||||
.position = 0,
|
||||
.lookahead_node = NULL,
|
||||
.lex_state = 0,
|
||||
.stack = calloc(INITIAL_STACK_SIZE, sizeof(TSStackEntry)),
|
||||
.stack_size = 0,
|
||||
.result = {
|
||||
.tree = NULL,
|
||||
.error = {
|
||||
.expected_inputs = NULL,
|
||||
.expected_input_count = 0
|
||||
},
|
||||
},
|
||||
};
|
||||
return result;
|
||||
}
|
||||
|
||||
static char TSParserLookaheadChar(const TSParser *parser) {
|
||||
return parser->input[parser->position];
|
||||
}
|
||||
|
||||
static long TSParserLookaheadSym(const TSParser *parser) {
|
||||
TSTree *node = parser->lookahead_node;
|
||||
return node ? node->value : -1;
|
||||
}
|
||||
|
||||
static TSState TSParserParseState(const TSParser *parser) {
|
||||
if (parser->stack_size == 0) return 0;
|
||||
return parser->stack[parser->stack_size - 1].state;
|
||||
}
|
||||
|
||||
static void TSParserShift(TSParser *parser, TSState parse_state) {
|
||||
DEBUG_PARSE("shift: %d \n", parse_state);
|
||||
TSStackEntry *entry = (parser->stack + parser->stack_size);
|
||||
entry->state = parse_state;
|
||||
entry->node = parser->lookahead_node;
|
||||
parser->lookahead_node = parser->prev_lookahead_node;
|
||||
parser->prev_lookahead_node = NULL;
|
||||
parser->stack_size++;
|
||||
}
|
||||
|
||||
static void TSParserReduce(TSParser *parser, TSSymbol symbol, int immediate_child_count, const int *collapse_flags) {
|
||||
parser->stack_size -= immediate_child_count;
|
||||
|
||||
int total_child_count = 0;
|
||||
for (int i = 0; i < immediate_child_count; i++) {
|
||||
TSTree *child = parser->stack[parser->stack_size + i].node;
|
||||
if (collapse_flags[i]) {
|
||||
total_child_count += child->child_count;
|
||||
} else {
|
||||
total_child_count++;
|
||||
}
|
||||
}
|
||||
|
||||
TSTree **children = malloc(total_child_count * sizeof(TSTree *));
|
||||
int n = 0;
|
||||
for (int i = 0; i < immediate_child_count; i++) {
|
||||
TSTree *child = parser->stack[parser->stack_size + i].node;
|
||||
if (collapse_flags[i]) {
|
||||
memcpy(children + n, child->children, (child->child_count * sizeof(TSTree *)));
|
||||
n += child->child_count;
|
||||
} else {
|
||||
children[n] = child;
|
||||
n++;
|
||||
}
|
||||
}
|
||||
|
||||
parser->prev_lookahead_node = parser->lookahead_node;
|
||||
parser->lookahead_node = TSTreeMake(symbol, total_child_count, children);
|
||||
DEBUG_PARSE("reduce: %s, state: %u \n", ts_symbol_names[symbol], TSParserParseState(parser));
|
||||
}
|
||||
|
||||
static void TSParserError(TSParser *parser, size_t count, const char **expected_inputs) {
|
||||
TSParseError *error = &parser->result.error;
|
||||
error->position = parser->position;
|
||||
error->expected_input_count = count;
|
||||
error->expected_inputs = expected_inputs;
|
||||
error->lookahead_sym = TSParserLookaheadSym(parser);
|
||||
}
|
||||
|
||||
static int TSParserHasError(const TSParser *parser) {
|
||||
return (parser->result.error.expected_inputs != NULL);
|
||||
}
|
||||
|
||||
static void TSParserAdvance(TSParser *parser, TSState lex_state) {
|
||||
DEBUG_LEX("character: '%c' \n", TSParserLookaheadChar(parser));
|
||||
parser->position++;
|
||||
parser->lex_state = lex_state;
|
||||
}
|
||||
|
||||
static void TSParserSetLookaheadSym(TSParser *parser, TSSymbol symbol) {
|
||||
DEBUG_LEX("token: %s \n", ts_symbol_names[symbol]);
|
||||
parser->lookahead_node = TSTreeMake(symbol, 0, NULL);
|
||||
}
|
||||
|
||||
static void TSParserAcceptInput(TSParser *parser) {
|
||||
parser->result.tree = parser->stack[parser->stack_size - 1].node;
|
||||
DEBUG_PARSE("accept \n");
|
||||
}
|
||||
|
||||
#pragma mark - DSL
|
||||
|
||||
#define START_PARSER() \
|
||||
TSParser p = TSParserMake(input), *parser = &p; \
|
||||
next_state:
|
||||
|
||||
#define START_LEXER() \
|
||||
next_state:
|
||||
|
||||
#define LOOKAHEAD_SYM() \
|
||||
TSParserLookaheadSym(parser)
|
||||
|
||||
#define LOOKAHEAD_CHAR() \
|
||||
TSParserLookaheadChar(parser)
|
||||
|
||||
#define PARSE_STATE() \
|
||||
TSParserParseState(parser)
|
||||
|
||||
#define LEX_STATE() \
|
||||
parser->lex_state
|
||||
|
||||
#define SET_LEX_STATE(state_index) \
|
||||
{ \
|
||||
parser->lex_state = state_index; \
|
||||
if (LOOKAHEAD_SYM() < 0) ts_lex(parser); \
|
||||
if (TSParserHasError(parser)) goto done; \
|
||||
}
|
||||
|
||||
#define SHIFT(state) \
|
||||
{ TSParserShift(parser, state); goto next_state; }
|
||||
|
||||
#define ADVANCE(state_index) \
|
||||
{ TSParserAdvance(parser, state_index); goto next_state; }
|
||||
|
||||
#define REDUCE(symbol, child_count, collapse_flags) \
|
||||
{ \
|
||||
static const int flags[] = collapse_flags; \
|
||||
TSParserReduce(parser, symbol, child_count, flags); \
|
||||
goto next_state; \
|
||||
}
|
||||
|
||||
#define ACCEPT_INPUT() \
|
||||
{ TSParserAcceptInput(parser); goto done; }
|
||||
|
||||
#define ACCEPT_TOKEN(symbol) \
|
||||
{ TSParserSetLookaheadSym(parser, symbol); goto done; }
|
||||
|
||||
#define LEX_ERROR(count, inputs) \
|
||||
{ \
|
||||
static const char *expected_inputs[] = inputs; \
|
||||
TSParserError(parser, count, expected_inputs); \
|
||||
goto done; \
|
||||
}
|
||||
|
||||
#define LEX_PANIC() \
|
||||
printf("Lex error: unexpected state %ud", LEX_STATE());
|
||||
|
||||
#define PARSE_PANIC() \
|
||||
printf("Parse error: unexpected state %ud", PARSE_STATE());
|
||||
|
||||
#define EXPECT(...) __VA_ARGS__
|
||||
#define COLLAPSE(...) __VA_ARGS__
|
||||
|
||||
#define FINISH_PARSER() \
|
||||
done: \
|
||||
return parser->result;
|
||||
|
||||
#define FINISH_LEXER() \
|
||||
done:
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
59
include/tree_sitter/runtime.h
Normal file
59
include/tree_sitter/runtime.h
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
#ifndef tree_sitter_runtime_h
|
||||
#define tree_sitter_runtime_h
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
typedef struct {
|
||||
const char **expected_inputs;
|
||||
size_t expected_input_count;
|
||||
size_t position;
|
||||
long lookahead_sym;
|
||||
} TSParseError;
|
||||
|
||||
const char * TSParseErrorToString(const TSParseError *error, const char *input_string, const char **symbol_names);
|
||||
|
||||
typedef size_t TSSymbol;
|
||||
|
||||
typedef struct TSTree {
|
||||
TSSymbol value;
|
||||
struct TSTree **children;
|
||||
size_t child_count;
|
||||
size_t ref_count;
|
||||
} TSTree;
|
||||
|
||||
TSTree * TSTreeMake(TSSymbol value, size_t child_count, TSTree **children);
|
||||
void TSTreeRetain(TSTree *tree);
|
||||
void TSTreeRelease(TSTree *tree);
|
||||
int TSTreeEquals(const TSTree *tree1, const TSTree *tree2);
|
||||
char * TSTreeToString(const TSTree *tree, const char **names);
|
||||
|
||||
typedef struct {
|
||||
TSParseError error;
|
||||
TSTree *tree;
|
||||
} TSParseResult;
|
||||
|
||||
typedef TSParseResult TSParseFn(const char *);
|
||||
|
||||
typedef struct {
|
||||
TSParseFn *parse_fn;
|
||||
const char **symbol_names;
|
||||
} TSParseConfig;
|
||||
|
||||
typedef struct TSDocument TSDocument;
|
||||
|
||||
TSDocument * TSDocumentMake();
|
||||
void TSDocumentSetUp(TSDocument *document, TSParseConfig config);
|
||||
void TSDocumentSetText(TSDocument *document, const char *text);
|
||||
TSTree * TSDocumentTree(const TSDocument *document);
|
||||
const char * TSDocumentToString(const TSDocument *document);
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
Loading…
Add table
Add a link
Reference in a new issue