traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rename RENAME rule to ALIAS, allow it to create anonymous nodes

Browse source
This commit is contained in:
Max Brunsfeld 2017-07-31 11:45:24 -07:00
parent b5f421cafb
commit cb5fe80348
28 changed files with 304 additions and 270 deletions
Download patch file Download diff file Expand all files Collapse all files

11
doc/grammar-schema.json
View file

@ -143,21 +143,24 @@
"required": ["type", "members"]
},
"rename-rule": {
"alias-rule": {
"type": "object",
"properties": {
"type": {
"type": "string",
"pattern": "^RENAME$"
"pattern": "^ALIAS$"
},
"value": {
"type": "string"
},
"named": {
"type": "boolean"
},
"content": {
"$ref": "#/definitions/rule"
}
},
"required": ["type", "content", "value"]
"required": ["type", "named", "content", "value"]
},
"repeat-rule": {
@ -221,13 +224,13 @@
"rule": {
"oneOf": [
{ "$ref": "#/definitions/alias-rule" },
{ "$ref": "#/definitions/blank-rule" },
{ "$ref": "#/definitions/string-rule" },
{ "$ref": "#/definitions/pattern-rule" },
{ "$ref": "#/definitions/symbol-rule" },
{ "$ref": "#/definitions/seq-rule" },
{ "$ref": "#/definitions/choice-rule" },
{ "$ref": "#/definitions/rename-rule" },
{ "$ref": "#/definitions/repeat1-rule" },
{ "$ref": "#/definitions/repeat-rule" },
{ "$ref": "#/definitions/token-rule" },

44
include/tree_sitter/parser.h
View file

@ -47,7 +47,7 @@ typedef struct {
TSSymbol symbol;
int16_t dynamic_precedence;
uint8_t child_count;
uint8_t rename_sequence_id : 7;
uint8_t alias_sequence_id : 7;
bool fragile : 1;
};
} params;
@ -71,7 +71,7 @@ typedef union {
typedef struct TSLanguage {
uint32_t version;
uint32_t symbol_count;
uint32_t rename_symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
const char **symbol_names;
@ -79,8 +79,8 @@ typedef struct TSLanguage {
const uint16_t *parse_table;
const TSParseActionEntry *parse_actions;
const TSLexMode *lex_modes;
const TSSymbol *rename_sequences;
uint16_t max_rename_sequence_length;
const TSSymbol *alias_sequences;
uint16_t max_alias_sequence_length;
bool (*lex_fn)(TSLexer *, TSStateId);
struct {
const bool *states;
@ -172,24 +172,24 @@ typedef struct TSLanguage {
{ .type = TSParseActionTypeAccept } \
}
#define GET_LANGUAGE(...) \
static TSLanguage language = { \
.version = LANGUAGE_VERSION, \
.symbol_count = SYMBOL_COUNT, \
.rename_symbol_count = RENAME_SYMBOL_COUNT, \
.token_count = TOKEN_COUNT, \
.symbol_metadata = ts_symbol_metadata, \
.parse_table = (const unsigned short *)ts_parse_table, \
.parse_actions = ts_parse_actions, \
.lex_modes = ts_lex_modes, \
.symbol_names = ts_symbol_names, \
.rename_sequences = (const TSSymbol *)ts_rename_sequences, \
.max_rename_sequence_length = MAX_RENAME_SEQUENCE_LENGTH, \
.lex_fn = ts_lex, \
.external_token_count = EXTERNAL_TOKEN_COUNT, \
.external_scanner = {__VA_ARGS__} \
}; \
return &language \
#define GET_LANGUAGE(...) \
static TSLanguage language = { \
.version = LANGUAGE_VERSION, \
.symbol_count = SYMBOL_COUNT, \
.alias_count = ALIAS_COUNT, \
.token_count = TOKEN_COUNT, \
.symbol_metadata = ts_symbol_metadata, \
.parse_table = (const unsigned short *)ts_parse_table, \
.parse_actions = ts_parse_actions, \
.lex_modes = ts_lex_modes, \
.symbol_names = ts_symbol_names, \
.alias_sequences = (const TSSymbol *)ts_alias_sequences, \
.max_alias_sequence_length = MAX_ALIAS_SEQUENCE_LENGTH, \
.lex_fn = ts_lex, \
.external_token_count = EXTERNAL_TOKEN_COUNT, \
.external_scanner = {__VA_ARGS__} \
}; \
return &language \
#ifdef __cplusplus
}

34
src/compiler/build_tables/build_parse_table.cc
View file

@ -64,7 +64,7 @@ class ParseTableBuilder {
pair<ParseTable, CompileError> build() {
// Ensure that the empty rename sequence has index 0.
parse_table.rename_sequences.push_back({});
parse_table.alias_sequences.push_back({});
// Ensure that the error state has index 0.
ParseStateId error_state_id = add_parse_state({}, ParseItemSet{});
@ -73,7 +73,7 @@ class ParseTableBuilder {
Symbol start_symbol = grammar.variables.empty() ?
Symbol::terminal(0) :
Symbol::non_terminal(0);
Production start_production{{{start_symbol, 0, rules::AssociativityNone, ""}}, 0};
Production start_production{{{start_symbol, 0, rules::AssociativityNone, {"", false}}}, 0};
add_parse_state({}, ParseItemSet{{
{
ParseItem(rules::START(), start_production, 0),
@ -204,7 +204,7 @@ class ParseTableBuilder {
item.precedence(),
item.production->dynamic_precedence,
item.associativity(),
get_rename_sequence_id(*item.production)
get_alias_sequence_id(*item.production)
);
lookahead_symbols.for_each([&](Symbol lookahead) {
@ -716,30 +716,30 @@ class ParseTableBuilder {
}
}
unsigned get_rename_sequence_id(const Production &production) {
bool has_rename = false;
RenameSequence rename_sequence;
unsigned get_alias_sequence_id(const Production &production) {
bool has_alias = false;
AliasSequence alias_sequence;
for (unsigned i = 0, n = production.size(); i < n; i++) {
auto &step = production.at(i);
if (!step.name_replacement.empty()) {
has_rename = true;
rename_sequence.resize(i + 1);
rename_sequence[i] = step.name_replacement;
if (!step.alias.value.empty()) {
has_alias = true;
alias_sequence.resize(i + 1);
alias_sequence[i] = step.alias;
}
}
if (has_rename && production.size() > parse_table.max_rename_sequence_length) {
parse_table.max_rename_sequence_length = production.size();
if (has_alias && production.size() > parse_table.max_alias_sequence_length) {
parse_table.max_alias_sequence_length = production.size();
}
auto begin = parse_table.rename_sequences.begin();
auto end = parse_table.rename_sequences.end();
auto iter = find(begin, end, rename_sequence);
auto begin = parse_table.alias_sequences.begin();
auto end = parse_table.alias_sequences.end();
auto iter = find(begin, end, alias_sequence);
if (iter != end) {
return iter - begin;
} else {
parse_table.rename_sequences.push_back(move(rename_sequence));
return parse_table.rename_sequences.size() - 1;
parse_table.alias_sequences.push_back(move(alias_sequence));
return parse_table.alias_sequences.size() - 1;
}
}

11
src/compiler/build_tables/parse_item.cc
View file

@ -28,9 +28,7 @@ bool ParseItem::operator==(const ParseItem &other) const {
if (variable_index != other.variable_index) return false;
if (production->size() != other.production->size()) return false;
for (size_t i = 0; i < step_index; i++) {
if (production->at(i).name_replacement != other.production->at(i).name_replacement) {
return false;
}
if (production->at(i).alias != other.production->at(i).alias) return false;
}
if (is_done()) {
if (!production->empty()) {
@ -53,8 +51,8 @@ bool ParseItem::operator<(const ParseItem &other) const {
if (production->size() < other.production->size()) return true;
if (other.production->size() < production->size()) return false;
for (size_t i = 0; i < step_index; i++) {
if (production->at(i).name_replacement < other.production->at(i).name_replacement) return true;
if (other.production->at(i).name_replacement < production->at(i).name_replacement) return false;
if (production->at(i).alias < other.production->at(i).alias) return true;
if (other.production->at(i).alias < production->at(i).alias) return false;
}
if (is_done()) {
if (!production->empty()) {
@ -158,7 +156,8 @@ struct hash<ParseItem> {
hash_combine(&result, item.production->dynamic_precedence);
hash_combine(&result, item.production->size());
for (size_t i = 0; i < item.step_index; i++) {
hash_combine(&result, item.production->at(i).name_replacement);
hash_combine(&result, item.production->at(i).alias.value);
hash_combine(&result, item.production->at(i).alias.is_named);
}
if (item.is_done()) {
if (!item.production->empty()) {

4
src/compiler/build_tables/parse_item_set_builder.cc
View file

@ -211,8 +211,8 @@ const vector<Production> &ParseItemSetBuilder::inline_production(const ParseItem
Production production{{begin, step}, item.production->dynamic_precedence};
for (auto &step : *production_to_insert) {
production.steps.push_back(step);
if (!inlined_step.name_replacement.empty()) {
production.steps.back().name_replacement = inlined_step.name_replacement;
if (!inlined_step.alias.value.empty()) {
production.steps.back().alias = inlined_step.alias;
}
}
production.back().precedence = inlined_step.precedence;

62
src/compiler/generate_code/c_code.cc
View file

@ -25,6 +25,7 @@ using std::to_string;
using std::vector;
using util::escape_char;
using rules::Symbol;
using rules::Alias;
static const map<char, string> REPLACEMENTS({
{ '~', "TILDE" },
@ -76,7 +77,7 @@ class CCodeGenerator {
vector<pair<size_t, ParseTableEntry>> parse_table_entries;
vector<set<Symbol::Index>> external_scanner_states;
size_t next_parse_action_list_index;
set<string> unique_replacement_names;
set<Alias> unique_aliases;
public:
CCodeGenerator(string name, const ParseTable &parse_table,
@ -99,7 +100,7 @@ class CCodeGenerator {
add_symbol_enum();
add_symbol_names_list();
add_symbol_metadata_list();
add_rename_sequences();
add_alias_sequences();
add_lex_function();
add_lex_modes_list();
@ -141,10 +142,10 @@ class CCodeGenerator {
}
}
for (const RenameSequence &rename_sequence : parse_table.rename_sequences) {
for (const string &name_replacement : rename_sequence) {
if (!name_replacement.empty()) {
unique_replacement_names.insert(name_replacement);
for (const AliasSequence &alias_sequence : parse_table.alias_sequences) {
for (const Alias &alias : alias_sequence) {
if (!alias.value.empty()) {
unique_aliases.insert(alias);
}
}
}
@ -152,10 +153,10 @@ class CCodeGenerator {
line("#define LANGUAGE_VERSION " + to_string(TREE_SITTER_LANGUAGE_VERSION));
line("#define STATE_COUNT " + to_string(parse_table.states.size()));
line("#define SYMBOL_COUNT " + to_string(parse_table.symbols.size()));
line("#define RENAME_SYMBOL_COUNT " + to_string(unique_replacement_names.size()));
line("#define ALIAS_COUNT " + to_string(unique_aliases.size()));
line("#define TOKEN_COUNT " + to_string(token_count));
line("#define EXTERNAL_TOKEN_COUNT " + to_string(syntax_grammar.external_tokens.size()));
line("#define MAX_RENAME_SEQUENCE_LENGTH " + to_string(parse_table.max_rename_sequence_length));
line("#define MAX_ALIAS_SEQUENCE_LENGTH " + to_string(parse_table.max_alias_sequence_length));
line();
}
@ -171,8 +172,8 @@ class CCodeGenerator {
}
}
for (const string &replacement_name : unique_replacement_names) {
line(rename_id(replacement_name) + " = " + to_string(i) + ",");
for (const Alias &alias : unique_aliases) {
line(alias_id(alias) + " = " + to_string(i) + ",");
i++;
}
});
@ -190,10 +191,10 @@ class CCodeGenerator {
);
}
for (const string &replacement_name : unique_replacement_names) {
for (const Alias &alias : unique_aliases) {
line(
"[" + rename_id(replacement_name) + "] = \"" +
sanitize_name_for_string(replacement_name) + "\","
"[" + alias_id(alias) + "] = \"" +
sanitize_name_for_string(alias.value) + "\","
);
}
});
@ -201,22 +202,21 @@ class CCodeGenerator {
line();
}
void add_rename_sequences() {
void add_alias_sequences() {
line(
"static TSSymbol ts_rename_sequences[" +
to_string(parse_table.rename_sequences.size()) +
"][MAX_RENAME_SEQUENCE_LENGTH] = {"
"static TSSymbol ts_alias_sequences[" +
to_string(parse_table.alias_sequences.size()) +
"][MAX_ALIAS_SEQUENCE_LENGTH] = {"
);
indent([&]() {
for (unsigned i = 1, n = parse_table.rename_sequences.size(); i < n; i++) {
const RenameSequence &sequence = parse_table.rename_sequences[i];
for (unsigned i = 1, n = parse_table.alias_sequences.size(); i < n; i++) {
const AliasSequence &sequence = parse_table.alias_sequences[i];
line("[" + to_string(i) + "] = {");
indent([&]() {
for (unsigned j = 0, n = sequence.size(); j < n; j++) {
if (!sequence[j].empty()) {
line("[" + to_string(j) + "] = " + rename_id(sequence[j]) + ",");
if (!sequence[j].value.empty()) {
line("[" + to_string(j) + "] = " + alias_id(sequence[j]) + ",");
}
}
});
@ -260,11 +260,11 @@ class CCodeGenerator {
line("},");
}
for (const string &replacement_name : unique_replacement_names) {
line("[" + rename_id(replacement_name) + "] = {");
for (const Alias &alias : unique_aliases) {
line("[" + alias_id(alias) + "] = {");
indent([&]() {
line(".visible = true,");
line(".named = true,");
line(".named = " + _boolean(alias.is_named) + ",");
line(".structural = true,");
line(".extra = true,");
});
@ -616,8 +616,8 @@ class CCodeGenerator {
add(", .dynamic_precedence = " + to_string(action.dynamic_precedence));
}
if (action.rename_sequence_id != 0) {
add(", .rename_sequence_id = " + to_string(action.rename_sequence_id));
if (action.alias_sequence_id != 0) {
add(", .alias_sequence_id = " + to_string(action.alias_sequence_id));
}
add(")");
@ -671,8 +671,12 @@ class CCodeGenerator {
}
}
string rename_id(const string &name) {
return "rename_sym_" + sanitize_name(name);
string alias_id(const Alias &alias) {
if (alias.is_named) {
return "alias_sym_" + sanitize_name(alias.value);
} else {
return "anon_alias_sym_" + sanitize_name(alias.value);
}
}
string symbol_name(const Symbol &symbol) {

17
src/compiler/parse_grammar.cc
View file

@ -198,18 +198,27 @@ ParseRuleResult parse_rule(json_value *rule_json) {
return Rule(Metadata::prec_dynamic(precedence_json.u.integer, result.rule));
}
if (type == "RENAME") {
json_value name_json = rule_json->operator[]("value");
if (name_json.type != json_string) {
if (type == "ALIAS") {
json_value value_json = rule_json->operator[]("value");
if (value_json.type != json_string) {
return "Rename value must be a string";
}
json_value is_named_json = rule_json->operator[]("named");
if (is_named_json.type != json_boolean) {
return "Rename named value must be a boolean";
}
json_value content_json = rule_json->operator[]("content");
auto result = parse_rule(&content_json);
if (!result.error_message.empty()) {
return "Invalid rename content: " + result.error_message;
}
return Rule(Metadata::rename(string(name_json.u.string.ptr), result.rule));
return Rule(Metadata::alias(
string(value_json.u.string.ptr),
is_named_json.u.boolean,
result.rule
));
}
return "Unknown rule type: " + type;

10
src/compiler/parse_table.cc
View file

@ -19,7 +19,7 @@ ParseAction::ParseAction() :
precedence(0),
dynamic_precedence(0),
associativity(rules::AssociativityNone),
rename_sequence_id(0),
alias_sequence_id(0),
fragile(false),
extra(false) {}
@ -56,7 +56,7 @@ ParseAction ParseAction::ShiftExtra() {
ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count,
int precedence, int dynamic_precedence,
rules::Associativity associativity, unsigned rename_sequence_id) {
rules::Associativity associativity, unsigned alias_sequence_id) {
ParseAction result;
result.type = ParseActionTypeReduce;
result.symbol = symbol;
@ -64,7 +64,7 @@ ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count,
result.precedence = precedence;
result.dynamic_precedence = dynamic_precedence;
result.associativity = associativity;
result.rename_sequence_id = rename_sequence_id;
result.alias_sequence_id = alias_sequence_id;
return result;
}
@ -77,7 +77,7 @@ bool ParseAction::operator==(const ParseAction &other) const {
precedence == other.precedence &&
dynamic_precedence == other.dynamic_precedence &&
associativity == other.associativity &&
rename_sequence_id == other.rename_sequence_id &&
alias_sequence_id == other.alias_sequence_id &&
extra == other.extra &&
fragile == other.fragile;
}
@ -101,7 +101,7 @@ bool ParseAction::operator<(const ParseAction &other) const {
if (other.extra && !extra) return false;
if (fragile && !other.fragile) return true;
if (other.fragile && !fragile) return false;
return rename_sequence_id < other.rename_sequence_id;
return alias_sequence_id < other.alias_sequence_id;
}
ParseTableEntry::ParseTableEntry()

10
src/compiler/parse_table.h
View file

@ -30,7 +30,7 @@ struct ParseAction {
static ParseAction Recover(ParseStateId state_index);
static ParseAction Reduce(rules::Symbol symbol, size_t child_count,
int precedence, int dynamic_precedence, rules::Associativity,
unsigned rename_sequence_id);
unsigned alias_sequence_id);
static ParseAction ShiftExtra();
bool operator==(const ParseAction &) const;
bool operator<(const ParseAction &) const;
@ -42,7 +42,7 @@ struct ParseAction {
int precedence;
int dynamic_precedence;
rules::Associativity associativity;
unsigned rename_sequence_id;
unsigned alias_sequence_id;
bool fragile;
bool extra;
};
@ -77,7 +77,7 @@ struct ParseTableSymbolMetadata {
bool structural;
};
using RenameSequence = std::vector<std::string>;
using AliasSequence = std::vector<rules::Alias>;
struct ParseTable {
ParseAction &add_terminal_action(ParseStateId state_id, rules::Symbol, ParseAction);
@ -85,8 +85,8 @@ struct ParseTable {
std::vector<ParseState> states;
std::map<rules::Symbol, ParseTableSymbolMetadata> symbols;
std::vector<RenameSequence> rename_sequences;
unsigned max_rename_sequence_length = 0;
std::vector<AliasSequence> alias_sequences;
unsigned max_alias_sequence_length = 0;
};
} // namespace tree_sitter

14
src/compiler/prepare_grammar/flatten_grammar.cc
View file

@ -22,7 +22,7 @@ class FlattenRule {
private:
vector<int> precedence_stack;
vector<rules::Associativity> associativity_stack;
vector<string> name_replacement_stack;
vector<rules::Alias> alias_stack;
Production production;
void apply(const Rule &rule, bool at_end) {
@ -32,7 +32,7 @@ class FlattenRule {
symbol,
precedence_stack.back(),
associativity_stack.back(),
name_replacement_stack.back()
alias_stack.back()
});
},
@ -45,8 +45,8 @@ class FlattenRule {
associativity_stack.push_back(metadata.params.associativity);
}
if (!metadata.params.name_replacement.empty()) {
name_replacement_stack.push_back(metadata.params.name_replacement);
if (!metadata.params.alias.value.empty()) {
alias_stack.push_back(metadata.params.alias);
}
if (abs(metadata.params.dynamic_precedence) > abs(production.dynamic_precedence)) {
@ -65,8 +65,8 @@ class FlattenRule {
if (!at_end) production.back().associativity = associativity_stack.back();
}
if (!metadata.params.name_replacement.empty()) {
name_replacement_stack.pop_back();
if (!metadata.params.alias.value.empty()) {
alias_stack.pop_back();
}
},
@ -87,7 +87,7 @@ class FlattenRule {
FlattenRule() :
precedence_stack({0}),
associativity_stack({rules::AssociativityNone}),
name_replacement_stack({""}) {}
alias_stack({rules::Alias{}}) {}
Production flatten(const Rule &rule) {
apply(rule, true);

18
src/compiler/rules/metadata.cc
View file

@ -9,6 +9,20 @@ namespace rules {
using std::move;
using std::string;
bool Alias::operator==(const Alias &other) const {
return value == other.value && is_named == other.is_named;
}
bool Alias::operator!=(const Alias &other) const {
return !operator==(other);
}
bool Alias::operator<(const Alias &other) const {
if (value < other.value) return true;
if (other.value < value) return false;
return is_named < other.is_named;
}
Metadata::Metadata(const Rule &rule, MetadataParams params) :
rule(std::make_shared<Rule>(rule)), params(params) {}
@ -77,9 +91,9 @@ Metadata Metadata::main_token(const Rule &rule) {
return Metadata{rule, params};
}
Metadata Metadata::rename(string &&name, const Rule &rule) {
Metadata Metadata::alias(string &&value, bool is_named, const Rule &rule) {
MetadataParams params;
params.name_replacement = move(name);
params.alias = {move(value), is_named};
return Metadata{rule, params};
}

14
src/compiler/rules/metadata.h
View file

@ -13,6 +13,14 @@ enum Associativity {
AssociativityRight,
};
struct Alias {
std::string value = "";
bool is_named = false;
bool operator==(const Alias &) const;
bool operator!=(const Alias &) const;
bool operator<(const Alias &) const;
};
struct MetadataParams {
int precedence;
int dynamic_precedence;
@ -23,7 +31,7 @@ struct MetadataParams {
bool is_string;
bool is_active;
bool is_main_token;
std::string name_replacement;
Alias alias;
inline MetadataParams() :
precedence{0}, dynamic_precedence{0}, associativity{AssociativityNone},
@ -41,7 +49,7 @@ struct MetadataParams {
is_string == other.is_string &&
is_active == other.is_active &&
is_main_token == other.is_main_token &&
name_replacement == other.name_replacement
alias == other.alias
);
}
};
@ -62,7 +70,7 @@ struct Metadata {
static Metadata prec_dynamic(int precedence, const Rule &rule);
static Metadata separator(const Rule &rule);
static Metadata main_token(const Rule &rule);
static Metadata rename(std::string &&name, const Rule &rule);
static Metadata alias(std::string &&value, bool is_named, const Rule &rule);
bool operator==(const Metadata &other) const;
};

4
src/compiler/syntax_grammar.cc
View file

@ -6,7 +6,7 @@ bool ProductionStep::operator==(const ProductionStep &other) const {
return symbol == other.symbol &&
precedence == other.precedence &&
associativity == other.associativity &&
name_replacement == other.name_replacement;
alias == other.alias;
}
bool ProductionStep::operator!=(const ProductionStep &other) const {
@ -20,7 +20,7 @@ bool ProductionStep::operator<(const ProductionStep &other) const {
if (other.precedence < precedence) return false;
if (associativity < other.associativity) return true;
if (other.associativity < associativity) return false;
return name_replacement < other.name_replacement;
return alias < other.alias;
}
bool Production::operator==(const Production &other) const {

2
src/compiler/syntax_grammar.h
View file

@ -13,7 +13,7 @@ struct ProductionStep {
rules::Symbol symbol;
int precedence;
rules::Associativity associativity;
std::string name_replacement;
rules::Alias alias;
bool operator==(const ProductionStep &) const;
bool operator!=(const ProductionStep &) const;

2
src/runtime/language.c
View file

@ -32,7 +32,7 @@ void ts_language_table_entry(const TSLanguage *self, TSStateId state,
}
uint32_t ts_language_symbol_count(const TSLanguage *language) {
return language->symbol_count + language->rename_symbol_count;
return language->symbol_count + language->alias_count;
}
uint32_t ts_language_version(const TSLanguage *language) {

4
src/runtime/language.h
View file

@ -64,9 +64,9 @@ ts_language_enabled_external_tokens(const TSLanguage *self,
}
static inline const TSSymbol *
ts_language_rename_sequence(const TSLanguage *self, unsigned id) {
ts_language_alias_sequence(const TSLanguage *self, unsigned id) {
return id > 0 ?
self->rename_sequences + id * self->max_rename_sequence_length :
self->alias_sequences + id * self->max_alias_sequence_length :
NULL;
}

6
src/runtime/node.c
View file

@ -33,7 +33,7 @@ static inline uint32_t ts_node__offset_row(TSNode self) {
static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) {
const Tree *tree = ts_node__tree(self);
if (tree->context.rename_symbol > 0) {
if (tree->context.alias_symbol > 0) {
return true;
} else {
return include_anonymous ? tree->visible : tree->visible && tree->named;
@ -269,7 +269,7 @@ TSPoint ts_node_end_point(TSNode self) {
TSSymbol ts_node_symbol(TSNode self) {
const Tree *tree = ts_node__tree(self);
return tree->context.rename_symbol ? tree->context.rename_symbol : tree->symbol;
return tree->context.alias_symbol ? tree->context.alias_symbol : tree->symbol;
}
TSSymbolIterator ts_node_symbols(TSNode self) {
@ -308,7 +308,7 @@ bool ts_node_eq(TSNode self, TSNode other) {
bool ts_node_is_named(TSNode self) {
const Tree *tree = ts_node__tree(self);
return tree->named || tree->context.rename_symbol != 0;
return tree->context.alias_symbol ? tree->context.alias_is_named : tree->named;
}
bool ts_node_has_changes(TSNode self) {

49
src/runtime/parser.c
View file

@ -338,7 +338,7 @@ static Tree *parser__lex(Parser *self, StackVersion version) {
if (skipped_error) {
Length padding = length_sub(error_start_position, start_position);
Length size = length_sub(error_end_position, error_start_position);
result = ts_tree_make_error(size, padding, first_error_character);
result = ts_tree_make_error(size, padding, first_error_character, self->language);
} else {
TSSymbol symbol = self->lexer.data.result_symbol;
if (found_external_token) {
@ -347,8 +347,7 @@ static Tree *parser__lex(Parser *self, StackVersion version) {
Length padding = length_sub(self->lexer.token_start_position, start_position);
Length size = length_sub(self->lexer.token_end_position, self->lexer.token_start_position);
TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, symbol);
result = ts_tree_make_leaf(symbol, padding, size, metadata);
result = ts_tree_make_leaf(symbol, padding, size, self->language);
if (found_external_token) {
result->has_external_tokens = true;
@ -544,12 +543,7 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state,
static bool parser__replace_children(Parser *self, Tree *tree, Tree **children, uint32_t count) {
self->scratch_tree.symbol = tree->symbol;
self->scratch_tree.child_count = 0;
ts_tree_set_children(
&self->scratch_tree,
count,
children,
ts_language_rename_sequence(self->language, tree->rename_sequence_id)
);
ts_tree_set_children(&self->scratch_tree, count, children, self->language);
if (parser__select_tree(self, tree, &self->scratch_tree)) {
tree->size = self->scratch_tree.size;
tree->padding = self->scratch_tree.padding;
@ -566,16 +560,13 @@ static bool parser__replace_children(Parser *self, Tree *tree, Tree **children,
static StackPopResult parser__reduce(Parser *self, StackVersion version,
TSSymbol symbol, uint32_t count,
int dynamic_precedence, uint16_t rename_sequence_id,
int dynamic_precedence, uint16_t alias_sequence_id,
bool fragile, bool allow_skipping) {
uint32_t initial_version_count = ts_stack_version_count(self->stack);
StackPopResult pop = ts_stack_pop_count(self->stack, version, count);
if (pop.stopped_at_error) return pop;
TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, symbol);
const TSSymbol *rename_sequence = ts_language_rename_sequence(self->language, rename_sequence_id);
for (uint32_t i = 0; i < pop.slices.size; i++) {
StackSlice slice = pop.slices.contents[i];
@ -588,7 +579,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version,
}
Tree *parent = ts_tree_make_node(
symbol, child_count, slice.trees.contents, metadata, rename_sequence
symbol, child_count, slice.trees.contents, alias_sequence_id, self->language
);
// This pop operation may have caused multiple stack versions to collapse
@ -614,7 +605,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version,
}
parent->dynamic_precedence += dynamic_precedence;
parent->rename_sequence_id = rename_sequence_id;
parent->alias_sequence_id = alias_sequence_id;
TSStateId state = ts_stack_top_state(self->stack, slice.version);
TSStateId next_state = ts_language_next_state(self->language, state, symbol);
@ -840,7 +831,7 @@ static bool parser__repair_error(Parser *self, StackSlice slice,
TreeArray skipped_children = ts_tree_array_remove_last_n(&children, skip_count);
TreeArray trailing_extras = ts_tree_array_remove_trailing_extras(&skipped_children);
Tree *error = ts_tree_make_error_node(&skipped_children);
Tree *error = ts_tree_make_error_node(&skipped_children, self->language);
array_push(&children, error);
array_push_all(&children, &trailing_extras);
trailing_extras.size = 0;
@ -852,8 +843,7 @@ static bool parser__repair_error(Parser *self, StackSlice slice,
Tree *parent = ts_tree_make_node(
symbol, children.size, children.contents,
ts_language_symbol_metadata(self->language, symbol),
NULL
0, self->language
);
parser__push(self, slice.version, parent, next_state);
ts_stack_decrease_push_count(self->stack, slice.version, error->child_count);
@ -912,11 +902,7 @@ static void parser__accept(Parser *self, StackVersion version,
for (uint32_t k = 0; k < child->child_count; k++)
ts_tree_retain(child->children[k]);
array_splice(&trees, j, 1, child->child_count, child->children);
const TSSymbol *rename_sequence = ts_language_rename_sequence(
self->language,
root->rename_sequence_id
);
ts_tree_set_children(root, trees.size, trees.contents, rename_sequence);
ts_tree_set_children(root, trees.size, trees.contents, self->language);
ts_tree_release(child);
break;
}
@ -965,7 +951,7 @@ static bool parser__do_potential_reductions(Parser *self, StackVersion version)
.symbol = action.params.symbol,
.count = action.params.child_count,
.dynamic_precedence = action.params.dynamic_precedence,
.rename_sequence_id = action.params.rename_sequence_id,
.alias_sequence_id = action.params.alias_sequence_id,
});
default:
break;
@ -978,7 +964,7 @@ static bool parser__do_potential_reductions(Parser *self, StackVersion version)
ReduceAction action = self->reduce_actions.contents[i];
StackPopResult reduction = parser__reduce(
self, version, action.symbol, action.count,
action.dynamic_precedence, action.rename_sequence_id,
action.dynamic_precedence, action.alias_sequence_id,
true, false
);
if (reduction.stopped_at_error) {
@ -1039,7 +1025,7 @@ static bool parser__skip_preceding_trees(Parser *self, StackVersion version,
}
previous_version = slice.version;
Tree *error = ts_tree_make_error_node(&slice.trees);
Tree *error = ts_tree_make_error_node(&slice.trees, self->language);
error->extra = true;
TSStateId state = ts_stack_top_state(self->stack, slice.version);
parser__push(self, slice.version, error, state);
@ -1104,16 +1090,15 @@ static void parser__halt_parse(Parser *self) {
ts_stack_top_position(self->stack, 0)
);
Tree *filler_node = ts_tree_make_error(remaining_length, length_zero(), 0);
Tree *filler_node = ts_tree_make_error(remaining_length, length_zero(), 0, self->language);
filler_node->visible = false;
parser__push(self, 0, filler_node, 0);
TreeArray children = array_new();
Tree *root_error = ts_tree_make_error_node(&children);
Tree *root_error = ts_tree_make_error_node(&children, self->language);
parser__push(self, 0, root_error, 0);
TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, ts_builtin_sym_end);
Tree *eof = ts_tree_make_leaf(ts_builtin_sym_end, length_zero(), length_zero(), metadata);
Tree *eof = ts_tree_make_leaf(ts_builtin_sym_end, length_zero(), length_zero(), self->language);
parser__accept(self, 0, eof);
ts_tree_release(eof);
}
@ -1123,7 +1108,7 @@ static void parser__recover(Parser *self, StackVersion version, TSStateId state,
if (lookahead->symbol == ts_builtin_sym_end) {
LOG("recover_eof");
TreeArray children = array_new();
Tree *parent = ts_tree_make_error_node(&children);
Tree *parent = ts_tree_make_error_node(&children, self->language);
parser__push(self, version, parent, 1);
parser__accept(self, version, lookahead);
return;
@ -1218,7 +1203,7 @@ static void parser__advance(Parser *self, StackVersion version,
LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.params.symbol), action.params.child_count);
StackPopResult reduction = parser__reduce(
self, version, action.params.symbol, action.params.child_count,
action.params.dynamic_precedence, action.params.rename_sequence_id,
action.params.dynamic_precedence, action.params.alias_sequence_id,
action.params.fragile, true
);
StackSlice slice = *array_front(&reduction.slices);

2
src/runtime/reduce_action.h
View file

@ -12,7 +12,7 @@ typedef struct {
uint32_t count;
TSSymbol symbol;
int dynamic_precedence;
unsigned short rename_sequence_id;
unsigned short alias_sequence_id;
} ReduceAction;
typedef Array(ReduceAction) ReduceActionSet;

94
src/runtime/tree.c
View file

@ -42,23 +42,23 @@ bool ts_external_token_state_eq(const TSExternalTokenState *a, const TSExternalT
memcmp(ts_external_token_state_data(a), ts_external_token_state_data(b), a->length) == 0);
}
Tree *ts_tree_make_leaf(TSSymbol sym, Length padding, Length size,
TSSymbolMetadata metadata) {
Tree *ts_tree_make_leaf(TSSymbol symbol, Length padding, Length size, const TSLanguage *language) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol);
Tree *result = ts_malloc(sizeof(Tree));
*result = (Tree){
.ref_count = 1,
.symbol = sym,
.symbol = symbol,
.size = size,
.child_count = 0,
.visible_child_count = 0,
.named_child_count = 0,
.children = NULL,
.rename_sequence_id = 0,
.alias_sequence_id = 0,
.padding = padding,
.visible = metadata.visible,
.named = metadata.named,
.has_changes = false,
.first_leaf.symbol = sym,
.first_leaf.symbol = symbol,
.has_external_tokens = false,
};
return result;
@ -134,11 +134,9 @@ TreeArray ts_tree_array_remove_trailing_extras(TreeArray *self) {
return result;
}
Tree *ts_tree_make_error(Length size, Length padding, int32_t lookahead_char) {
Tree *result = ts_tree_make_leaf(ts_builtin_sym_error, padding, size,
(TSSymbolMetadata){
.visible = true, .named = true,
});
Tree *ts_tree_make_error(Length size, Length padding, int32_t lookahead_char,
const TSLanguage *language) {
Tree *result = ts_tree_make_leaf(ts_builtin_sym_error, padding, size, language);
result->fragile_left = true;
result->fragile_right = true;
result->lookahead_char = lookahead_char;
@ -158,7 +156,7 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua
while (path->size > 0) {
Tree *tree = array_pop(path).tree;
Length offset = length_zero();
const TSSymbol *rename_sequence = ts_language_rename_sequence(language, tree->rename_sequence_id);
const TSSymbol *alias_sequence = ts_language_alias_sequence(language, tree->alias_sequence_id);
uint32_t non_extra_index = 0;
for (uint32_t i = 0; i < tree->child_count; i++) {
Tree *child = tree->children[i];
@ -166,10 +164,12 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua
child->context.parent = tree;
child->context.index = i;
child->context.offset = offset;
if (!child->extra && rename_sequence && rename_sequence[non_extra_index] != 0) {
child->context.rename_symbol = rename_sequence[non_extra_index];
if (!child->extra && alias_sequence && alias_sequence[non_extra_index] != 0) {
TSSymbolMetadata metadata = ts_language_symbol_metadata(language, alias_sequence[non_extra_index]);
child->context.alias_symbol = alias_sequence[non_extra_index];
child->context.alias_is_named = metadata.named;
} else {
child->context.rename_symbol = 0;
child->context.alias_symbol = 0;
}
array_push(path, ((TreePathEntry){child, length_zero(), 0}));
}
@ -181,9 +181,8 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua
void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
const TSSymbol *rename_sequence) {
if (self->child_count > 0)
ts_free(self->children);
const TSLanguage *language) {
if (self->child_count > 0) ts_free(self->children);
self->children = children;
self->child_count = child_count;
@ -194,6 +193,7 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
self->dynamic_precedence = 0;
uint32_t non_extra_index = 0;
const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id);
for (uint32_t i = 0; i < child_count; i++) {
Tree *child = children[i];
@ -211,9 +211,11 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
self->error_cost += child->error_cost;
self->dynamic_precedence += child->dynamic_precedence;
if (rename_sequence && rename_sequence[non_extra_index] != 0 && !child->extra) {
if (alias_sequence && alias_sequence[non_extra_index] != 0 && !child->extra) {
self->visible_child_count++;
self->named_child_count++;
if (ts_language_symbol_metadata(language, alias_sequence[non_extra_index]).named) {
self->named_child_count++;
}
} else if (child->visible) {
self->visible_child_count++;
if (child->named) self->named_child_count++;
@ -250,14 +252,14 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children,
}
Tree *ts_tree_make_node(TSSymbol symbol, uint32_t child_count, Tree **children,
TSSymbolMetadata metadata, const TSSymbol *rename_sequence) {
Tree *result =
ts_tree_make_leaf(symbol, length_zero(), length_zero(), metadata);
ts_tree_set_children(result, child_count, children, rename_sequence);
unsigned alias_sequence_id, const TSLanguage *language) {
Tree *result = ts_tree_make_leaf(symbol, length_zero(), length_zero(), language);
result->alias_sequence_id = alias_sequence_id;
ts_tree_set_children(result, child_count, children, language);
return result;
}
Tree *ts_tree_make_error_node(TreeArray *children) {
Tree *ts_tree_make_error_node(TreeArray *children, const TSLanguage *language) {
for (uint32_t i = 0; i < children->size; i++) {
Tree *child = children->contents[i];
if (child->symbol == ts_builtin_sym_error && child->child_count > 0) {
@ -269,9 +271,8 @@ Tree *ts_tree_make_error_node(TreeArray *children) {
}
}
Tree *result = ts_tree_make_node(
ts_builtin_sym_error, children->size, children->contents,
(TSSymbolMetadata){.extra = false, .visible = true, .named = true }, NULL);
Tree *result =
ts_tree_make_node(ts_builtin_sym_error, children->size, children->contents, 0, language);
result->fragile_left = true;
result->fragile_right = true;
@ -328,29 +329,26 @@ uint32_t ts_tree_end_column(const Tree *self) {
bool ts_tree_eq(const Tree *self, const Tree *other) {
if (self) {
if (!other)
return false;
if (!other) return false;
} else {
return !other;
}
if (self->symbol != other->symbol)
return false;
if (self->visible != other->visible)
return false;
if (self->named != other->named)
return false;
if (self->symbol == ts_builtin_sym_error)
return self->lookahead_char == other->lookahead_char;
if (self->child_count != other->child_count)
return false;
if (self->visible_child_count != other->visible_child_count)
return false;
if (self->named_child_count != other->named_child_count)
return false;
for (uint32_t i = 0; i < self->child_count; i++)
if (!ts_tree_eq(self->children[i], other->children[i]))
if (self->symbol != other->symbol) return false;
if (self->visible != other->visible) return false;
if (self->named != other->named) return false;
if (self->padding.bytes != other->padding.bytes) return false;
if (self->size.bytes != other->size.bytes) return false;
if (self->symbol == ts_builtin_sym_error) return self->lookahead_char == other->lookahead_char;
if (self->child_count != other->child_count) return false;
if (self->visible_child_count != other->visible_child_count) return false;
if (self->named_child_count != other->named_child_count) return false;
for (uint32_t i = 0; i < self->child_count; i++) {
if (!ts_tree_eq(self->children[i], other->children[i])) {
return false;
}
}
return true;
}
@ -528,7 +526,7 @@ static size_t ts_tree__write_to_string(const Tree *self,
include_all ||
is_root ||
(self->visible && self->named) ||
self->context.rename_symbol != 0;
self->context.alias_is_named;
if (visible && !is_root) {
cursor += snprintf(*writer, limit, " ");
@ -539,7 +537,7 @@ static size_t ts_tree__write_to_string(const Tree *self,
cursor += snprintf(*writer, limit, "(UNEXPECTED ");
cursor += ts_tree__write_char_to_string(*writer, limit, self->lookahead_char);
} else {
TSSymbol symbol = self->context.rename_symbol ? self->context.rename_symbol : self->symbol;
TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol;
cursor += snprintf(*writer, limit, "(%s", ts_language_symbol_name(language, symbol));
}
}
@ -566,7 +564,7 @@ char *ts_tree_string(const Tree *self, const TSLanguage *language,
void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset,
const TSLanguage *language, FILE *f) {
TSSymbol symbol = self->context.rename_symbol ? self->context.rename_symbol : self->symbol;
TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol;
fprintf(f, "tree_%p [label=\"%s\"", self, ts_language_symbol_name(language, symbol));
if (self->child_count == 0)

15
src/runtime/tree.h
View file

@ -30,7 +30,8 @@ typedef struct Tree {
struct Tree *parent;
uint32_t index;
Length offset;
TSSymbol rename_symbol;
TSSymbol alias_symbol : 15;
bool alias_is_named : 1;
} context;
uint32_t child_count;
@ -39,7 +40,7 @@ typedef struct Tree {
struct Tree **children;
uint32_t visible_child_count;
uint32_t named_child_count;
unsigned short rename_sequence_id;
unsigned short alias_sequence_id;
};
TSExternalTokenState external_token_state;
int32_t lookahead_char;
@ -85,11 +86,11 @@ uint32_t ts_tree_array_essential_count(const TreeArray *);
TreeArray ts_tree_array_remove_last_n(TreeArray *, uint32_t);
TreeArray ts_tree_array_remove_trailing_extras(TreeArray *);
Tree *ts_tree_make_leaf(TSSymbol, Length, Length, TSSymbolMetadata);
Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, TSSymbolMetadata, const TSSymbol *);
Tree *ts_tree_make_leaf(TSSymbol, Length, Length, const TSLanguage *);
Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, unsigned, const TSLanguage *);
Tree *ts_tree_make_copy(Tree *child);
Tree *ts_tree_make_error_node(TreeArray *);
Tree *ts_tree_make_error(Length, Length, int32_t);
Tree *ts_tree_make_error_node(TreeArray *, const TSLanguage *);
Tree *ts_tree_make_error(Length, Length, int32_t, const TSLanguage *);
void ts_tree_retain(Tree *tree);
void ts_tree_release(Tree *tree);
bool ts_tree_eq(const Tree *tree1, const Tree *tree2);
@ -97,7 +98,7 @@ int ts_tree_compare(const Tree *tree1, const Tree *tree2);
uint32_t ts_tree_start_column(const Tree *self);
uint32_t ts_tree_end_column(const Tree *self);
void ts_tree_set_children(Tree *, uint32_t, Tree **, const TSSymbol *);
void ts_tree_set_children(Tree *, uint32_t, Tree **, const TSLanguage *);
void ts_tree_assign_parents(Tree *, TreePath *, const TSLanguage *);
void ts_tree_edit(Tree *, const TSInputEdit *edit);
char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all);

22
src/runtime/tree_path.h
View file

@ -133,19 +133,19 @@ TreePathComparison tree_path_compare(const TreePath *old_path,
const TreePath *new_path,
const TSLanguage *language) {
Tree *old_tree = NULL;
TSSymbol old_rename_symbol = 0;
TSSymbol old_alias_symbol = 0;
Length old_start = length_zero();
for (uint32_t i = old_path->size - 1; i + 1 > 0; i--) {
old_tree = old_path->contents[i].tree;
if (old_tree->visible) {
old_start = old_path->contents[i].position;
if (i > 0) {
const TSSymbol *rename_sequence = ts_language_rename_sequence(
const TSSymbol *alias_sequence = ts_language_alias_sequence(
language,
old_path->contents[i - 1].tree->rename_sequence_id
old_path->contents[i - 1].tree->alias_sequence_id
);
if (rename_sequence) {
old_rename_symbol = rename_sequence[old_path->contents[i].child_index];
if (alias_sequence) {
old_alias_symbol = alias_sequence[old_path->contents[i].child_index];
}
}
break;
@ -153,26 +153,26 @@ TreePathComparison tree_path_compare(const TreePath *old_path,
}
Tree *new_tree = NULL;
TSSymbol new_rename_symbol = 0;
TSSymbol new_alias_symbol = 0;
Length new_start = length_zero();
for (uint32_t i = new_path->size - 1; i + 1 > 0; i--) {
new_tree = new_path->contents[i].tree;
if (new_tree->visible) {
new_start = old_path->contents[i].position;
if (i > 0) {
const TSSymbol *rename_sequence = ts_language_rename_sequence(
const TSSymbol *alias_sequence = ts_language_alias_sequence(
language,
new_path->contents[i - 1].tree->rename_sequence_id
new_path->contents[i - 1].tree->alias_sequence_id
);
if (rename_sequence) {
new_rename_symbol = rename_sequence[new_path->contents[i].child_index];
if (alias_sequence) {
new_alias_symbol = alias_sequence[new_path->contents[i].child_index];
}
}
break;
}
}
if (old_rename_symbol == new_rename_symbol) {
if (old_alias_symbol == new_alias_symbol) {
if (old_start.bytes == new_start.bytes) {
if (!old_tree->has_changes &&
old_tree->symbol == new_tree->symbol &&

22
test/compiler/build_tables/parse_item_set_builder_test.cc
View file

@ -28,23 +28,23 @@ describe("ParseItemSetBuilder", []() {
SyntaxGrammar grammar{{
SyntaxVariable{"rule0", VariableTypeNamed, {
Production{{
{Symbol::non_terminal(1), 0, AssociativityNone, ""},
{Symbol::terminal(11), 0, AssociativityNone, ""},
{Symbol::non_terminal(1), 0, AssociativityNone, Alias{}},
{Symbol::terminal(11), 0, AssociativityNone, Alias{}},
}, 0},
}},
SyntaxVariable{"rule1", VariableTypeNamed, {
Production{{
{Symbol::terminal(12), 0, AssociativityNone, ""},
{Symbol::terminal(13), 0, AssociativityNone, ""},
{Symbol::terminal(12), 0, AssociativityNone, Alias{}},
{Symbol::terminal(13), 0, AssociativityNone, Alias{}},
}, 0},
Production{{
{Symbol::non_terminal(2), 0, AssociativityNone, ""},
{Symbol::non_terminal(2), 0, AssociativityNone, Alias{}},
}, 0}
}},
SyntaxVariable{"rule2", VariableTypeNamed, {
Production{{
{Symbol::terminal(14), 0, AssociativityNone, ""},
{Symbol::terminal(15), 0, AssociativityNone, ""},
{Symbol::terminal(14), 0, AssociativityNone, Alias{}},
{Symbol::terminal(15), 0, AssociativityNone, Alias{}},
}, 0}
}},
}, {}, {}, {}, {}};
@ -87,14 +87,14 @@ describe("ParseItemSetBuilder", []() {
SyntaxGrammar grammar{{
SyntaxVariable{"rule0", VariableTypeNamed, {
Production{{
{Symbol::non_terminal(1), 0, AssociativityNone, ""},
{Symbol::terminal(11), 0, AssociativityNone, ""},
{Symbol::non_terminal(1), 0, AssociativityNone, Alias{}},
{Symbol::terminal(11), 0, AssociativityNone, Alias{}},
}, 0},
}},
SyntaxVariable{"rule1", VariableTypeNamed, {
Production{{
{Symbol::terminal(12), 0, AssociativityNone, ""},
{Symbol::terminal(13), 0, AssociativityNone, ""},
{Symbol::terminal(12), 0, AssociativityNone, Alias{}},
{Symbol::terminal(13), 0, AssociativityNone, Alias{}},
}, 0},
Production{{}, 0}
}},

6
test/fixtures/test_grammars/inlined_renamed_rules/grammar.json vendored
View file

@ -24,8 +24,9 @@
{"type": "SYMBOL", "name": "call_expression"},
{"type": "SYMBOL", "name": "member_expression"},
{
"type": "RENAME",
"type": "ALIAS",
"value": "variable_name",
"named": true,
"content": {
"type": "SYMBOL",
"name": "identifier"
@ -57,7 +58,8 @@
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "."},
{
"type": "RENAME",
"type": "ALIAS",
"named": true,
"value": "property_name",
"content": {
"type": "SYMBOL",

6
test/fixtures/test_grammars/renamed_inlined_rules/grammar.json vendored
View file

@ -15,8 +15,9 @@
"members": [
{"type": "SYMBOL", "name": "member_expression"},
{
"type": "RENAME",
"type": "ALIAS",
"value": "variable_name",
"named": true,
"content": {
"type": "SYMBOL",
"name": "identifier"
@ -34,8 +35,9 @@
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "."},
{
"type": "RENAME",
"type": "ALIAS",
"value": "property_name",
"named": true,
"content": {
"type": "SYMBOL",
"name": "identifier"

6
test/fixtures/test_grammars/renamed_rules/grammar.json vendored
View file

@ -20,7 +20,8 @@
{"type": "SYMBOL", "name": "call_expression"},
{"type": "SYMBOL", "name": "member_expression"},
{
"type": "RENAME",
"type": "ALIAS",
"named": true,
"value": "variable_name",
"content": {
"type": "SYMBOL",
@ -53,7 +54,8 @@
{"type": "SYMBOL", "name": "_expression"},
{"type": "STRING", "value": "."},
{
"type": "RENAME",
"type": "ALIAS",
"named": true,
"value": "property_name",
"content": {
"type": "SYMBOL",

9
test/runtime/stack_test.cc
View file

@ -78,10 +78,11 @@ describe("Stack", [&]() {
stack = ts_stack_new();
for (size_t i = 0; i < tree_count; i++)
trees[i] = ts_tree_make_leaf(i, length_zero(), tree_len, {
true, true, false, true,
});
TSLanguage dummy_language;
for (size_t i = 0; i < tree_count; i++) {
trees[i] = ts_tree_make_leaf(i, length_zero(), tree_len, &dummy_language);
}
});
after_each([&]() {

76
test/runtime/tree_test.cc
View file

@ -35,12 +35,14 @@ describe("Tree", []() {
symbol9,
};
TSSymbolMetadata visible = {true, true, false, true};
TSSymbolMetadata invisible = {false, false, false, true};
TSSymbolMetadata metadata_list[30] = {};
TSLanguage language;
language.symbol_metadata = metadata_list;
describe("make_leaf", [&]() {
it("does not mark the tree as fragile", [&]() {
Tree *tree = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, visible);
Tree *tree = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, &language);
AssertThat(tree->fragile_left, IsFalse());
AssertThat(tree->fragile_right, IsFalse());
});
@ -51,7 +53,9 @@ describe("Tree", []() {
Tree *error_tree = ts_tree_make_error(
length_zero(),
length_zero(),
'z');
'z',
&language
);
AssertThat(error_tree->fragile_left, IsTrue());
AssertThat(error_tree->fragile_right, IsTrue());
@ -64,15 +68,15 @@ describe("Tree", []() {
Tree *tree1, *tree2, *parent1;
before_each([&]() {
tree1 = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, visible);
tree2 = ts_tree_make_leaf(symbol2, {1, 1, {0, 1}}, {3, 3, {0, 3}}, visible);
tree1 = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, &language);
tree2 = ts_tree_make_leaf(symbol2, {1, 1, {0, 1}}, {3, 3, {0, 3}}, &language);
ts_tree_retain(tree1);
ts_tree_retain(tree2);
parent1 = ts_tree_make_node(symbol3, 2, tree_array({
tree1,
tree2,
}), visible, nullptr);
}), 0, &language);
});
after_each([&]() {
@ -103,7 +107,7 @@ describe("Tree", []() {
parent = ts_tree_make_node(symbol3, 2, tree_array({
tree1,
tree2,
}), visible, nullptr);
}), 0, &language);
});
after_each([&]() {
@ -127,7 +131,7 @@ describe("Tree", []() {
parent = ts_tree_make_node(symbol3, 2, tree_array({
tree1,
tree2,
}), visible, nullptr);
}), 0, &language);
});
after_each([&]() {
@ -151,7 +155,7 @@ describe("Tree", []() {
parent = ts_tree_make_node(symbol3, 2, tree_array({
tree1,
tree2,
}), visible, nullptr);
}), 0, &language);
});
after_each([&]() {
@ -170,10 +174,10 @@ describe("Tree", []() {
before_each([&]() {
tree = ts_tree_make_node(symbol1, 3, tree_array({
ts_tree_make_leaf(symbol2, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible),
ts_tree_make_leaf(symbol3, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible),
ts_tree_make_leaf(symbol4, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible),
}), visible, nullptr);
ts_tree_make_leaf(symbol2, {2, 2, {0, 2}}, {3, 3, {0, 3}}, &language),
ts_tree_make_leaf(symbol3, {2, 2, {0, 2}}, {3, 3, {0, 3}}, &language),
ts_tree_make_leaf(symbol4, {2, 2, {0, 2}}, {3, 3, {0, 3}}, &language),
}), 0, &language);
AssertThat(tree->padding, Equals<Length>({2, 2, {0, 2}}));
AssertThat(tree->size, Equals<Length>({13, 13, {0, 13}}));
@ -336,7 +340,7 @@ describe("Tree", []() {
Tree *leaf;
before_each([&]() {
leaf = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, visible);
leaf = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, &language);
});
after_each([&]() {
@ -344,20 +348,20 @@ describe("Tree", []() {
});
it("returns true for identical trees", [&]() {
Tree *leaf_copy = ts_tree_make_leaf(symbol1, {2, 1, {1, 1}}, {5, 4, {1, 4}}, visible);
Tree *leaf_copy = ts_tree_make_leaf(symbol1, {2, 1, {1, 1}}, {5, 4, {1, 4}}, &language);
AssertThat(ts_tree_eq(leaf, leaf_copy), IsTrue());
Tree *parent = ts_tree_make_node(symbol2, 2, tree_array({
leaf,
leaf_copy,
}), visible, nullptr);
}), 0, &language);
ts_tree_retain(leaf);
ts_tree_retain(leaf_copy);
Tree *parent_copy = ts_tree_make_node(symbol2, 2, tree_array({
leaf,
leaf_copy,
}), visible, nullptr);
}), 0, &language);
ts_tree_retain(leaf);
ts_tree_retain(leaf_copy);
@ -373,42 +377,44 @@ describe("Tree", []() {
leaf->symbol + 1,
leaf->padding,
leaf->size,
visible);
&language
);
AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse());
ts_tree_release(different_leaf);
});
it("returns false for trees with different options", [&]() {
Tree *different_leaf = ts_tree_make_leaf(symbol1, leaf->padding, leaf->size, invisible);
Tree *different_leaf = ts_tree_make_leaf(leaf->symbol, leaf->padding, leaf->size, &language);
different_leaf->visible = !leaf->visible;
AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse());
ts_tree_release(different_leaf);
});
it("returns false for trees with different sizes", [&]() {
Tree *different_leaf = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, leaf->size, invisible);
it("returns false for trees with different paddings or sizes", [&]() {
Tree *different_leaf = ts_tree_make_leaf(leaf->symbol, {}, leaf->size, &language);
AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse());
ts_tree_release(different_leaf);
different_leaf = ts_tree_make_leaf(symbol1, leaf->padding, {5, 4, {1, 10}}, invisible);
different_leaf = ts_tree_make_leaf(symbol1, leaf->padding, {}, &language);
AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse());
ts_tree_release(different_leaf);
});
it("returns false for trees with different children", [&]() {
Tree *leaf2 = ts_tree_make_leaf(symbol2, {1, 1, {0, 1}}, {3, 3, {0, 3}}, visible);
Tree *leaf2 = ts_tree_make_leaf(symbol2, {1, 1, {0, 1}}, {3, 3, {0, 3}}, &language);
Tree *parent = ts_tree_make_node(symbol2, 2, tree_array({
leaf,
leaf2,
}), visible, nullptr);
}), 0, &language);
ts_tree_retain(leaf);
ts_tree_retain(leaf2);
Tree *different_parent = ts_tree_make_node(symbol2, 2, tree_array({
leaf2,
leaf,
}), visible, nullptr);
}), 0, &language);
ts_tree_retain(leaf2);
ts_tree_retain(leaf);
@ -435,17 +441,17 @@ describe("Tree", []() {
tree1 = ts_tree_make_node(symbol1, 2, tree_array({
(tree2 = ts_tree_make_node(symbol2, 3, tree_array({
(tree3 = make_external(ts_tree_make_leaf(symbol3, padding, size, visible))),
(tree4 = ts_tree_make_leaf(symbol4, padding, size, visible)),
(tree5 = ts_tree_make_leaf(symbol5, padding, size, visible)),
}), visible, nullptr)),
(tree3 = make_external(ts_tree_make_leaf(symbol3, padding, size, &language))),
(tree4 = ts_tree_make_leaf(symbol4, padding, size, &language)),
(tree5 = ts_tree_make_leaf(symbol5, padding, size, &language)),
}), 0, &language)),
(tree6 = ts_tree_make_node(symbol6, 2, tree_array({
(tree7 = ts_tree_make_node(symbol7, 1, tree_array({
(tree8 = ts_tree_make_leaf(symbol8, padding, size, visible)),
}), visible, nullptr)),
(tree9 = ts_tree_make_leaf(symbol9, padding, size, visible)),
}), visible, nullptr)),
}), visible, nullptr);
(tree8 = ts_tree_make_leaf(symbol8, padding, size, &language)),
}), 0, &language)),
(tree9 = ts_tree_make_leaf(symbol9, padding, size, &language)),
}), 0, &language)),
}), 0, &language);
auto token = ts_tree_last_external_token(tree1);
AssertThat(token, Equals(tree3));