From cb5fe8034808ef7984d27fc3c8efa799d27724de Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 31 Jul 2017 11:45:24 -0700 Subject: [PATCH] Rename RENAME rule to ALIAS, allow it to create anonymous nodes --- doc/grammar-schema.json | 11 ++- include/tree_sitter/parser.h | 44 ++++----- .../build_tables/build_parse_table.cc | 34 +++---- src/compiler/build_tables/parse_item.cc | 11 +-- .../build_tables/parse_item_set_builder.cc | 4 +- src/compiler/generate_code/c_code.cc | 62 ++++++------ src/compiler/parse_grammar.cc | 17 +++- src/compiler/parse_table.cc | 10 +- src/compiler/parse_table.h | 10 +- .../prepare_grammar/flatten_grammar.cc | 14 +-- src/compiler/rules/metadata.cc | 18 +++- src/compiler/rules/metadata.h | 14 ++- src/compiler/syntax_grammar.cc | 4 +- src/compiler/syntax_grammar.h | 2 +- src/runtime/language.c | 2 +- src/runtime/language.h | 4 +- src/runtime/node.c | 6 +- src/runtime/parser.c | 49 ++++------ src/runtime/reduce_action.h | 2 +- src/runtime/tree.c | 94 +++++++++---------- src/runtime/tree.h | 15 +-- src/runtime/tree_path.h | 22 ++--- .../parse_item_set_builder_test.cc | 22 ++--- .../inlined_renamed_rules/grammar.json | 6 +- .../renamed_inlined_rules/grammar.json | 6 +- .../test_grammars/renamed_rules/grammar.json | 6 +- test/runtime/stack_test.cc | 9 +- test/runtime/tree_test.cc | 76 ++++++++------- 28 files changed, 304 insertions(+), 270 deletions(-) diff --git a/doc/grammar-schema.json b/doc/grammar-schema.json index e66fe021..aeafc78c 100644 --- a/doc/grammar-schema.json +++ b/doc/grammar-schema.json @@ -143,21 +143,24 @@ "required": ["type", "members"] }, - "rename-rule": { + "alias-rule": { "type": "object", "properties": { "type": { "type": "string", - "pattern": "^RENAME$" + "pattern": "^ALIAS$" }, "value": { "type": "string" }, + "named": { + "type": "boolean" + }, "content": { "$ref": "#/definitions/rule" } }, - "required": ["type", "content", "value"] + "required": ["type", "named", "content", "value"] }, "repeat-rule": { @@ -221,13 +224,13 @@ "rule": { "oneOf": [ + { "$ref": "#/definitions/alias-rule" }, { "$ref": "#/definitions/blank-rule" }, { "$ref": "#/definitions/string-rule" }, { "$ref": "#/definitions/pattern-rule" }, { "$ref": "#/definitions/symbol-rule" }, { "$ref": "#/definitions/seq-rule" }, { "$ref": "#/definitions/choice-rule" }, - { "$ref": "#/definitions/rename-rule" }, { "$ref": "#/definitions/repeat1-rule" }, { "$ref": "#/definitions/repeat-rule" }, { "$ref": "#/definitions/token-rule" }, diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index 4ad4880e..59fd5606 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -47,7 +47,7 @@ typedef struct { TSSymbol symbol; int16_t dynamic_precedence; uint8_t child_count; - uint8_t rename_sequence_id : 7; + uint8_t alias_sequence_id : 7; bool fragile : 1; }; } params; @@ -71,7 +71,7 @@ typedef union { typedef struct TSLanguage { uint32_t version; uint32_t symbol_count; - uint32_t rename_symbol_count; + uint32_t alias_count; uint32_t token_count; uint32_t external_token_count; const char **symbol_names; @@ -79,8 +79,8 @@ typedef struct TSLanguage { const uint16_t *parse_table; const TSParseActionEntry *parse_actions; const TSLexMode *lex_modes; - const TSSymbol *rename_sequences; - uint16_t max_rename_sequence_length; + const TSSymbol *alias_sequences; + uint16_t max_alias_sequence_length; bool (*lex_fn)(TSLexer *, TSStateId); struct { const bool *states; @@ -172,24 +172,24 @@ typedef struct TSLanguage { { .type = TSParseActionTypeAccept } \ } -#define GET_LANGUAGE(...) \ - static TSLanguage language = { \ - .version = LANGUAGE_VERSION, \ - .symbol_count = SYMBOL_COUNT, \ - .rename_symbol_count = RENAME_SYMBOL_COUNT, \ - .token_count = TOKEN_COUNT, \ - .symbol_metadata = ts_symbol_metadata, \ - .parse_table = (const unsigned short *)ts_parse_table, \ - .parse_actions = ts_parse_actions, \ - .lex_modes = ts_lex_modes, \ - .symbol_names = ts_symbol_names, \ - .rename_sequences = (const TSSymbol *)ts_rename_sequences, \ - .max_rename_sequence_length = MAX_RENAME_SEQUENCE_LENGTH, \ - .lex_fn = ts_lex, \ - .external_token_count = EXTERNAL_TOKEN_COUNT, \ - .external_scanner = {__VA_ARGS__} \ - }; \ - return &language \ +#define GET_LANGUAGE(...) \ + static TSLanguage language = { \ + .version = LANGUAGE_VERSION, \ + .symbol_count = SYMBOL_COUNT, \ + .alias_count = ALIAS_COUNT, \ + .token_count = TOKEN_COUNT, \ + .symbol_metadata = ts_symbol_metadata, \ + .parse_table = (const unsigned short *)ts_parse_table, \ + .parse_actions = ts_parse_actions, \ + .lex_modes = ts_lex_modes, \ + .symbol_names = ts_symbol_names, \ + .alias_sequences = (const TSSymbol *)ts_alias_sequences, \ + .max_alias_sequence_length = MAX_ALIAS_SEQUENCE_LENGTH, \ + .lex_fn = ts_lex, \ + .external_token_count = EXTERNAL_TOKEN_COUNT, \ + .external_scanner = {__VA_ARGS__} \ + }; \ + return &language \ #ifdef __cplusplus } diff --git a/src/compiler/build_tables/build_parse_table.cc b/src/compiler/build_tables/build_parse_table.cc index fe4c43ee..00ecba5f 100644 --- a/src/compiler/build_tables/build_parse_table.cc +++ b/src/compiler/build_tables/build_parse_table.cc @@ -64,7 +64,7 @@ class ParseTableBuilder { pair build() { // Ensure that the empty rename sequence has index 0. - parse_table.rename_sequences.push_back({}); + parse_table.alias_sequences.push_back({}); // Ensure that the error state has index 0. ParseStateId error_state_id = add_parse_state({}, ParseItemSet{}); @@ -73,7 +73,7 @@ class ParseTableBuilder { Symbol start_symbol = grammar.variables.empty() ? Symbol::terminal(0) : Symbol::non_terminal(0); - Production start_production{{{start_symbol, 0, rules::AssociativityNone, ""}}, 0}; + Production start_production{{{start_symbol, 0, rules::AssociativityNone, {"", false}}}, 0}; add_parse_state({}, ParseItemSet{{ { ParseItem(rules::START(), start_production, 0), @@ -204,7 +204,7 @@ class ParseTableBuilder { item.precedence(), item.production->dynamic_precedence, item.associativity(), - get_rename_sequence_id(*item.production) + get_alias_sequence_id(*item.production) ); lookahead_symbols.for_each([&](Symbol lookahead) { @@ -716,30 +716,30 @@ class ParseTableBuilder { } } - unsigned get_rename_sequence_id(const Production &production) { - bool has_rename = false; - RenameSequence rename_sequence; + unsigned get_alias_sequence_id(const Production &production) { + bool has_alias = false; + AliasSequence alias_sequence; for (unsigned i = 0, n = production.size(); i < n; i++) { auto &step = production.at(i); - if (!step.name_replacement.empty()) { - has_rename = true; - rename_sequence.resize(i + 1); - rename_sequence[i] = step.name_replacement; + if (!step.alias.value.empty()) { + has_alias = true; + alias_sequence.resize(i + 1); + alias_sequence[i] = step.alias; } } - if (has_rename && production.size() > parse_table.max_rename_sequence_length) { - parse_table.max_rename_sequence_length = production.size(); + if (has_alias && production.size() > parse_table.max_alias_sequence_length) { + parse_table.max_alias_sequence_length = production.size(); } - auto begin = parse_table.rename_sequences.begin(); - auto end = parse_table.rename_sequences.end(); - auto iter = find(begin, end, rename_sequence); + auto begin = parse_table.alias_sequences.begin(); + auto end = parse_table.alias_sequences.end(); + auto iter = find(begin, end, alias_sequence); if (iter != end) { return iter - begin; } else { - parse_table.rename_sequences.push_back(move(rename_sequence)); - return parse_table.rename_sequences.size() - 1; + parse_table.alias_sequences.push_back(move(alias_sequence)); + return parse_table.alias_sequences.size() - 1; } } diff --git a/src/compiler/build_tables/parse_item.cc b/src/compiler/build_tables/parse_item.cc index 5184a98a..7e4d0c64 100644 --- a/src/compiler/build_tables/parse_item.cc +++ b/src/compiler/build_tables/parse_item.cc @@ -28,9 +28,7 @@ bool ParseItem::operator==(const ParseItem &other) const { if (variable_index != other.variable_index) return false; if (production->size() != other.production->size()) return false; for (size_t i = 0; i < step_index; i++) { - if (production->at(i).name_replacement != other.production->at(i).name_replacement) { - return false; - } + if (production->at(i).alias != other.production->at(i).alias) return false; } if (is_done()) { if (!production->empty()) { @@ -53,8 +51,8 @@ bool ParseItem::operator<(const ParseItem &other) const { if (production->size() < other.production->size()) return true; if (other.production->size() < production->size()) return false; for (size_t i = 0; i < step_index; i++) { - if (production->at(i).name_replacement < other.production->at(i).name_replacement) return true; - if (other.production->at(i).name_replacement < production->at(i).name_replacement) return false; + if (production->at(i).alias < other.production->at(i).alias) return true; + if (other.production->at(i).alias < production->at(i).alias) return false; } if (is_done()) { if (!production->empty()) { @@ -158,7 +156,8 @@ struct hash { hash_combine(&result, item.production->dynamic_precedence); hash_combine(&result, item.production->size()); for (size_t i = 0; i < item.step_index; i++) { - hash_combine(&result, item.production->at(i).name_replacement); + hash_combine(&result, item.production->at(i).alias.value); + hash_combine(&result, item.production->at(i).alias.is_named); } if (item.is_done()) { if (!item.production->empty()) { diff --git a/src/compiler/build_tables/parse_item_set_builder.cc b/src/compiler/build_tables/parse_item_set_builder.cc index 5bfb2745..6994d200 100644 --- a/src/compiler/build_tables/parse_item_set_builder.cc +++ b/src/compiler/build_tables/parse_item_set_builder.cc @@ -211,8 +211,8 @@ const vector &ParseItemSetBuilder::inline_production(const ParseItem Production production{{begin, step}, item.production->dynamic_precedence}; for (auto &step : *production_to_insert) { production.steps.push_back(step); - if (!inlined_step.name_replacement.empty()) { - production.steps.back().name_replacement = inlined_step.name_replacement; + if (!inlined_step.alias.value.empty()) { + production.steps.back().alias = inlined_step.alias; } } production.back().precedence = inlined_step.precedence; diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index 95feb03b..8f50496e 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -25,6 +25,7 @@ using std::to_string; using std::vector; using util::escape_char; using rules::Symbol; +using rules::Alias; static const map REPLACEMENTS({ { '~', "TILDE" }, @@ -76,7 +77,7 @@ class CCodeGenerator { vector> parse_table_entries; vector> external_scanner_states; size_t next_parse_action_list_index; - set unique_replacement_names; + set unique_aliases; public: CCodeGenerator(string name, const ParseTable &parse_table, @@ -99,7 +100,7 @@ class CCodeGenerator { add_symbol_enum(); add_symbol_names_list(); add_symbol_metadata_list(); - add_rename_sequences(); + add_alias_sequences(); add_lex_function(); add_lex_modes_list(); @@ -141,10 +142,10 @@ class CCodeGenerator { } } - for (const RenameSequence &rename_sequence : parse_table.rename_sequences) { - for (const string &name_replacement : rename_sequence) { - if (!name_replacement.empty()) { - unique_replacement_names.insert(name_replacement); + for (const AliasSequence &alias_sequence : parse_table.alias_sequences) { + for (const Alias &alias : alias_sequence) { + if (!alias.value.empty()) { + unique_aliases.insert(alias); } } } @@ -152,10 +153,10 @@ class CCodeGenerator { line("#define LANGUAGE_VERSION " + to_string(TREE_SITTER_LANGUAGE_VERSION)); line("#define STATE_COUNT " + to_string(parse_table.states.size())); line("#define SYMBOL_COUNT " + to_string(parse_table.symbols.size())); - line("#define RENAME_SYMBOL_COUNT " + to_string(unique_replacement_names.size())); + line("#define ALIAS_COUNT " + to_string(unique_aliases.size())); line("#define TOKEN_COUNT " + to_string(token_count)); line("#define EXTERNAL_TOKEN_COUNT " + to_string(syntax_grammar.external_tokens.size())); - line("#define MAX_RENAME_SEQUENCE_LENGTH " + to_string(parse_table.max_rename_sequence_length)); + line("#define MAX_ALIAS_SEQUENCE_LENGTH " + to_string(parse_table.max_alias_sequence_length)); line(); } @@ -171,8 +172,8 @@ class CCodeGenerator { } } - for (const string &replacement_name : unique_replacement_names) { - line(rename_id(replacement_name) + " = " + to_string(i) + ","); + for (const Alias &alias : unique_aliases) { + line(alias_id(alias) + " = " + to_string(i) + ","); i++; } }); @@ -190,10 +191,10 @@ class CCodeGenerator { ); } - for (const string &replacement_name : unique_replacement_names) { + for (const Alias &alias : unique_aliases) { line( - "[" + rename_id(replacement_name) + "] = \"" + - sanitize_name_for_string(replacement_name) + "\"," + "[" + alias_id(alias) + "] = \"" + + sanitize_name_for_string(alias.value) + "\"," ); } }); @@ -201,22 +202,21 @@ class CCodeGenerator { line(); } - void add_rename_sequences() { - + void add_alias_sequences() { line( - "static TSSymbol ts_rename_sequences[" + - to_string(parse_table.rename_sequences.size()) + - "][MAX_RENAME_SEQUENCE_LENGTH] = {" + "static TSSymbol ts_alias_sequences[" + + to_string(parse_table.alias_sequences.size()) + + "][MAX_ALIAS_SEQUENCE_LENGTH] = {" ); indent([&]() { - for (unsigned i = 1, n = parse_table.rename_sequences.size(); i < n; i++) { - const RenameSequence &sequence = parse_table.rename_sequences[i]; + for (unsigned i = 1, n = parse_table.alias_sequences.size(); i < n; i++) { + const AliasSequence &sequence = parse_table.alias_sequences[i]; line("[" + to_string(i) + "] = {"); indent([&]() { for (unsigned j = 0, n = sequence.size(); j < n; j++) { - if (!sequence[j].empty()) { - line("[" + to_string(j) + "] = " + rename_id(sequence[j]) + ","); + if (!sequence[j].value.empty()) { + line("[" + to_string(j) + "] = " + alias_id(sequence[j]) + ","); } } }); @@ -260,11 +260,11 @@ class CCodeGenerator { line("},"); } - for (const string &replacement_name : unique_replacement_names) { - line("[" + rename_id(replacement_name) + "] = {"); + for (const Alias &alias : unique_aliases) { + line("[" + alias_id(alias) + "] = {"); indent([&]() { line(".visible = true,"); - line(".named = true,"); + line(".named = " + _boolean(alias.is_named) + ","); line(".structural = true,"); line(".extra = true,"); }); @@ -616,8 +616,8 @@ class CCodeGenerator { add(", .dynamic_precedence = " + to_string(action.dynamic_precedence)); } - if (action.rename_sequence_id != 0) { - add(", .rename_sequence_id = " + to_string(action.rename_sequence_id)); + if (action.alias_sequence_id != 0) { + add(", .alias_sequence_id = " + to_string(action.alias_sequence_id)); } add(")"); @@ -671,8 +671,12 @@ class CCodeGenerator { } } - string rename_id(const string &name) { - return "rename_sym_" + sanitize_name(name); + string alias_id(const Alias &alias) { + if (alias.is_named) { + return "alias_sym_" + sanitize_name(alias.value); + } else { + return "anon_alias_sym_" + sanitize_name(alias.value); + } } string symbol_name(const Symbol &symbol) { diff --git a/src/compiler/parse_grammar.cc b/src/compiler/parse_grammar.cc index 08ebb6c7..4764542a 100644 --- a/src/compiler/parse_grammar.cc +++ b/src/compiler/parse_grammar.cc @@ -198,18 +198,27 @@ ParseRuleResult parse_rule(json_value *rule_json) { return Rule(Metadata::prec_dynamic(precedence_json.u.integer, result.rule)); } - if (type == "RENAME") { - json_value name_json = rule_json->operator[]("value"); - if (name_json.type != json_string) { + if (type == "ALIAS") { + json_value value_json = rule_json->operator[]("value"); + if (value_json.type != json_string) { return "Rename value must be a string"; } + json_value is_named_json = rule_json->operator[]("named"); + if (is_named_json.type != json_boolean) { + return "Rename named value must be a boolean"; + } + json_value content_json = rule_json->operator[]("content"); auto result = parse_rule(&content_json); if (!result.error_message.empty()) { return "Invalid rename content: " + result.error_message; } - return Rule(Metadata::rename(string(name_json.u.string.ptr), result.rule)); + return Rule(Metadata::alias( + string(value_json.u.string.ptr), + is_named_json.u.boolean, + result.rule + )); } return "Unknown rule type: " + type; diff --git a/src/compiler/parse_table.cc b/src/compiler/parse_table.cc index 369b3778..e79e02dd 100644 --- a/src/compiler/parse_table.cc +++ b/src/compiler/parse_table.cc @@ -19,7 +19,7 @@ ParseAction::ParseAction() : precedence(0), dynamic_precedence(0), associativity(rules::AssociativityNone), - rename_sequence_id(0), + alias_sequence_id(0), fragile(false), extra(false) {} @@ -56,7 +56,7 @@ ParseAction ParseAction::ShiftExtra() { ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count, int precedence, int dynamic_precedence, - rules::Associativity associativity, unsigned rename_sequence_id) { + rules::Associativity associativity, unsigned alias_sequence_id) { ParseAction result; result.type = ParseActionTypeReduce; result.symbol = symbol; @@ -64,7 +64,7 @@ ParseAction ParseAction::Reduce(Symbol symbol, size_t consumed_symbol_count, result.precedence = precedence; result.dynamic_precedence = dynamic_precedence; result.associativity = associativity; - result.rename_sequence_id = rename_sequence_id; + result.alias_sequence_id = alias_sequence_id; return result; } @@ -77,7 +77,7 @@ bool ParseAction::operator==(const ParseAction &other) const { precedence == other.precedence && dynamic_precedence == other.dynamic_precedence && associativity == other.associativity && - rename_sequence_id == other.rename_sequence_id && + alias_sequence_id == other.alias_sequence_id && extra == other.extra && fragile == other.fragile; } @@ -101,7 +101,7 @@ bool ParseAction::operator<(const ParseAction &other) const { if (other.extra && !extra) return false; if (fragile && !other.fragile) return true; if (other.fragile && !fragile) return false; - return rename_sequence_id < other.rename_sequence_id; + return alias_sequence_id < other.alias_sequence_id; } ParseTableEntry::ParseTableEntry() diff --git a/src/compiler/parse_table.h b/src/compiler/parse_table.h index 82fcb747..40a44dfe 100644 --- a/src/compiler/parse_table.h +++ b/src/compiler/parse_table.h @@ -30,7 +30,7 @@ struct ParseAction { static ParseAction Recover(ParseStateId state_index); static ParseAction Reduce(rules::Symbol symbol, size_t child_count, int precedence, int dynamic_precedence, rules::Associativity, - unsigned rename_sequence_id); + unsigned alias_sequence_id); static ParseAction ShiftExtra(); bool operator==(const ParseAction &) const; bool operator<(const ParseAction &) const; @@ -42,7 +42,7 @@ struct ParseAction { int precedence; int dynamic_precedence; rules::Associativity associativity; - unsigned rename_sequence_id; + unsigned alias_sequence_id; bool fragile; bool extra; }; @@ -77,7 +77,7 @@ struct ParseTableSymbolMetadata { bool structural; }; -using RenameSequence = std::vector; +using AliasSequence = std::vector; struct ParseTable { ParseAction &add_terminal_action(ParseStateId state_id, rules::Symbol, ParseAction); @@ -85,8 +85,8 @@ struct ParseTable { std::vector states; std::map symbols; - std::vector rename_sequences; - unsigned max_rename_sequence_length = 0; + std::vector alias_sequences; + unsigned max_alias_sequence_length = 0; }; } // namespace tree_sitter diff --git a/src/compiler/prepare_grammar/flatten_grammar.cc b/src/compiler/prepare_grammar/flatten_grammar.cc index 667dc9a7..df8a1e1b 100644 --- a/src/compiler/prepare_grammar/flatten_grammar.cc +++ b/src/compiler/prepare_grammar/flatten_grammar.cc @@ -22,7 +22,7 @@ class FlattenRule { private: vector precedence_stack; vector associativity_stack; - vector name_replacement_stack; + vector alias_stack; Production production; void apply(const Rule &rule, bool at_end) { @@ -32,7 +32,7 @@ class FlattenRule { symbol, precedence_stack.back(), associativity_stack.back(), - name_replacement_stack.back() + alias_stack.back() }); }, @@ -45,8 +45,8 @@ class FlattenRule { associativity_stack.push_back(metadata.params.associativity); } - if (!metadata.params.name_replacement.empty()) { - name_replacement_stack.push_back(metadata.params.name_replacement); + if (!metadata.params.alias.value.empty()) { + alias_stack.push_back(metadata.params.alias); } if (abs(metadata.params.dynamic_precedence) > abs(production.dynamic_precedence)) { @@ -65,8 +65,8 @@ class FlattenRule { if (!at_end) production.back().associativity = associativity_stack.back(); } - if (!metadata.params.name_replacement.empty()) { - name_replacement_stack.pop_back(); + if (!metadata.params.alias.value.empty()) { + alias_stack.pop_back(); } }, @@ -87,7 +87,7 @@ class FlattenRule { FlattenRule() : precedence_stack({0}), associativity_stack({rules::AssociativityNone}), - name_replacement_stack({""}) {} + alias_stack({rules::Alias{}}) {} Production flatten(const Rule &rule) { apply(rule, true); diff --git a/src/compiler/rules/metadata.cc b/src/compiler/rules/metadata.cc index 5a6e59b0..a602b0f6 100644 --- a/src/compiler/rules/metadata.cc +++ b/src/compiler/rules/metadata.cc @@ -9,6 +9,20 @@ namespace rules { using std::move; using std::string; +bool Alias::operator==(const Alias &other) const { + return value == other.value && is_named == other.is_named; +} + +bool Alias::operator!=(const Alias &other) const { + return !operator==(other); +} + +bool Alias::operator<(const Alias &other) const { + if (value < other.value) return true; + if (other.value < value) return false; + return is_named < other.is_named; +} + Metadata::Metadata(const Rule &rule, MetadataParams params) : rule(std::make_shared(rule)), params(params) {} @@ -77,9 +91,9 @@ Metadata Metadata::main_token(const Rule &rule) { return Metadata{rule, params}; } -Metadata Metadata::rename(string &&name, const Rule &rule) { +Metadata Metadata::alias(string &&value, bool is_named, const Rule &rule) { MetadataParams params; - params.name_replacement = move(name); + params.alias = {move(value), is_named}; return Metadata{rule, params}; } diff --git a/src/compiler/rules/metadata.h b/src/compiler/rules/metadata.h index e9ff08f4..15d32c00 100644 --- a/src/compiler/rules/metadata.h +++ b/src/compiler/rules/metadata.h @@ -13,6 +13,14 @@ enum Associativity { AssociativityRight, }; +struct Alias { + std::string value = ""; + bool is_named = false; + bool operator==(const Alias &) const; + bool operator!=(const Alias &) const; + bool operator<(const Alias &) const; +}; + struct MetadataParams { int precedence; int dynamic_precedence; @@ -23,7 +31,7 @@ struct MetadataParams { bool is_string; bool is_active; bool is_main_token; - std::string name_replacement; + Alias alias; inline MetadataParams() : precedence{0}, dynamic_precedence{0}, associativity{AssociativityNone}, @@ -41,7 +49,7 @@ struct MetadataParams { is_string == other.is_string && is_active == other.is_active && is_main_token == other.is_main_token && - name_replacement == other.name_replacement + alias == other.alias ); } }; @@ -62,7 +70,7 @@ struct Metadata { static Metadata prec_dynamic(int precedence, const Rule &rule); static Metadata separator(const Rule &rule); static Metadata main_token(const Rule &rule); - static Metadata rename(std::string &&name, const Rule &rule); + static Metadata alias(std::string &&value, bool is_named, const Rule &rule); bool operator==(const Metadata &other) const; }; diff --git a/src/compiler/syntax_grammar.cc b/src/compiler/syntax_grammar.cc index 34534e3c..4c75173a 100644 --- a/src/compiler/syntax_grammar.cc +++ b/src/compiler/syntax_grammar.cc @@ -6,7 +6,7 @@ bool ProductionStep::operator==(const ProductionStep &other) const { return symbol == other.symbol && precedence == other.precedence && associativity == other.associativity && - name_replacement == other.name_replacement; + alias == other.alias; } bool ProductionStep::operator!=(const ProductionStep &other) const { @@ -20,7 +20,7 @@ bool ProductionStep::operator<(const ProductionStep &other) const { if (other.precedence < precedence) return false; if (associativity < other.associativity) return true; if (other.associativity < associativity) return false; - return name_replacement < other.name_replacement; + return alias < other.alias; } bool Production::operator==(const Production &other) const { diff --git a/src/compiler/syntax_grammar.h b/src/compiler/syntax_grammar.h index 4fbd082d..7714e475 100644 --- a/src/compiler/syntax_grammar.h +++ b/src/compiler/syntax_grammar.h @@ -13,7 +13,7 @@ struct ProductionStep { rules::Symbol symbol; int precedence; rules::Associativity associativity; - std::string name_replacement; + rules::Alias alias; bool operator==(const ProductionStep &) const; bool operator!=(const ProductionStep &) const; diff --git a/src/runtime/language.c b/src/runtime/language.c index 22cbdf94..8f6c37ac 100644 --- a/src/runtime/language.c +++ b/src/runtime/language.c @@ -32,7 +32,7 @@ void ts_language_table_entry(const TSLanguage *self, TSStateId state, } uint32_t ts_language_symbol_count(const TSLanguage *language) { - return language->symbol_count + language->rename_symbol_count; + return language->symbol_count + language->alias_count; } uint32_t ts_language_version(const TSLanguage *language) { diff --git a/src/runtime/language.h b/src/runtime/language.h index b1975fc7..11d7aeeb 100644 --- a/src/runtime/language.h +++ b/src/runtime/language.h @@ -64,9 +64,9 @@ ts_language_enabled_external_tokens(const TSLanguage *self, } static inline const TSSymbol * -ts_language_rename_sequence(const TSLanguage *self, unsigned id) { +ts_language_alias_sequence(const TSLanguage *self, unsigned id) { return id > 0 ? - self->rename_sequences + id * self->max_rename_sequence_length : + self->alias_sequences + id * self->max_alias_sequence_length : NULL; } diff --git a/src/runtime/node.c b/src/runtime/node.c index 8cf3b662..7ddeee35 100644 --- a/src/runtime/node.c +++ b/src/runtime/node.c @@ -33,7 +33,7 @@ static inline uint32_t ts_node__offset_row(TSNode self) { static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { const Tree *tree = ts_node__tree(self); - if (tree->context.rename_symbol > 0) { + if (tree->context.alias_symbol > 0) { return true; } else { return include_anonymous ? tree->visible : tree->visible && tree->named; @@ -269,7 +269,7 @@ TSPoint ts_node_end_point(TSNode self) { TSSymbol ts_node_symbol(TSNode self) { const Tree *tree = ts_node__tree(self); - return tree->context.rename_symbol ? tree->context.rename_symbol : tree->symbol; + return tree->context.alias_symbol ? tree->context.alias_symbol : tree->symbol; } TSSymbolIterator ts_node_symbols(TSNode self) { @@ -308,7 +308,7 @@ bool ts_node_eq(TSNode self, TSNode other) { bool ts_node_is_named(TSNode self) { const Tree *tree = ts_node__tree(self); - return tree->named || tree->context.rename_symbol != 0; + return tree->context.alias_symbol ? tree->context.alias_is_named : tree->named; } bool ts_node_has_changes(TSNode self) { diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 597ea739..7d0d052c 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -338,7 +338,7 @@ static Tree *parser__lex(Parser *self, StackVersion version) { if (skipped_error) { Length padding = length_sub(error_start_position, start_position); Length size = length_sub(error_end_position, error_start_position); - result = ts_tree_make_error(size, padding, first_error_character); + result = ts_tree_make_error(size, padding, first_error_character, self->language); } else { TSSymbol symbol = self->lexer.data.result_symbol; if (found_external_token) { @@ -347,8 +347,7 @@ static Tree *parser__lex(Parser *self, StackVersion version) { Length padding = length_sub(self->lexer.token_start_position, start_position); Length size = length_sub(self->lexer.token_end_position, self->lexer.token_start_position); - TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, symbol); - result = ts_tree_make_leaf(symbol, padding, size, metadata); + result = ts_tree_make_leaf(symbol, padding, size, self->language); if (found_external_token) { result->has_external_tokens = true; @@ -544,12 +543,7 @@ static void parser__shift(Parser *self, StackVersion version, TSStateId state, static bool parser__replace_children(Parser *self, Tree *tree, Tree **children, uint32_t count) { self->scratch_tree.symbol = tree->symbol; self->scratch_tree.child_count = 0; - ts_tree_set_children( - &self->scratch_tree, - count, - children, - ts_language_rename_sequence(self->language, tree->rename_sequence_id) - ); + ts_tree_set_children(&self->scratch_tree, count, children, self->language); if (parser__select_tree(self, tree, &self->scratch_tree)) { tree->size = self->scratch_tree.size; tree->padding = self->scratch_tree.padding; @@ -566,16 +560,13 @@ static bool parser__replace_children(Parser *self, Tree *tree, Tree **children, static StackPopResult parser__reduce(Parser *self, StackVersion version, TSSymbol symbol, uint32_t count, - int dynamic_precedence, uint16_t rename_sequence_id, + int dynamic_precedence, uint16_t alias_sequence_id, bool fragile, bool allow_skipping) { uint32_t initial_version_count = ts_stack_version_count(self->stack); StackPopResult pop = ts_stack_pop_count(self->stack, version, count); if (pop.stopped_at_error) return pop; - TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, symbol); - const TSSymbol *rename_sequence = ts_language_rename_sequence(self->language, rename_sequence_id); - for (uint32_t i = 0; i < pop.slices.size; i++) { StackSlice slice = pop.slices.contents[i]; @@ -588,7 +579,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version, } Tree *parent = ts_tree_make_node( - symbol, child_count, slice.trees.contents, metadata, rename_sequence + symbol, child_count, slice.trees.contents, alias_sequence_id, self->language ); // This pop operation may have caused multiple stack versions to collapse @@ -614,7 +605,7 @@ static StackPopResult parser__reduce(Parser *self, StackVersion version, } parent->dynamic_precedence += dynamic_precedence; - parent->rename_sequence_id = rename_sequence_id; + parent->alias_sequence_id = alias_sequence_id; TSStateId state = ts_stack_top_state(self->stack, slice.version); TSStateId next_state = ts_language_next_state(self->language, state, symbol); @@ -840,7 +831,7 @@ static bool parser__repair_error(Parser *self, StackSlice slice, TreeArray skipped_children = ts_tree_array_remove_last_n(&children, skip_count); TreeArray trailing_extras = ts_tree_array_remove_trailing_extras(&skipped_children); - Tree *error = ts_tree_make_error_node(&skipped_children); + Tree *error = ts_tree_make_error_node(&skipped_children, self->language); array_push(&children, error); array_push_all(&children, &trailing_extras); trailing_extras.size = 0; @@ -852,8 +843,7 @@ static bool parser__repair_error(Parser *self, StackSlice slice, Tree *parent = ts_tree_make_node( symbol, children.size, children.contents, - ts_language_symbol_metadata(self->language, symbol), - NULL + 0, self->language ); parser__push(self, slice.version, parent, next_state); ts_stack_decrease_push_count(self->stack, slice.version, error->child_count); @@ -912,11 +902,7 @@ static void parser__accept(Parser *self, StackVersion version, for (uint32_t k = 0; k < child->child_count; k++) ts_tree_retain(child->children[k]); array_splice(&trees, j, 1, child->child_count, child->children); - const TSSymbol *rename_sequence = ts_language_rename_sequence( - self->language, - root->rename_sequence_id - ); - ts_tree_set_children(root, trees.size, trees.contents, rename_sequence); + ts_tree_set_children(root, trees.size, trees.contents, self->language); ts_tree_release(child); break; } @@ -965,7 +951,7 @@ static bool parser__do_potential_reductions(Parser *self, StackVersion version) .symbol = action.params.symbol, .count = action.params.child_count, .dynamic_precedence = action.params.dynamic_precedence, - .rename_sequence_id = action.params.rename_sequence_id, + .alias_sequence_id = action.params.alias_sequence_id, }); default: break; @@ -978,7 +964,7 @@ static bool parser__do_potential_reductions(Parser *self, StackVersion version) ReduceAction action = self->reduce_actions.contents[i]; StackPopResult reduction = parser__reduce( self, version, action.symbol, action.count, - action.dynamic_precedence, action.rename_sequence_id, + action.dynamic_precedence, action.alias_sequence_id, true, false ); if (reduction.stopped_at_error) { @@ -1039,7 +1025,7 @@ static bool parser__skip_preceding_trees(Parser *self, StackVersion version, } previous_version = slice.version; - Tree *error = ts_tree_make_error_node(&slice.trees); + Tree *error = ts_tree_make_error_node(&slice.trees, self->language); error->extra = true; TSStateId state = ts_stack_top_state(self->stack, slice.version); parser__push(self, slice.version, error, state); @@ -1104,16 +1090,15 @@ static void parser__halt_parse(Parser *self) { ts_stack_top_position(self->stack, 0) ); - Tree *filler_node = ts_tree_make_error(remaining_length, length_zero(), 0); + Tree *filler_node = ts_tree_make_error(remaining_length, length_zero(), 0, self->language); filler_node->visible = false; parser__push(self, 0, filler_node, 0); TreeArray children = array_new(); - Tree *root_error = ts_tree_make_error_node(&children); + Tree *root_error = ts_tree_make_error_node(&children, self->language); parser__push(self, 0, root_error, 0); - TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, ts_builtin_sym_end); - Tree *eof = ts_tree_make_leaf(ts_builtin_sym_end, length_zero(), length_zero(), metadata); + Tree *eof = ts_tree_make_leaf(ts_builtin_sym_end, length_zero(), length_zero(), self->language); parser__accept(self, 0, eof); ts_tree_release(eof); } @@ -1123,7 +1108,7 @@ static void parser__recover(Parser *self, StackVersion version, TSStateId state, if (lookahead->symbol == ts_builtin_sym_end) { LOG("recover_eof"); TreeArray children = array_new(); - Tree *parent = ts_tree_make_error_node(&children); + Tree *parent = ts_tree_make_error_node(&children, self->language); parser__push(self, version, parent, 1); parser__accept(self, version, lookahead); return; @@ -1218,7 +1203,7 @@ static void parser__advance(Parser *self, StackVersion version, LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.params.symbol), action.params.child_count); StackPopResult reduction = parser__reduce( self, version, action.params.symbol, action.params.child_count, - action.params.dynamic_precedence, action.params.rename_sequence_id, + action.params.dynamic_precedence, action.params.alias_sequence_id, action.params.fragile, true ); StackSlice slice = *array_front(&reduction.slices); diff --git a/src/runtime/reduce_action.h b/src/runtime/reduce_action.h index 5ae876f2..75267c3f 100644 --- a/src/runtime/reduce_action.h +++ b/src/runtime/reduce_action.h @@ -12,7 +12,7 @@ typedef struct { uint32_t count; TSSymbol symbol; int dynamic_precedence; - unsigned short rename_sequence_id; + unsigned short alias_sequence_id; } ReduceAction; typedef Array(ReduceAction) ReduceActionSet; diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 42214e23..630ee0b1 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -42,23 +42,23 @@ bool ts_external_token_state_eq(const TSExternalTokenState *a, const TSExternalT memcmp(ts_external_token_state_data(a), ts_external_token_state_data(b), a->length) == 0); } -Tree *ts_tree_make_leaf(TSSymbol sym, Length padding, Length size, - TSSymbolMetadata metadata) { +Tree *ts_tree_make_leaf(TSSymbol symbol, Length padding, Length size, const TSLanguage *language) { + TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); Tree *result = ts_malloc(sizeof(Tree)); *result = (Tree){ .ref_count = 1, - .symbol = sym, + .symbol = symbol, .size = size, .child_count = 0, .visible_child_count = 0, .named_child_count = 0, .children = NULL, - .rename_sequence_id = 0, + .alias_sequence_id = 0, .padding = padding, .visible = metadata.visible, .named = metadata.named, .has_changes = false, - .first_leaf.symbol = sym, + .first_leaf.symbol = symbol, .has_external_tokens = false, }; return result; @@ -134,11 +134,9 @@ TreeArray ts_tree_array_remove_trailing_extras(TreeArray *self) { return result; } -Tree *ts_tree_make_error(Length size, Length padding, int32_t lookahead_char) { - Tree *result = ts_tree_make_leaf(ts_builtin_sym_error, padding, size, - (TSSymbolMetadata){ - .visible = true, .named = true, - }); +Tree *ts_tree_make_error(Length size, Length padding, int32_t lookahead_char, + const TSLanguage *language) { + Tree *result = ts_tree_make_leaf(ts_builtin_sym_error, padding, size, language); result->fragile_left = true; result->fragile_right = true; result->lookahead_char = lookahead_char; @@ -158,7 +156,7 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua while (path->size > 0) { Tree *tree = array_pop(path).tree; Length offset = length_zero(); - const TSSymbol *rename_sequence = ts_language_rename_sequence(language, tree->rename_sequence_id); + const TSSymbol *alias_sequence = ts_language_alias_sequence(language, tree->alias_sequence_id); uint32_t non_extra_index = 0; for (uint32_t i = 0; i < tree->child_count; i++) { Tree *child = tree->children[i]; @@ -166,10 +164,12 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua child->context.parent = tree; child->context.index = i; child->context.offset = offset; - if (!child->extra && rename_sequence && rename_sequence[non_extra_index] != 0) { - child->context.rename_symbol = rename_sequence[non_extra_index]; + if (!child->extra && alias_sequence && alias_sequence[non_extra_index] != 0) { + TSSymbolMetadata metadata = ts_language_symbol_metadata(language, alias_sequence[non_extra_index]); + child->context.alias_symbol = alias_sequence[non_extra_index]; + child->context.alias_is_named = metadata.named; } else { - child->context.rename_symbol = 0; + child->context.alias_symbol = 0; } array_push(path, ((TreePathEntry){child, length_zero(), 0})); } @@ -181,9 +181,8 @@ void ts_tree_assign_parents(Tree *self, TreePath *path, const TSLanguage *langua void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, - const TSSymbol *rename_sequence) { - if (self->child_count > 0) - ts_free(self->children); + const TSLanguage *language) { + if (self->child_count > 0) ts_free(self->children); self->children = children; self->child_count = child_count; @@ -194,6 +193,7 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, self->dynamic_precedence = 0; uint32_t non_extra_index = 0; + const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self->alias_sequence_id); for (uint32_t i = 0; i < child_count; i++) { Tree *child = children[i]; @@ -211,9 +211,11 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, self->error_cost += child->error_cost; self->dynamic_precedence += child->dynamic_precedence; - if (rename_sequence && rename_sequence[non_extra_index] != 0 && !child->extra) { + if (alias_sequence && alias_sequence[non_extra_index] != 0 && !child->extra) { self->visible_child_count++; - self->named_child_count++; + if (ts_language_symbol_metadata(language, alias_sequence[non_extra_index]).named) { + self->named_child_count++; + } } else if (child->visible) { self->visible_child_count++; if (child->named) self->named_child_count++; @@ -250,14 +252,14 @@ void ts_tree_set_children(Tree *self, uint32_t child_count, Tree **children, } Tree *ts_tree_make_node(TSSymbol symbol, uint32_t child_count, Tree **children, - TSSymbolMetadata metadata, const TSSymbol *rename_sequence) { - Tree *result = - ts_tree_make_leaf(symbol, length_zero(), length_zero(), metadata); - ts_tree_set_children(result, child_count, children, rename_sequence); + unsigned alias_sequence_id, const TSLanguage *language) { + Tree *result = ts_tree_make_leaf(symbol, length_zero(), length_zero(), language); + result->alias_sequence_id = alias_sequence_id; + ts_tree_set_children(result, child_count, children, language); return result; } -Tree *ts_tree_make_error_node(TreeArray *children) { +Tree *ts_tree_make_error_node(TreeArray *children, const TSLanguage *language) { for (uint32_t i = 0; i < children->size; i++) { Tree *child = children->contents[i]; if (child->symbol == ts_builtin_sym_error && child->child_count > 0) { @@ -269,9 +271,8 @@ Tree *ts_tree_make_error_node(TreeArray *children) { } } - Tree *result = ts_tree_make_node( - ts_builtin_sym_error, children->size, children->contents, - (TSSymbolMetadata){.extra = false, .visible = true, .named = true }, NULL); + Tree *result = + ts_tree_make_node(ts_builtin_sym_error, children->size, children->contents, 0, language); result->fragile_left = true; result->fragile_right = true; @@ -328,29 +329,26 @@ uint32_t ts_tree_end_column(const Tree *self) { bool ts_tree_eq(const Tree *self, const Tree *other) { if (self) { - if (!other) - return false; + if (!other) return false; } else { return !other; } - if (self->symbol != other->symbol) - return false; - if (self->visible != other->visible) - return false; - if (self->named != other->named) - return false; - if (self->symbol == ts_builtin_sym_error) - return self->lookahead_char == other->lookahead_char; - if (self->child_count != other->child_count) - return false; - if (self->visible_child_count != other->visible_child_count) - return false; - if (self->named_child_count != other->named_child_count) - return false; - for (uint32_t i = 0; i < self->child_count; i++) - if (!ts_tree_eq(self->children[i], other->children[i])) + if (self->symbol != other->symbol) return false; + if (self->visible != other->visible) return false; + if (self->named != other->named) return false; + if (self->padding.bytes != other->padding.bytes) return false; + if (self->size.bytes != other->size.bytes) return false; + if (self->symbol == ts_builtin_sym_error) return self->lookahead_char == other->lookahead_char; + if (self->child_count != other->child_count) return false; + if (self->visible_child_count != other->visible_child_count) return false; + if (self->named_child_count != other->named_child_count) return false; + + for (uint32_t i = 0; i < self->child_count; i++) { + if (!ts_tree_eq(self->children[i], other->children[i])) { return false; + } + } return true; } @@ -528,7 +526,7 @@ static size_t ts_tree__write_to_string(const Tree *self, include_all || is_root || (self->visible && self->named) || - self->context.rename_symbol != 0; + self->context.alias_is_named; if (visible && !is_root) { cursor += snprintf(*writer, limit, " "); @@ -539,7 +537,7 @@ static size_t ts_tree__write_to_string(const Tree *self, cursor += snprintf(*writer, limit, "(UNEXPECTED "); cursor += ts_tree__write_char_to_string(*writer, limit, self->lookahead_char); } else { - TSSymbol symbol = self->context.rename_symbol ? self->context.rename_symbol : self->symbol; + TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol; cursor += snprintf(*writer, limit, "(%s", ts_language_symbol_name(language, symbol)); } } @@ -566,7 +564,7 @@ char *ts_tree_string(const Tree *self, const TSLanguage *language, void ts_tree__print_dot_graph(const Tree *self, uint32_t byte_offset, const TSLanguage *language, FILE *f) { - TSSymbol symbol = self->context.rename_symbol ? self->context.rename_symbol : self->symbol; + TSSymbol symbol = self->context.alias_symbol ? self->context.alias_symbol : self->symbol; fprintf(f, "tree_%p [label=\"%s\"", self, ts_language_symbol_name(language, symbol)); if (self->child_count == 0) diff --git a/src/runtime/tree.h b/src/runtime/tree.h index 63ff9e9f..eb3a5e80 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -30,7 +30,8 @@ typedef struct Tree { struct Tree *parent; uint32_t index; Length offset; - TSSymbol rename_symbol; + TSSymbol alias_symbol : 15; + bool alias_is_named : 1; } context; uint32_t child_count; @@ -39,7 +40,7 @@ typedef struct Tree { struct Tree **children; uint32_t visible_child_count; uint32_t named_child_count; - unsigned short rename_sequence_id; + unsigned short alias_sequence_id; }; TSExternalTokenState external_token_state; int32_t lookahead_char; @@ -85,11 +86,11 @@ uint32_t ts_tree_array_essential_count(const TreeArray *); TreeArray ts_tree_array_remove_last_n(TreeArray *, uint32_t); TreeArray ts_tree_array_remove_trailing_extras(TreeArray *); -Tree *ts_tree_make_leaf(TSSymbol, Length, Length, TSSymbolMetadata); -Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, TSSymbolMetadata, const TSSymbol *); +Tree *ts_tree_make_leaf(TSSymbol, Length, Length, const TSLanguage *); +Tree *ts_tree_make_node(TSSymbol, uint32_t, Tree **, unsigned, const TSLanguage *); Tree *ts_tree_make_copy(Tree *child); -Tree *ts_tree_make_error_node(TreeArray *); -Tree *ts_tree_make_error(Length, Length, int32_t); +Tree *ts_tree_make_error_node(TreeArray *, const TSLanguage *); +Tree *ts_tree_make_error(Length, Length, int32_t, const TSLanguage *); void ts_tree_retain(Tree *tree); void ts_tree_release(Tree *tree); bool ts_tree_eq(const Tree *tree1, const Tree *tree2); @@ -97,7 +98,7 @@ int ts_tree_compare(const Tree *tree1, const Tree *tree2); uint32_t ts_tree_start_column(const Tree *self); uint32_t ts_tree_end_column(const Tree *self); -void ts_tree_set_children(Tree *, uint32_t, Tree **, const TSSymbol *); +void ts_tree_set_children(Tree *, uint32_t, Tree **, const TSLanguage *); void ts_tree_assign_parents(Tree *, TreePath *, const TSLanguage *); void ts_tree_edit(Tree *, const TSInputEdit *edit); char *ts_tree_string(const Tree *, const TSLanguage *, bool include_all); diff --git a/src/runtime/tree_path.h b/src/runtime/tree_path.h index c8b63f7c..efd61af5 100644 --- a/src/runtime/tree_path.h +++ b/src/runtime/tree_path.h @@ -133,19 +133,19 @@ TreePathComparison tree_path_compare(const TreePath *old_path, const TreePath *new_path, const TSLanguage *language) { Tree *old_tree = NULL; - TSSymbol old_rename_symbol = 0; + TSSymbol old_alias_symbol = 0; Length old_start = length_zero(); for (uint32_t i = old_path->size - 1; i + 1 > 0; i--) { old_tree = old_path->contents[i].tree; if (old_tree->visible) { old_start = old_path->contents[i].position; if (i > 0) { - const TSSymbol *rename_sequence = ts_language_rename_sequence( + const TSSymbol *alias_sequence = ts_language_alias_sequence( language, - old_path->contents[i - 1].tree->rename_sequence_id + old_path->contents[i - 1].tree->alias_sequence_id ); - if (rename_sequence) { - old_rename_symbol = rename_sequence[old_path->contents[i].child_index]; + if (alias_sequence) { + old_alias_symbol = alias_sequence[old_path->contents[i].child_index]; } } break; @@ -153,26 +153,26 @@ TreePathComparison tree_path_compare(const TreePath *old_path, } Tree *new_tree = NULL; - TSSymbol new_rename_symbol = 0; + TSSymbol new_alias_symbol = 0; Length new_start = length_zero(); for (uint32_t i = new_path->size - 1; i + 1 > 0; i--) { new_tree = new_path->contents[i].tree; if (new_tree->visible) { new_start = old_path->contents[i].position; if (i > 0) { - const TSSymbol *rename_sequence = ts_language_rename_sequence( + const TSSymbol *alias_sequence = ts_language_alias_sequence( language, - new_path->contents[i - 1].tree->rename_sequence_id + new_path->contents[i - 1].tree->alias_sequence_id ); - if (rename_sequence) { - new_rename_symbol = rename_sequence[new_path->contents[i].child_index]; + if (alias_sequence) { + new_alias_symbol = alias_sequence[new_path->contents[i].child_index]; } } break; } } - if (old_rename_symbol == new_rename_symbol) { + if (old_alias_symbol == new_alias_symbol) { if (old_start.bytes == new_start.bytes) { if (!old_tree->has_changes && old_tree->symbol == new_tree->symbol && diff --git a/test/compiler/build_tables/parse_item_set_builder_test.cc b/test/compiler/build_tables/parse_item_set_builder_test.cc index efa1add4..caceaad0 100644 --- a/test/compiler/build_tables/parse_item_set_builder_test.cc +++ b/test/compiler/build_tables/parse_item_set_builder_test.cc @@ -28,23 +28,23 @@ describe("ParseItemSetBuilder", []() { SyntaxGrammar grammar{{ SyntaxVariable{"rule0", VariableTypeNamed, { Production{{ - {Symbol::non_terminal(1), 0, AssociativityNone, ""}, - {Symbol::terminal(11), 0, AssociativityNone, ""}, + {Symbol::non_terminal(1), 0, AssociativityNone, Alias{}}, + {Symbol::terminal(11), 0, AssociativityNone, Alias{}}, }, 0}, }}, SyntaxVariable{"rule1", VariableTypeNamed, { Production{{ - {Symbol::terminal(12), 0, AssociativityNone, ""}, - {Symbol::terminal(13), 0, AssociativityNone, ""}, + {Symbol::terminal(12), 0, AssociativityNone, Alias{}}, + {Symbol::terminal(13), 0, AssociativityNone, Alias{}}, }, 0}, Production{{ - {Symbol::non_terminal(2), 0, AssociativityNone, ""}, + {Symbol::non_terminal(2), 0, AssociativityNone, Alias{}}, }, 0} }}, SyntaxVariable{"rule2", VariableTypeNamed, { Production{{ - {Symbol::terminal(14), 0, AssociativityNone, ""}, - {Symbol::terminal(15), 0, AssociativityNone, ""}, + {Symbol::terminal(14), 0, AssociativityNone, Alias{}}, + {Symbol::terminal(15), 0, AssociativityNone, Alias{}}, }, 0} }}, }, {}, {}, {}, {}}; @@ -87,14 +87,14 @@ describe("ParseItemSetBuilder", []() { SyntaxGrammar grammar{{ SyntaxVariable{"rule0", VariableTypeNamed, { Production{{ - {Symbol::non_terminal(1), 0, AssociativityNone, ""}, - {Symbol::terminal(11), 0, AssociativityNone, ""}, + {Symbol::non_terminal(1), 0, AssociativityNone, Alias{}}, + {Symbol::terminal(11), 0, AssociativityNone, Alias{}}, }, 0}, }}, SyntaxVariable{"rule1", VariableTypeNamed, { Production{{ - {Symbol::terminal(12), 0, AssociativityNone, ""}, - {Symbol::terminal(13), 0, AssociativityNone, ""}, + {Symbol::terminal(12), 0, AssociativityNone, Alias{}}, + {Symbol::terminal(13), 0, AssociativityNone, Alias{}}, }, 0}, Production{{}, 0} }}, diff --git a/test/fixtures/test_grammars/inlined_renamed_rules/grammar.json b/test/fixtures/test_grammars/inlined_renamed_rules/grammar.json index 1e399941..d29f65d7 100644 --- a/test/fixtures/test_grammars/inlined_renamed_rules/grammar.json +++ b/test/fixtures/test_grammars/inlined_renamed_rules/grammar.json @@ -24,8 +24,9 @@ {"type": "SYMBOL", "name": "call_expression"}, {"type": "SYMBOL", "name": "member_expression"}, { - "type": "RENAME", + "type": "ALIAS", "value": "variable_name", + "named": true, "content": { "type": "SYMBOL", "name": "identifier" @@ -57,7 +58,8 @@ {"type": "SYMBOL", "name": "expression"}, {"type": "STRING", "value": "."}, { - "type": "RENAME", + "type": "ALIAS", + "named": true, "value": "property_name", "content": { "type": "SYMBOL", diff --git a/test/fixtures/test_grammars/renamed_inlined_rules/grammar.json b/test/fixtures/test_grammars/renamed_inlined_rules/grammar.json index 6dfb4ebf..4e68cd88 100644 --- a/test/fixtures/test_grammars/renamed_inlined_rules/grammar.json +++ b/test/fixtures/test_grammars/renamed_inlined_rules/grammar.json @@ -15,8 +15,9 @@ "members": [ {"type": "SYMBOL", "name": "member_expression"}, { - "type": "RENAME", + "type": "ALIAS", "value": "variable_name", + "named": true, "content": { "type": "SYMBOL", "name": "identifier" @@ -34,8 +35,9 @@ {"type": "SYMBOL", "name": "expression"}, {"type": "STRING", "value": "."}, { - "type": "RENAME", + "type": "ALIAS", "value": "property_name", + "named": true, "content": { "type": "SYMBOL", "name": "identifier" diff --git a/test/fixtures/test_grammars/renamed_rules/grammar.json b/test/fixtures/test_grammars/renamed_rules/grammar.json index c6e86052..1d49df92 100644 --- a/test/fixtures/test_grammars/renamed_rules/grammar.json +++ b/test/fixtures/test_grammars/renamed_rules/grammar.json @@ -20,7 +20,8 @@ {"type": "SYMBOL", "name": "call_expression"}, {"type": "SYMBOL", "name": "member_expression"}, { - "type": "RENAME", + "type": "ALIAS", + "named": true, "value": "variable_name", "content": { "type": "SYMBOL", @@ -53,7 +54,8 @@ {"type": "SYMBOL", "name": "_expression"}, {"type": "STRING", "value": "."}, { - "type": "RENAME", + "type": "ALIAS", + "named": true, "value": "property_name", "content": { "type": "SYMBOL", diff --git a/test/runtime/stack_test.cc b/test/runtime/stack_test.cc index e2617aff..f7d2a9fd 100644 --- a/test/runtime/stack_test.cc +++ b/test/runtime/stack_test.cc @@ -78,10 +78,11 @@ describe("Stack", [&]() { stack = ts_stack_new(); - for (size_t i = 0; i < tree_count; i++) - trees[i] = ts_tree_make_leaf(i, length_zero(), tree_len, { - true, true, false, true, - }); + TSLanguage dummy_language; + + for (size_t i = 0; i < tree_count; i++) { + trees[i] = ts_tree_make_leaf(i, length_zero(), tree_len, &dummy_language); + } }); after_each([&]() { diff --git a/test/runtime/tree_test.cc b/test/runtime/tree_test.cc index 40286ac7..fad9b9c1 100644 --- a/test/runtime/tree_test.cc +++ b/test/runtime/tree_test.cc @@ -35,12 +35,14 @@ describe("Tree", []() { symbol9, }; - TSSymbolMetadata visible = {true, true, false, true}; - TSSymbolMetadata invisible = {false, false, false, true}; + TSSymbolMetadata metadata_list[30] = {}; + + TSLanguage language; + language.symbol_metadata = metadata_list; describe("make_leaf", [&]() { it("does not mark the tree as fragile", [&]() { - Tree *tree = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, visible); + Tree *tree = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, &language); AssertThat(tree->fragile_left, IsFalse()); AssertThat(tree->fragile_right, IsFalse()); }); @@ -51,7 +53,9 @@ describe("Tree", []() { Tree *error_tree = ts_tree_make_error( length_zero(), length_zero(), - 'z'); + 'z', + &language + ); AssertThat(error_tree->fragile_left, IsTrue()); AssertThat(error_tree->fragile_right, IsTrue()); @@ -64,15 +68,15 @@ describe("Tree", []() { Tree *tree1, *tree2, *parent1; before_each([&]() { - tree1 = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, visible); - tree2 = ts_tree_make_leaf(symbol2, {1, 1, {0, 1}}, {3, 3, {0, 3}}, visible); + tree1 = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, &language); + tree2 = ts_tree_make_leaf(symbol2, {1, 1, {0, 1}}, {3, 3, {0, 3}}, &language); ts_tree_retain(tree1); ts_tree_retain(tree2); parent1 = ts_tree_make_node(symbol3, 2, tree_array({ tree1, tree2, - }), visible, nullptr); + }), 0, &language); }); after_each([&]() { @@ -103,7 +107,7 @@ describe("Tree", []() { parent = ts_tree_make_node(symbol3, 2, tree_array({ tree1, tree2, - }), visible, nullptr); + }), 0, &language); }); after_each([&]() { @@ -127,7 +131,7 @@ describe("Tree", []() { parent = ts_tree_make_node(symbol3, 2, tree_array({ tree1, tree2, - }), visible, nullptr); + }), 0, &language); }); after_each([&]() { @@ -151,7 +155,7 @@ describe("Tree", []() { parent = ts_tree_make_node(symbol3, 2, tree_array({ tree1, tree2, - }), visible, nullptr); + }), 0, &language); }); after_each([&]() { @@ -170,10 +174,10 @@ describe("Tree", []() { before_each([&]() { tree = ts_tree_make_node(symbol1, 3, tree_array({ - ts_tree_make_leaf(symbol2, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible), - ts_tree_make_leaf(symbol3, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible), - ts_tree_make_leaf(symbol4, {2, 2, {0, 2}}, {3, 3, {0, 3}}, visible), - }), visible, nullptr); + ts_tree_make_leaf(symbol2, {2, 2, {0, 2}}, {3, 3, {0, 3}}, &language), + ts_tree_make_leaf(symbol3, {2, 2, {0, 2}}, {3, 3, {0, 3}}, &language), + ts_tree_make_leaf(symbol4, {2, 2, {0, 2}}, {3, 3, {0, 3}}, &language), + }), 0, &language); AssertThat(tree->padding, Equals({2, 2, {0, 2}})); AssertThat(tree->size, Equals({13, 13, {0, 13}})); @@ -336,7 +340,7 @@ describe("Tree", []() { Tree *leaf; before_each([&]() { - leaf = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, visible); + leaf = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, {5, 4, {0, 4}}, &language); }); after_each([&]() { @@ -344,20 +348,20 @@ describe("Tree", []() { }); it("returns true for identical trees", [&]() { - Tree *leaf_copy = ts_tree_make_leaf(symbol1, {2, 1, {1, 1}}, {5, 4, {1, 4}}, visible); + Tree *leaf_copy = ts_tree_make_leaf(symbol1, {2, 1, {1, 1}}, {5, 4, {1, 4}}, &language); AssertThat(ts_tree_eq(leaf, leaf_copy), IsTrue()); Tree *parent = ts_tree_make_node(symbol2, 2, tree_array({ leaf, leaf_copy, - }), visible, nullptr); + }), 0, &language); ts_tree_retain(leaf); ts_tree_retain(leaf_copy); Tree *parent_copy = ts_tree_make_node(symbol2, 2, tree_array({ leaf, leaf_copy, - }), visible, nullptr); + }), 0, &language); ts_tree_retain(leaf); ts_tree_retain(leaf_copy); @@ -373,42 +377,44 @@ describe("Tree", []() { leaf->symbol + 1, leaf->padding, leaf->size, - visible); + &language + ); AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse()); ts_tree_release(different_leaf); }); it("returns false for trees with different options", [&]() { - Tree *different_leaf = ts_tree_make_leaf(symbol1, leaf->padding, leaf->size, invisible); + Tree *different_leaf = ts_tree_make_leaf(leaf->symbol, leaf->padding, leaf->size, &language); + different_leaf->visible = !leaf->visible; AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse()); ts_tree_release(different_leaf); }); - it("returns false for trees with different sizes", [&]() { - Tree *different_leaf = ts_tree_make_leaf(symbol1, {2, 1, {0, 1}}, leaf->size, invisible); + it("returns false for trees with different paddings or sizes", [&]() { + Tree *different_leaf = ts_tree_make_leaf(leaf->symbol, {}, leaf->size, &language); AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse()); ts_tree_release(different_leaf); - different_leaf = ts_tree_make_leaf(symbol1, leaf->padding, {5, 4, {1, 10}}, invisible); + different_leaf = ts_tree_make_leaf(symbol1, leaf->padding, {}, &language); AssertThat(ts_tree_eq(leaf, different_leaf), IsFalse()); ts_tree_release(different_leaf); }); it("returns false for trees with different children", [&]() { - Tree *leaf2 = ts_tree_make_leaf(symbol2, {1, 1, {0, 1}}, {3, 3, {0, 3}}, visible); + Tree *leaf2 = ts_tree_make_leaf(symbol2, {1, 1, {0, 1}}, {3, 3, {0, 3}}, &language); Tree *parent = ts_tree_make_node(symbol2, 2, tree_array({ leaf, leaf2, - }), visible, nullptr); + }), 0, &language); ts_tree_retain(leaf); ts_tree_retain(leaf2); Tree *different_parent = ts_tree_make_node(symbol2, 2, tree_array({ leaf2, leaf, - }), visible, nullptr); + }), 0, &language); ts_tree_retain(leaf2); ts_tree_retain(leaf); @@ -435,17 +441,17 @@ describe("Tree", []() { tree1 = ts_tree_make_node(symbol1, 2, tree_array({ (tree2 = ts_tree_make_node(symbol2, 3, tree_array({ - (tree3 = make_external(ts_tree_make_leaf(symbol3, padding, size, visible))), - (tree4 = ts_tree_make_leaf(symbol4, padding, size, visible)), - (tree5 = ts_tree_make_leaf(symbol5, padding, size, visible)), - }), visible, nullptr)), + (tree3 = make_external(ts_tree_make_leaf(symbol3, padding, size, &language))), + (tree4 = ts_tree_make_leaf(symbol4, padding, size, &language)), + (tree5 = ts_tree_make_leaf(symbol5, padding, size, &language)), + }), 0, &language)), (tree6 = ts_tree_make_node(symbol6, 2, tree_array({ (tree7 = ts_tree_make_node(symbol7, 1, tree_array({ - (tree8 = ts_tree_make_leaf(symbol8, padding, size, visible)), - }), visible, nullptr)), - (tree9 = ts_tree_make_leaf(symbol9, padding, size, visible)), - }), visible, nullptr)), - }), visible, nullptr); + (tree8 = ts_tree_make_leaf(symbol8, padding, size, &language)), + }), 0, &language)), + (tree9 = ts_tree_make_leaf(symbol9, padding, size, &language)), + }), 0, &language)), + }), 0, &language); auto token = ts_tree_last_external_token(tree1); AssertThat(token, Equals(tree3));