From 1c6ad5f7e4f08ae6c1b940223e7ec9a48a51dc20 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 17 Dec 2015 15:50:48 -0800 Subject: [PATCH] Rename ubiquitous_tokens -> extra_tokens in compiler API They were already called this in the runtime code. 'Extra' is just easier to say. --- README.md | 4 ++-- include/tree_sitter/compiler.h | 6 +++--- .../prepare_grammar/extract_tokens_spec.cc | 18 +++++++++--------- .../prepare_grammar/intern_symbols_spec.cc | 8 ++++---- spec/fixtures/grammars/anonymous_tokens.cc | 2 +- spec/fixtures/grammars/arithmetic.cc | 2 +- spec/fixtures/grammars/c.cc | 2 +- spec/fixtures/grammars/cpp.cc | 2 +- spec/fixtures/grammars/golang.cc | 2 +- spec/fixtures/grammars/javascript.cc | 2 +- spec/fixtures/grammars/json.cc | 2 +- spec/runtime/parser_spec.cc | 8 ++++---- src/compiler/build_tables/build_parse_table.cc | 10 +++++----- src/compiler/generate_code/c_code.cc | 2 +- src/compiler/grammar.cc | 10 +++++----- src/compiler/prepare_grammar/expand_repeats.cc | 2 +- src/compiler/prepare_grammar/extract_tokens.cc | 8 ++++---- .../prepare_grammar/flatten_grammar.cc | 2 +- .../prepare_grammar/initial_syntax_grammar.h | 2 +- src/compiler/prepare_grammar/intern_symbols.cc | 4 ++-- .../prepare_grammar/interned_grammar.h | 2 +- src/compiler/syntax_grammar.h | 2 +- 22 files changed, 51 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index ed63e8fb..96b26a09 100644 --- a/README.md +++ b/README.md @@ -74,10 +74,10 @@ int main() { { "variable", pattern("[a-zA-Z]+\\w*") }, { "comment", pattern("//.*") }, - }).ubiquitous_tokens({ + }).extra_tokens({ // Things that can appear anywhere in the language are expressed as - // 'ubiquitous tokens'. + // 'extra tokens'. sym("comment"), pattern("\\s+") }); diff --git a/include/tree_sitter/compiler.h b/include/tree_sitter/compiler.h index 333b3249..fbc18010 100644 --- a/include/tree_sitter/compiler.h +++ b/include/tree_sitter/compiler.h @@ -29,15 +29,15 @@ rule_ptr token(const rule_ptr &rule); class Grammar { const std::vector> rules_; - std::vector ubiquitous_tokens_; + std::vector extra_tokens_; std::vector> expected_conflicts_; public: explicit Grammar(const std::vector> &); - Grammar &ubiquitous_tokens(const std::vector &); + Grammar &extra_tokens(const std::vector &); Grammar &expected_conflicts(const std::vector> &); const std::vector> &rules() const; - const std::vector &ubiquitous_tokens() const; + const std::vector &extra_tokens() const; const std::vector> &expected_conflicts() const; }; diff --git a/spec/compiler/prepare_grammar/extract_tokens_spec.cc b/spec/compiler/prepare_grammar/extract_tokens_spec.cc index 365f0713..6bce5ee1 100644 --- a/spec/compiler/prepare_grammar/extract_tokens_spec.cc +++ b/spec/compiler/prepare_grammar/extract_tokens_spec.cc @@ -141,8 +141,8 @@ describe("extract_tokens", []() { }))); }); - describe("handling ubiquitous tokens", [&]() { - it("adds inline ubiquitous tokens to the lexical grammar's separators", [&]() { + describe("handling extra tokens", [&]() { + it("adds inline extra tokens to the lexical grammar's separators", [&]() { auto result = extract_tokens(InternedGrammar{{ Variable("rule_A", VariableTypeNamed, str("x")), }, { @@ -156,10 +156,10 @@ describe("extract_tokens", []() { AssertThat(get<1>(result).separators[0], EqualsPointer(str("y"))); AssertThat(get<1>(result).separators[1], EqualsPointer(pattern("\\s+"))); - AssertThat(get<0>(result).ubiquitous_tokens, IsEmpty()); + AssertThat(get<0>(result).extra_tokens, IsEmpty()); }); - it("handles inline ubiquitous tokens that match tokens in the grammar", [&]() { + it("handles inline extra tokens that match tokens in the grammar", [&]() { auto result = extract_tokens(InternedGrammar{{ Variable("rule_A", VariableTypeNamed, str("x")), Variable("rule_B", VariableTypeNamed, str("y")), @@ -169,10 +169,10 @@ describe("extract_tokens", []() { AssertThat(get<2>(result), Equals(nullptr)); AssertThat(get<1>(result).separators.size(), Equals(0)); - AssertThat(get<0>(result).ubiquitous_tokens, Equals(set({ Symbol(1, true) }))); + AssertThat(get<0>(result).extra_tokens, Equals(set({ Symbol(1, true) }))); }); - it("updates ubiquitous symbols according to the new symbol numbers", [&]() { + it("updates extra symbols according to the new symbol numbers", [&]() { auto result = extract_tokens(InternedGrammar{{ Variable("rule_A", VariableTypeNamed, seq({ str("w"), str("x"), i_sym(1) })), Variable("rule_B", VariableTypeNamed, str("y")), @@ -183,14 +183,14 @@ describe("extract_tokens", []() { AssertThat(get<2>(result), Equals(nullptr)); - AssertThat(get<0>(result).ubiquitous_tokens, Equals(set({ + AssertThat(get<0>(result).extra_tokens, Equals(set({ { Symbol(3, true) }, }))); AssertThat(get<1>(result).separators, IsEmpty()); }); - it("returns an error if any ubiquitous tokens are non-token symbols", [&]() { + it("returns an error if any extra tokens are non-token symbols", [&]() { auto result = extract_tokens(InternedGrammar{{ Variable("rule_A", VariableTypeNamed, seq({ str("x"), i_sym(1) })), Variable("rule_B", VariableTypeNamed, seq({ str("y"), str("z") })), @@ -202,7 +202,7 @@ describe("extract_tokens", []() { "Not a token: rule_B"))); }); - it("returns an error if any ubiquitous tokens are non-token rules", [&]() { + it("returns an error if any extra tokens are non-token rules", [&]() { auto result = extract_tokens(InternedGrammar{{ Variable("rule_A", VariableTypeNamed, str("x")), Variable("rule_B", VariableTypeNamed, str("y")), diff --git a/spec/compiler/prepare_grammar/intern_symbols_spec.cc b/spec/compiler/prepare_grammar/intern_symbols_spec.cc index 5bc5d62a..dbf4871b 100644 --- a/spec/compiler/prepare_grammar/intern_symbols_spec.cc +++ b/spec/compiler/prepare_grammar/intern_symbols_spec.cc @@ -38,18 +38,18 @@ describe("intern_symbols", []() { }); }); - it("translates the grammar's optional 'ubiquitous_tokens' to numerical symbols", [&]() { + it("translates the grammar's optional 'extra_tokens' to numerical symbols", [&]() { auto grammar = Grammar({ { "x", choice({ sym("y"), sym("z") }) }, { "y", sym("z") }, { "z", str("stuff") } - }).ubiquitous_tokens({ sym("z") }); + }).extra_tokens({ sym("z") }); auto result = intern_symbols(grammar); AssertThat(result.second, Equals((GrammarError *)nullptr)); - AssertThat(result.first.ubiquitous_tokens.size(), Equals(1)); - AssertThat(*result.first.ubiquitous_tokens.begin(), EqualsPointer(i_sym(2))); + AssertThat(result.first.extra_tokens.size(), Equals(1)); + AssertThat(*result.first.extra_tokens.begin(), EqualsPointer(i_sym(2))); }); }); diff --git a/spec/fixtures/grammars/anonymous_tokens.cc b/spec/fixtures/grammars/anonymous_tokens.cc index c64c40b6..681442be 100644 --- a/spec/fixtures/grammars/anonymous_tokens.cc +++ b/spec/fixtures/grammars/anonymous_tokens.cc @@ -9,7 +9,7 @@ extern const Grammar anonymous_tokens = Grammar({ str("\r"), pattern("\\d"), str("\"hello\"") }) }, -}).ubiquitous_tokens({ +}).extra_tokens({ pattern("\\s"), }); diff --git a/spec/fixtures/grammars/arithmetic.cc b/spec/fixtures/grammars/arithmetic.cc index 6f45271f..b26f6e59 100644 --- a/spec/fixtures/grammars/arithmetic.cc +++ b/spec/fixtures/grammars/arithmetic.cc @@ -34,7 +34,7 @@ extern const Grammar arithmetic = Grammar({ { "variable", pattern("[a-zA-Z\u03b1-\u03c9]+\\d*") }, { "comment", pattern("#.*") }, -}).ubiquitous_tokens({ +}).extra_tokens({ sym("comment"), pattern("\\s"), }); diff --git a/spec/fixtures/grammars/c.cc b/spec/fixtures/grammars/c.cc index d8f4fbe4..02202a8e 100644 --- a/spec/fixtures/grammars/c.cc +++ b/spec/fixtures/grammars/c.cc @@ -251,7 +251,7 @@ extern const Grammar c = Grammar({ pattern("[^\\*]"), pattern("\\*[^/]") })), str("*/") }) })) }, -}).ubiquitous_tokens({ +}).extra_tokens({ sym("comment"), pattern("[ \t\r\n]"), }).expected_conflicts({ diff --git a/spec/fixtures/grammars/cpp.cc b/spec/fixtures/grammars/cpp.cc index 672d731f..3c047024 100644 --- a/spec/fixtures/grammars/cpp.cc +++ b/spec/fixtures/grammars/cpp.cc @@ -211,7 +211,7 @@ extern const Grammar cpp = Grammar({ { "number", pattern("\\d+(\\.\\d+)?") }, { "comment", pattern("//[^\n]*") }, -}).ubiquitous_tokens({ +}).extra_tokens({ sym("comment"), pattern("[ \t\r\n]"), }).expected_conflicts({ diff --git a/spec/fixtures/grammars/golang.cc b/spec/fixtures/grammars/golang.cc index e9ada843..9bbb33db 100644 --- a/spec/fixtures/grammars/golang.cc +++ b/spec/fixtures/grammars/golang.cc @@ -203,7 +203,7 @@ extern const Grammar golang = Grammar({ { "comment", pattern("//[^\n]*") }, -}).ubiquitous_tokens({ +}).extra_tokens({ sym("comment"), sym("_line_break"), pattern("[ \t\r]"), diff --git a/spec/fixtures/grammars/javascript.cc b/spec/fixtures/grammars/javascript.cc index 0c7d9c57..e7887888 100644 --- a/spec/fixtures/grammars/javascript.cc +++ b/spec/fixtures/grammars/javascript.cc @@ -349,7 +349,7 @@ extern const Grammar javascript = Grammar({ str(")"), sym("statement_block") }) }, -}).ubiquitous_tokens({ +}).extra_tokens({ sym("comment"), sym("_line_break"), pattern("[ \t\r]"), diff --git a/spec/fixtures/grammars/json.cc b/spec/fixtures/grammars/json.cc index 0d31a12b..e58bd0a7 100644 --- a/spec/fixtures/grammars/json.cc +++ b/spec/fixtures/grammars/json.cc @@ -22,7 +22,7 @@ extern const Grammar json = Grammar({ { "null", str("null") }, { "true", str("true") }, { "false", str("false") }, -}).ubiquitous_tokens({ +}).extra_tokens({ pattern("\\s"), }); diff --git a/spec/runtime/parser_spec.cc b/spec/runtime/parser_spec.cc index cf2895b9..aed6d833 100644 --- a/spec/runtime/parser_spec.cc +++ b/spec/runtime/parser_spec.cc @@ -151,9 +151,9 @@ describe("Parser", [&]() { }); }); - describe("handling ubiquitous tokens", [&]() { + describe("handling extra tokens", [&]() { // In the javascript example grammar, ASI works by using newlines as - // terminators in statements, but also as ubiquitous tokens. + // terminators in statements, but also as extra tokens. before_each([&]() { ts_document_set_language(doc, ts_language_javascript()); }); @@ -180,7 +180,7 @@ describe("Parser", [&]() { }); }); - describe("when several ubiquitous tokens appear in a row", [&]() { + describe("when several extra tokens appear in a row", [&]() { it("is incorporated into the tree", [&]() { set_text( "fn()\n\n" @@ -337,7 +337,7 @@ describe("Parser", [&]() { }); }); - describe("into a node containing a ubiquitous token", [&]() { + describe("into a node containing a extra token", [&]() { before_each([&]() { set_text("123 *\n" "# a-comment\n" diff --git a/src/compiler/build_tables/build_parse_table.cc b/src/compiler/build_tables/build_parse_table.cc index 660077bb..85ee9c5a 100644 --- a/src/compiler/build_tables/build_parse_table.cc +++ b/src/compiler/build_tables/build_parse_table.cc @@ -134,22 +134,22 @@ class ParseTableBuilder { void add_shift_extra_actions(ParseStateId state_id) { ParseAction action = ParseAction::ShiftExtra(); - for (const Symbol &ubiquitous_symbol : grammar.ubiquitous_tokens) - add_action(state_id, ubiquitous_symbol, action, null_item_set); + for (const Symbol &extra_symbol : grammar.extra_tokens) + add_action(state_id, extra_symbol, action, null_item_set); } void add_reduce_extra_actions(ParseStateId state_id) { const ParseState &state = parse_table.states[state_id]; - for (const Symbol &ubiquitous_symbol : grammar.ubiquitous_tokens) { - const auto &actions_for_symbol = state.actions.find(ubiquitous_symbol); + for (const Symbol &extra_symbol : grammar.extra_tokens) { + const auto &actions_for_symbol = state.actions.find(extra_symbol); if (actions_for_symbol == state.actions.end()) continue; for (const ParseAction &action : actions_for_symbol->second) if (action.type == ParseActionTypeShift && !action.extra) { size_t dest_state_id = action.state_index; - ParseAction reduce_extra = ParseAction::ReduceExtra(ubiquitous_symbol); + ParseAction reduce_extra = ParseAction::ReduceExtra(extra_symbol); for (const auto &pair : state.actions) add_action(dest_state_id, pair.first, reduce_extra, null_item_set); } diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index e8ad9fac..d6e2573c 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -172,7 +172,7 @@ class CCodeGenerator { add(", "); - if (syntax_grammar.ubiquitous_tokens.count(symbol)) + if (syntax_grammar.extra_tokens.count(symbol)) add(".extra = true"); else add(".extra = false"); diff --git a/src/compiler/grammar.cc b/src/compiler/grammar.cc index b0883b07..e5adccbf 100644 --- a/src/compiler/grammar.cc +++ b/src/compiler/grammar.cc @@ -9,22 +9,22 @@ using std::string; using std::vector; Grammar::Grammar(const vector> &rules) - : rules_(rules), ubiquitous_tokens_({}) {} + : rules_(rules), extra_tokens_({}) {} const vector> &Grammar::rules() const { return rules_; } -const vector &Grammar::ubiquitous_tokens() const { - return ubiquitous_tokens_; +const vector &Grammar::extra_tokens() const { + return extra_tokens_; } const vector> &Grammar::expected_conflicts() const { return expected_conflicts_; } -Grammar &Grammar::ubiquitous_tokens(const vector &ubiquitous_tokens) { - ubiquitous_tokens_ = ubiquitous_tokens; +Grammar &Grammar::extra_tokens(const vector &extra_tokens) { + extra_tokens_ = extra_tokens; return *this; } diff --git a/src/compiler/prepare_grammar/expand_repeats.cc b/src/compiler/prepare_grammar/expand_repeats.cc index c49cf5bb..132b1f7b 100644 --- a/src/compiler/prepare_grammar/expand_repeats.cc +++ b/src/compiler/prepare_grammar/expand_repeats.cc @@ -62,7 +62,7 @@ class ExpandRepeats : public rules::IdentityRuleFn { InitialSyntaxGrammar expand_repeats(const InitialSyntaxGrammar &grammar) { InitialSyntaxGrammar result; result.variables = grammar.variables; - result.ubiquitous_tokens = grammar.ubiquitous_tokens; + result.extra_tokens = grammar.extra_tokens; result.expected_conflicts = grammar.expected_conflicts; ExpandRepeats expander(result.variables.size()); diff --git a/src/compiler/prepare_grammar/extract_tokens.cc b/src/compiler/prepare_grammar/extract_tokens.cc index 6b9f9211..f0329fcd 100644 --- a/src/compiler/prepare_grammar/extract_tokens.cc +++ b/src/compiler/prepare_grammar/extract_tokens.cc @@ -147,18 +147,18 @@ tuple extract_tokens } /* - * The grammar's ubiquitous tokens can be either token rules or symbols + * The grammar's extra tokens can be either token rules or symbols * pointing to token rules. If they are symbols, then they'll be handled by * the parser; add them to the syntax grammar's ubiqutous tokens. If they * are anonymous rules, they can be handled by the lexer; add them to the * lexical grammar's separator rules. */ - for (const rule_ptr &rule : grammar.ubiquitous_tokens) { + for (const rule_ptr &rule : grammar.extra_tokens) { int i = 0; bool used_elsewhere_in_grammar = false; for (const Variable &variable : lexical_grammar.variables) { if (variable.rule->operator==(*rule)) { - syntax_grammar.ubiquitous_tokens.insert(Symbol(i, true)); + syntax_grammar.extra_tokens.insert(Symbol(i, true)); used_elsewhere_in_grammar = true; } i++; @@ -183,7 +183,7 @@ tuple extract_tokens syntax_grammar, lexical_grammar, ubiq_token_err(syntax_grammar.variables[new_symbol.index].name)); - syntax_grammar.ubiquitous_tokens.insert(new_symbol); + syntax_grammar.extra_tokens.insert(new_symbol); } return make_tuple(syntax_grammar, lexical_grammar, nullptr); diff --git a/src/compiler/prepare_grammar/flatten_grammar.cc b/src/compiler/prepare_grammar/flatten_grammar.cc index 44bf7006..a84cd43e 100644 --- a/src/compiler/prepare_grammar/flatten_grammar.cc +++ b/src/compiler/prepare_grammar/flatten_grammar.cc @@ -75,7 +75,7 @@ class FlattenRule : public rules::RuleFn { SyntaxGrammar flatten_grammar(const InitialSyntaxGrammar &grammar) { SyntaxGrammar result; result.expected_conflicts = grammar.expected_conflicts; - result.ubiquitous_tokens = grammar.ubiquitous_tokens; + result.extra_tokens = grammar.extra_tokens; for (const Variable &variable : grammar.variables) { vector productions; diff --git a/src/compiler/prepare_grammar/initial_syntax_grammar.h b/src/compiler/prepare_grammar/initial_syntax_grammar.h index ccbc19ff..fe1ff37d 100644 --- a/src/compiler/prepare_grammar/initial_syntax_grammar.h +++ b/src/compiler/prepare_grammar/initial_syntax_grammar.h @@ -14,7 +14,7 @@ namespace prepare_grammar { struct InitialSyntaxGrammar { std::vector variables; - std::set ubiquitous_tokens; + std::set extra_tokens; std::set expected_conflicts; }; diff --git a/src/compiler/prepare_grammar/intern_symbols.cc b/src/compiler/prepare_grammar/intern_symbols.cc index 20e2328c..cd77e6a9 100644 --- a/src/compiler/prepare_grammar/intern_symbols.cc +++ b/src/compiler/prepare_grammar/intern_symbols.cc @@ -61,11 +61,11 @@ pair intern_symbols(const Grammar &gramma new_rule)); } - for (auto &rule : grammar.ubiquitous_tokens()) { + for (auto &rule : grammar.extra_tokens()) { auto new_rule = interner.apply(rule); if (!interner.missing_rule_name.empty()) return { result, missing_rule_error(interner.missing_rule_name) }; - result.ubiquitous_tokens.push_back(new_rule); + result.extra_tokens.push_back(new_rule); } for (auto &names : grammar.expected_conflicts()) { diff --git a/src/compiler/prepare_grammar/interned_grammar.h b/src/compiler/prepare_grammar/interned_grammar.h index 00a722a8..c08c07dd 100644 --- a/src/compiler/prepare_grammar/interned_grammar.h +++ b/src/compiler/prepare_grammar/interned_grammar.h @@ -13,7 +13,7 @@ namespace prepare_grammar { struct InternedGrammar { std::vector variables; - std::vector ubiquitous_tokens; + std::vector extra_tokens; std::set expected_conflicts; }; diff --git a/src/compiler/syntax_grammar.h b/src/compiler/syntax_grammar.h index f582318d..67b36f3e 100644 --- a/src/compiler/syntax_grammar.h +++ b/src/compiler/syntax_grammar.h @@ -37,7 +37,7 @@ struct SyntaxGrammar { const std::vector &productions(const rules::Symbol &) const; std::vector variables; - std::set ubiquitous_tokens; + std::set extra_tokens; std::set expected_conflicts; };