From 998ae533da3f92a4eec69de0a648d93b1e9c1ca9 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 30 Oct 2015 16:48:37 -0700 Subject: [PATCH] Make completion_status() a method on LexItem --- project.gyp | 1 - spec/compiler/build_tables/lex_item_spec.cc | 23 ++++++++ spec/fixtures/parsers/javascript.c | 2 +- src/compiler/build_tables/build_lex_table.cc | 3 +- .../build_tables/get_completion_status.cc | 54 ------------------- .../build_tables/get_completion_status.h | 21 -------- src/compiler/build_tables/lex_item.cc | 44 +++++++++++++++ src/compiler/build_tables/lex_item.h | 26 +++++---- src/compiler/build_tables/parse_item.h | 15 +++--- src/compiler/rules/metadata.h | 3 +- 10 files changed, 96 insertions(+), 96 deletions(-) delete mode 100644 src/compiler/build_tables/get_completion_status.cc delete mode 100644 src/compiler/build_tables/get_completion_status.h diff --git a/project.gyp b/project.gyp index 61449009..4ea53144 100644 --- a/project.gyp +++ b/project.gyp @@ -14,7 +14,6 @@ 'src/compiler/build_tables/build_parse_table.cc', 'src/compiler/build_tables/build_tables.cc', 'src/compiler/build_tables/does_match_any_line.cc', - 'src/compiler/build_tables/get_completion_status.cc', 'src/compiler/build_tables/item_set_closure.cc', 'src/compiler/build_tables/lex_item.cc', 'src/compiler/build_tables/lex_item_transitions.cc', diff --git a/spec/compiler/build_tables/lex_item_spec.cc b/spec/compiler/build_tables/lex_item_spec.cc index f80ace52..9e301db1 100644 --- a/spec/compiler/build_tables/lex_item_spec.cc +++ b/spec/compiler/build_tables/lex_item_spec.cc @@ -38,6 +38,29 @@ describe("LexItem", []() { }); }); }); + + describe("completion_status()", [&]() { + it("indicates whether the item is done, its precedence, and whether it is a string", [&]() { + LexItem item1(Symbol(0, true), character({ 'a', 'b', 'c' })); + AssertThat(item1.completion_status().is_done, IsFalse()); + AssertThat(item1.completion_status().precedence, Equals(0)); + AssertThat(item1.completion_status().is_string, IsFalse()); + + LexItem item2(Symbol(0, true), choice({ + metadata(blank(), { {PRECEDENCE, 3}, {IS_STRING, 1} }), + character({ 'a', 'b', 'c' }) + })); + + AssertThat(item2.completion_status().is_done, IsTrue()); + AssertThat(item2.completion_status().precedence, Equals(3)); + AssertThat(item2.completion_status().is_string, IsTrue()); + + LexItem item3(Symbol(0, true), repeat(character({ ' ', '\t' }))); + AssertThat(item3.completion_status().is_done, IsTrue()); + AssertThat(item3.completion_status().precedence, Equals(0)); + AssertThat(item3.completion_status().is_string, IsFalse()); + }); + }); }); describe("LexItemSet::transitions()", [&]() { diff --git a/spec/fixtures/parsers/javascript.c b/spec/fixtures/parsers/javascript.c index c172dce0..216af2d8 100644 --- a/spec/fixtures/parsers/javascript.c +++ b/spec/fixtures/parsers/javascript.c @@ -594,7 +594,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { (lookahead == '\n') || (lookahead == 'g'))) ADVANCE(29); - ACCEPT_TOKEN(sym_comment); + ACCEPT_TOKEN(sym_regex); case 28: if (!((lookahead == 0) || (lookahead == '\n'))) diff --git a/src/compiler/build_tables/build_lex_table.cc b/src/compiler/build_tables/build_lex_table.cc index 583892f0..802dff48 100644 --- a/src/compiler/build_tables/build_lex_table.cc +++ b/src/compiler/build_tables/build_lex_table.cc @@ -6,7 +6,6 @@ #include #include #include "compiler/build_tables/lex_conflict_manager.h" -#include "compiler/build_tables/get_completion_status.h" #include "compiler/build_tables/lex_item.h" #include "compiler/build_tables/does_match_any_line.h" #include "compiler/parse_table.h" @@ -133,7 +132,7 @@ class LexTableBuilder { void add_accept_token_actions(const LexItemSet &item_set, LexStateId state_id) { for (const LexItem &item : item_set.entries) { - CompletionStatus completion_status = get_completion_status(item.rule); + LexItem::CompletionStatus completion_status = item.completion_status(); if (completion_status.is_done) { auto current_action = lex_table.state(state_id).default_action; auto action = LexAction::Accept(item.lhs, completion_status.precedence); diff --git a/src/compiler/build_tables/get_completion_status.cc b/src/compiler/build_tables/get_completion_status.cc deleted file mode 100644 index 4cc70c60..00000000 --- a/src/compiler/build_tables/get_completion_status.cc +++ /dev/null @@ -1,54 +0,0 @@ -#include "compiler/build_tables/get_completion_status.h" -#include "compiler/rules/visitor.h" -#include "compiler/rules/choice.h" -#include "compiler/rules/seq.h" -#include "compiler/rules/metadata.h" -#include "compiler/rules/repeat.h" - -namespace tree_sitter { -namespace build_tables { - -class GetCompletionStatus : public rules::RuleFn { - protected: - CompletionStatus apply_to(const rules::Choice *rule) { - for (const auto &element : rule->elements) { - CompletionStatus status = apply(element); - if (status.is_done) - return status; - } - return { false, 0, rules::AssociativityNone }; - } - - CompletionStatus apply_to(const rules::Metadata *rule) { - CompletionStatus result = apply(rule->rule); - if (result.is_done && !result.associativity) { - result.precedence = rule->value_for(rules::PRECEDENCE); - result.associativity = - (rules::Associativity)(rule->value_for(rules::ASSOCIATIVITY)); - } - return result; - } - - CompletionStatus apply_to(const rules::Repeat *rule) { - return apply(rule->content); - } - - CompletionStatus apply_to(const rules::Blank *rule) { - return { true, 0, rules::AssociativityNone }; - } - - CompletionStatus apply_to(const rules::Seq *rule) { - CompletionStatus left_status = apply(rule->left); - if (left_status.is_done) - return apply(rule->right); - else - return { false, 0, rules::AssociativityNone }; - } -}; - -CompletionStatus get_completion_status(const rule_ptr &rule) { - return GetCompletionStatus().apply(rule); -} - -} // namespace build_tables -} // namespace tree_sitter diff --git a/src/compiler/build_tables/get_completion_status.h b/src/compiler/build_tables/get_completion_status.h deleted file mode 100644 index 237f3eb2..00000000 --- a/src/compiler/build_tables/get_completion_status.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef COMPILER_BUILD_TABLES_GET_COMPLETION_STATUS_H_ -#define COMPILER_BUILD_TABLES_GET_COMPLETION_STATUS_H_ - -#include "tree_sitter/compiler.h" -#include "compiler/rules/metadata.h" - -namespace tree_sitter { -namespace build_tables { - -struct CompletionStatus { - bool is_done; - int precedence; - rules::Associativity associativity; -}; - -CompletionStatus get_completion_status(const rule_ptr &); - -} // namespace build_tables -} // namespace tree_sitter - -#endif // COMPILER_BUILD_TABLES_GET_COMPLETION_STATUS_H_ diff --git a/src/compiler/build_tables/lex_item.cc b/src/compiler/build_tables/lex_item.cc index db488e79..43c8fe72 100644 --- a/src/compiler/build_tables/lex_item.cc +++ b/src/compiler/build_tables/lex_item.cc @@ -6,6 +6,7 @@ #include "compiler/rules/metadata.h" #include "compiler/rules/seq.h" #include "compiler/rules/symbol.h" +#include "compiler/rules/repeat.h" #include "compiler/rules/visitor.h" namespace tree_sitter { @@ -47,6 +48,49 @@ bool LexItem::is_token_start() const { return IsTokenStart().apply(rule); } +LexItem::CompletionStatus LexItem::completion_status() const { + class GetCompletionStatus : public rules::RuleFn { + protected: + CompletionStatus apply_to(const rules::Choice *rule) { + for (const auto &element : rule->elements) { + CompletionStatus status = apply(element); + if (status.is_done) + return status; + } + return { false, 0, false }; + } + + CompletionStatus apply_to(const rules::Metadata *rule) { + CompletionStatus result = apply(rule->rule); + if (result.is_done) { + if (!result.precedence && rule->value_for(rules::PRECEDENCE)) + result.precedence = rule->value_for(rules::PRECEDENCE); + if (rule->value_for(rules::IS_STRING)) + result.is_string = true; + } + return result; + } + + CompletionStatus apply_to(const rules::Repeat *rule) { + return apply(rule->content); + } + + CompletionStatus apply_to(const rules::Blank *rule) { + return { true, 0, false }; + } + + CompletionStatus apply_to(const rules::Seq *rule) { + CompletionStatus left_status = apply(rule->left); + if (left_status.is_done) + return apply(rule->right); + else + return { false, 0, false }; + } + }; + + return GetCompletionStatus().apply(rule); +} + size_t LexItem::Hash::operator()(const LexItem &item) const { return hash()(item.lhs) ^ hash()(item.rule); } diff --git a/src/compiler/build_tables/lex_item.h b/src/compiler/build_tables/lex_item.h index f941de18..b547df4d 100644 --- a/src/compiler/build_tables/lex_item.h +++ b/src/compiler/build_tables/lex_item.h @@ -14,16 +14,24 @@ namespace build_tables { class LexItem { public: - LexItem(const rules::Symbol &lhs, rule_ptr rule); - bool operator==(const LexItem &other) const; - bool is_token_start() const; + LexItem(const rules::Symbol &, rule_ptr); - rules::Symbol lhs; - rule_ptr rule; + struct CompletionStatus { + bool is_done; + int precedence; + bool is_string; + }; struct Hash { size_t operator()(const LexItem &) const; }; + + bool operator==(const LexItem &other) const; + bool is_token_start() const; + CompletionStatus completion_status() const; + + rules::Symbol lhs; + rule_ptr rule; }; class LexItemSet { @@ -34,14 +42,14 @@ class LexItemSet { typedef std::map> TransitionMap; + struct Hash { + size_t operator()(const LexItemSet &) const; + }; + bool operator==(const LexItemSet &) const; TransitionMap transitions() const; std::unordered_set entries; - - struct Hash { - size_t operator()(const LexItemSet &) const; - }; }; } // namespace build_tables diff --git a/src/compiler/build_tables/parse_item.h b/src/compiler/build_tables/parse_item.h index f26f05a0..6a869b0f 100644 --- a/src/compiler/build_tables/parse_item.h +++ b/src/compiler/build_tables/parse_item.h @@ -22,6 +22,10 @@ class ParseItem { rules::Associativity associativity; }; + struct Hash { + size_t operator()(const ParseItem &) const; + }; + bool operator==(const ParseItem &other) const; bool operator<(const ParseItem &other) const; rules::Symbol lhs() const; @@ -34,9 +38,6 @@ class ParseItem { const Production *production; unsigned int step_index; - struct Hash { - size_t operator()(const ParseItem &) const; - }; }; class ParseItemSet { @@ -47,14 +48,14 @@ class ParseItemSet { typedef std::map> TransitionMap; + struct Hash { + size_t operator()(const ParseItemSet &) const; + }; + TransitionMap transitions() const; bool operator==(const ParseItemSet &) const; std::map entries; - - struct Hash { - size_t operator()(const ParseItemSet &) const; - }; }; } // namespace build_tables diff --git a/src/compiler/rules/metadata.h b/src/compiler/rules/metadata.h index ea4f9ba5..bcf23bd8 100644 --- a/src/compiler/rules/metadata.h +++ b/src/compiler/rules/metadata.h @@ -18,8 +18,9 @@ enum Associativity { enum MetadataKey { START_TOKEN, PRECEDENCE, - IS_TOKEN, ASSOCIATIVITY, + IS_TOKEN, + IS_STRING, }; class Metadata : public Rule {