traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make completion_status() a method on LexItem

Browse source
This commit is contained in:
Max Brunsfeld 2015-10-30 16:48:37 -07:00
parent c8be143f65
commit 998ae533da
10 changed files with 96 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

1
project.gyp
View file

@ -14,7 +14,6 @@
'src/compiler/build_tables/build_parse_table.cc',
'src/compiler/build_tables/build_tables.cc',
'src/compiler/build_tables/does_match_any_line.cc',
'src/compiler/build_tables/get_completion_status.cc',
'src/compiler/build_tables/item_set_closure.cc',
'src/compiler/build_tables/lex_item.cc',
'src/compiler/build_tables/lex_item_transitions.cc',

23
spec/compiler/build_tables/lex_item_spec.cc
View file

@ -38,6 +38,29 @@ describe("LexItem", []() {
});
});
});
describe("completion_status()", [&]() {
it("indicates whether the item is done, its precedence, and whether it is a string", [&]() {
LexItem item1(Symbol(0, true), character({ 'a', 'b', 'c' }));
AssertThat(item1.completion_status().is_done, IsFalse());
AssertThat(item1.completion_status().precedence, Equals(0));
AssertThat(item1.completion_status().is_string, IsFalse());
LexItem item2(Symbol(0, true), choice({
metadata(blank(), { {PRECEDENCE, 3}, {IS_STRING, 1} }),
character({ 'a', 'b', 'c' })
}));
AssertThat(item2.completion_status().is_done, IsTrue());
AssertThat(item2.completion_status().precedence, Equals(3));
AssertThat(item2.completion_status().is_string, IsTrue());
LexItem item3(Symbol(0, true), repeat(character({ ' ', '\t' })));
AssertThat(item3.completion_status().is_done, IsTrue());
AssertThat(item3.completion_status().precedence, Equals(0));
AssertThat(item3.completion_status().is_string, IsFalse());
});
});
});
describe("LexItemSet::transitions()", [&]() {

2
spec/fixtures/parsers/javascript.c vendored
View file

@ -594,7 +594,7 @@ static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) {
(lookahead == '\n') ||
(lookahead == 'g')))
ADVANCE(29);
ACCEPT_TOKEN(sym_comment);
ACCEPT_TOKEN(sym_regex);
case 28:
if (!((lookahead == 0) ||
(lookahead == '\n')))

3
src/compiler/build_tables/build_lex_table.cc
View file

@ -6,7 +6,6 @@
#include <utility>
#include <vector>
#include "compiler/build_tables/lex_conflict_manager.h"
#include "compiler/build_tables/get_completion_status.h"
#include "compiler/build_tables/lex_item.h"
#include "compiler/build_tables/does_match_any_line.h"
#include "compiler/parse_table.h"
@ -133,7 +132,7 @@ class LexTableBuilder {
void add_accept_token_actions(const LexItemSet &item_set, LexStateId state_id) {
for (const LexItem &item : item_set.entries) {
CompletionStatus completion_status = get_completion_status(item.rule);
LexItem::CompletionStatus completion_status = item.completion_status();
if (completion_status.is_done) {
auto current_action = lex_table.state(state_id).default_action;
auto action = LexAction::Accept(item.lhs, completion_status.precedence);

54
src/compiler/build_tables/get_completion_status.cc
View file

@ -1,54 +0,0 @@
#include "compiler/build_tables/get_completion_status.h"
#include "compiler/rules/visitor.h"
#include "compiler/rules/choice.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/metadata.h"
#include "compiler/rules/repeat.h"
namespace tree_sitter {
namespace build_tables {
class GetCompletionStatus : public rules::RuleFn<CompletionStatus> {
protected:
CompletionStatus apply_to(const rules::Choice *rule) {
for (const auto &element : rule->elements) {
CompletionStatus status = apply(element);
if (status.is_done)
return status;
}
return { false, 0, rules::AssociativityNone };
}
CompletionStatus apply_to(const rules::Metadata *rule) {
CompletionStatus result = apply(rule->rule);
if (result.is_done && !result.associativity) {
result.precedence = rule->value_for(rules::PRECEDENCE);
result.associativity =
(rules::Associativity)(rule->value_for(rules::ASSOCIATIVITY));
}
return result;
}
CompletionStatus apply_to(const rules::Repeat *rule) {
return apply(rule->content);
}
CompletionStatus apply_to(const rules::Blank *rule) {
return { true, 0, rules::AssociativityNone };
}
CompletionStatus apply_to(const rules::Seq *rule) {
CompletionStatus left_status = apply(rule->left);
if (left_status.is_done)
return apply(rule->right);
else
return { false, 0, rules::AssociativityNone };
}
};
CompletionStatus get_completion_status(const rule_ptr &rule) {
return GetCompletionStatus().apply(rule);
}
} // namespace build_tables
} // namespace tree_sitter

21
src/compiler/build_tables/get_completion_status.h
View file

@ -1,21 +0,0 @@
#ifndef COMPILER_BUILD_TABLES_GET_COMPLETION_STATUS_H_
#define COMPILER_BUILD_TABLES_GET_COMPLETION_STATUS_H_
#include "tree_sitter/compiler.h"
#include "compiler/rules/metadata.h"
namespace tree_sitter {
namespace build_tables {
struct CompletionStatus {
bool is_done;
int precedence;
rules::Associativity associativity;
};
CompletionStatus get_completion_status(const rule_ptr &);
} // namespace build_tables
} // namespace tree_sitter
#endif // COMPILER_BUILD_TABLES_GET_COMPLETION_STATUS_H_

44
src/compiler/build_tables/lex_item.cc
View file

@ -6,6 +6,7 @@
#include "compiler/rules/metadata.h"
#include "compiler/rules/seq.h"
#include "compiler/rules/symbol.h"
#include "compiler/rules/repeat.h"
#include "compiler/rules/visitor.h"
namespace tree_sitter {
@ -47,6 +48,49 @@ bool LexItem::is_token_start() const {
return IsTokenStart().apply(rule);
}
LexItem::CompletionStatus LexItem::completion_status() const {
class GetCompletionStatus : public rules::RuleFn<CompletionStatus> {
protected:
CompletionStatus apply_to(const rules::Choice *rule) {
for (const auto &element : rule->elements) {
CompletionStatus status = apply(element);
if (status.is_done)
return status;
}
return { false, 0, false };
}
CompletionStatus apply_to(const rules::Metadata *rule) {
CompletionStatus result = apply(rule->rule);
if (result.is_done) {
if (!result.precedence && rule->value_for(rules::PRECEDENCE))
result.precedence = rule->value_for(rules::PRECEDENCE);
if (rule->value_for(rules::IS_STRING))
result.is_string = true;
}
return result;
}
CompletionStatus apply_to(const rules::Repeat *rule) {
return apply(rule->content);
}
CompletionStatus apply_to(const rules::Blank *rule) {
return { true, 0, false };
}
CompletionStatus apply_to(const rules::Seq *rule) {
CompletionStatus left_status = apply(rule->left);
if (left_status.is_done)
return apply(rule->right);
else
return { false, 0, false };
}
};
return GetCompletionStatus().apply(rule);
}
size_t LexItem::Hash::operator()(const LexItem &item) const {
return hash<Symbol>()(item.lhs) ^ hash<rule_ptr>()(item.rule);
}

26
src/compiler/build_tables/lex_item.h
View file

@ -14,16 +14,24 @@ namespace build_tables {
class LexItem {
public:
LexItem(const rules::Symbol &lhs, rule_ptr rule);
bool operator==(const LexItem &other) const;
bool is_token_start() const;
LexItem(const rules::Symbol &, rule_ptr);
rules::Symbol lhs;
rule_ptr rule;
struct CompletionStatus {
bool is_done;
int precedence;
bool is_string;
};
struct Hash {
size_t operator()(const LexItem &) const;
};
bool operator==(const LexItem &other) const;
bool is_token_start() const;
CompletionStatus completion_status() const;
rules::Symbol lhs;
rule_ptr rule;
};
class LexItemSet {
@ -34,14 +42,14 @@ class LexItemSet {
typedef std::map<rules::CharacterSet, std::pair<LexItemSet, PrecedenceRange>>
TransitionMap;
struct Hash {
size_t operator()(const LexItemSet &) const;
};
bool operator==(const LexItemSet &) const;
TransitionMap transitions() const;
std::unordered_set<LexItem, LexItem::Hash> entries;
struct Hash {
size_t operator()(const LexItemSet &) const;
};
};
} // namespace build_tables

15
src/compiler/build_tables/parse_item.h
View file

@ -22,6 +22,10 @@ class ParseItem {
rules::Associativity associativity;
};
struct Hash {
size_t operator()(const ParseItem &) const;
};
bool operator==(const ParseItem &other) const;
bool operator<(const ParseItem &other) const;
rules::Symbol lhs() const;
@ -34,9 +38,6 @@ class ParseItem {
const Production *production;
unsigned int step_index;
struct Hash {
size_t operator()(const ParseItem &) const;
};
};
class ParseItemSet {
@ -47,14 +48,14 @@ class ParseItemSet {
typedef std::map<rules::Symbol, std::pair<ParseItemSet, PrecedenceRange>>
TransitionMap;
struct Hash {
size_t operator()(const ParseItemSet &) const;
};
TransitionMap transitions() const;
bool operator==(const ParseItemSet &) const;
std::map<ParseItem, LookaheadSet> entries;
struct Hash {
size_t operator()(const ParseItemSet &) const;
};
};
} // namespace build_tables

3
src/compiler/rules/metadata.h
View file

@ -18,8 +18,9 @@ enum Associativity {
enum MetadataKey {
START_TOKEN,
PRECEDENCE,
IS_TOKEN,
ASSOCIATIVITY,
IS_TOKEN,
IS_STRING,
};
class Metadata : public Rule {