traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Handle external tokens in apply_transitive_closure

Browse source
This commit is contained in:
Max Brunsfeld 2016-12-04 10:40:32 -08:00
parent 0f8e130687
commit d72b49316b
2 changed files with 39 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

58
spec/integration/compile_grammar_spec.cc
View file

@ -518,29 +518,43 @@ describe("compile_grammar", []() {
"percent_string_end"
],
"extras": [
{"type": "PATTERN", "value": "\\s"}
],
"rules": {
"expression": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "string"},
{"type": "SYMBOL", "name": "sum"},
{"type": "SYMBOL", "name": "identifier"}
]
},
"sum": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "+"},
{"type": "SYMBOL", "name": "expression"}
]
}
},
"string": {
"type": "CHOICE",
"members": [
{
"type": "EXTERNAL_TOKEN",
"name": "percent_string"
},
{"type": "EXTERNAL_TOKEN", "name": "percent_string"},
{
"type": "SEQ",
"members": [
{
"type": "EXTERNAL_TOKEN",
"name": "percent_string_start"
},
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "EXTERNAL_TOKEN",
"name": "percent_string_end"
}
{"type": "EXTERNAL_TOKEN", "name": "percent_string_start"},
{"type": "SYMBOL", "name": "expression"},
{"type": "EXTERNAL_TOKEN", "name": "percent_string_end"}
]
},
]
@ -562,17 +576,13 @@ describe("compile_grammar", []() {
"spec/fixtures/external_scanners/external_scan.c"
));
ts_document_set_input_string(document, "%(sup (external) scanner?)");
ts_document_set_input_string(document, "x + %(sup (external) scanner?)");
ts_document_parse(document);
assert_root_node("(string)");
assert_root_node("(expression (sum (expression (identifier)) (expression (string))))");
ts_document_set_input_string(document, "%{sup {} external {} scanner?}");
ts_document_set_input_string(document, "%{sup {} #{x + y} {} scanner?}");
ts_document_parse(document);
assert_root_node("(string)");
ts_document_set_input_string(document, "%(1 #{two} three)");
ts_document_parse(document);
assert_root_node("(string (identifier))");
assert_root_node("(expression (string (expression (sum (expression (identifier)) (expression (identifier))))))");
});
});

5
src/compiler/build_tables/parse_item_set_builder.cc
View file

@ -31,6 +31,11 @@ ParseItemSetBuilder::ParseItemSetBuilder(const SyntaxGrammar &grammar,
first_sets.insert({symbol, LookaheadSet({ symbol })});
}
for (size_t i = 0, n = grammar.external_tokens.size(); i < n; i++) {
Symbol symbol(i, Symbol::External);
first_sets.insert({symbol, LookaheadSet({ symbol })});
}
for (size_t i = 0, n = grammar.variables.size(); i < n; i++) {
Symbol symbol(i, Symbol::NonTerminal);
LookaheadSet first_set;