traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Convert some of the fixture grammars from JSON to JS

Browse source 
These tests are easier to write and maintain if the grammars are just JS,
like grammars normally are. It doesn't slow the tests down significantly
to shell out to `node` for each of these grammars.
This commit is contained in:
Max Brunsfeld 2021-10-22 18:47:13 -06:00
parent ddb12dc0c6
commit d05c665863
43 changed files with 386 additions and 792 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cli/src/generate/mod.rs
View file

@ -157,7 +157,7 @@ fn generate_parser_for_grammar_with_opts(
})
}
fn load_grammar_file(grammar_path: &Path) -> Result<String> {
pub fn load_grammar_file(grammar_path: &Path) -> Result<String> {
match grammar_path.extension().and_then(|e| e.to_str()) {
Some("js") => Ok(load_js_grammar_file(grammar_path)?),
Some("json") => Ok(fs::read_to_string(grammar_path)?),

7
cli/src/tests/corpus_test.rs
View file

@ -269,9 +269,12 @@ fn test_feature_corpus_files() {
}
let test_path = entry.path();
let grammar_path = test_path.join("grammar.json");
let mut grammar_path = test_path.join("grammar.js");
if !grammar_path.exists() {
grammar_path = test_path.join("grammar.json");
}
let error_message_path = test_path.join("expected_error.txt");
let grammar_json = fs::read_to_string(grammar_path).unwrap();
let grammar_json = generate::load_grammar_file(&grammar_path).unwrap();
let generate_result = generate::generate_parser_for_grammar(&grammar_json);
if error_message_path.exists() {

11
test/fixtures/test_grammars/aliased_inlined_rules/corpus.txt vendored
View file

@ -2,12 +2,13 @@
OK
=========================
a.b.c
a.b.c;
---
(expression (member_expression
(expression (member_expression
(expression (variable_name))
(statement
(member_expression
(member_expression
(variable_name)
(property_name))
(property_name)))
(property_name)))

28
test/fixtures/test_grammars/aliased_inlined_rules/grammar.js vendored Normal file
View file

@ -0,0 +1,28 @@
// This grammar shows that `ALIAS` rules can *contain* a rule that is marked as `inline`. It also
// shows that you can alias a rule that would otherwise be anonymous, and it will then appear as a
// named node.
module.exports = grammar({
name: 'aliased_inlined_rules',
extras: $ => [/\s/],
inline: $ => [$.identifier],
rules: {
statement: $ => seq($._expression, ';'),
_expression: $ => choice(
$.member_expression,
alias($.identifier, $.variable_name),
),
member_expression: $ => prec.left(1, seq(
$._expression,
'.',
alias($.identifier, $.property_name)
)),
identifier: $ => choice('a', 'b', 'c')
}
});

59
test/fixtures/test_grammars/aliased_inlined_rules/grammar.json vendored
View file

@ -1,59 +0,0 @@
{
"name": "aliased_inlined_rules",
"extras": [
{"type": "PATTERN", "value": "\\s"}
],
"inline": [
"identifier"
],
"rules": {
"expression": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "member_expression"},
{
"type": "ALIAS",
"value": "variable_name",
"named": true,
"content": {
"type": "SYMBOL",
"name": "identifier"
}
}
]
},
"member_expression": {
"type": "PREC_LEFT",
"value": 1,
"content": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "."},
{
"type": "ALIAS",
"value": "property_name",
"named": true,
"content": {
"type": "SYMBOL",
"name": "identifier"
}
}
]
}
},
"identifier": {
"type": "CHOICE",
"members": [
{"type": "STRING", "value": "a"},
{"type": "STRING", "value": "b"},
{"type": "STRING", "value": "c"}
]
}
}
}

1
test/fixtures/test_grammars/aliased_inlined_rules/readme.md vendored
View file

@ -1 +0,0 @@
This grammar shows that `ALIAS` rules can *contain* a rule that is marked as `inline`. It also shows that you can alias a rule that would otherwise be anonymous, and it will then appear as a named node.

30
test/fixtures/test_grammars/aliased_rules/grammar.js vendored Normal file
View file

@ -0,0 +1,30 @@
module.exports = grammar({
name: 'aliased_rules',
extras: $ => [/\s/],
rules: {
statement: $ => seq($._expression, ';'),
_expression: $ => choice(
$.call_expression,
$.member_expression,
alias($.identifier, $.variable_name),
),
call_expression: $ => prec.left(seq(
$._expression,
'(',
$._expression,
')'
)),
member_expression: $ => prec.left(1, seq(
$._expression,
'.',
alias($.identifier, $.property_name)
)),
identifier: $ => /[a-z]+/
}
});

71
test/fixtures/test_grammars/aliased_rules/grammar.json vendored
View file

@ -1,71 +0,0 @@
{
"name": "aliased_rules",
"extras": [
{"type": "PATTERN", "value": "\\s"}
],
"rules": {
"statement": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "_expression"},
{"type": "STRING", "value": ";"}
]
},
"_expression": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "call_expression"},
{"type": "SYMBOL", "name": "member_expression"},
{
"type": "ALIAS",
"named": true,
"value": "variable_name",
"content": {
"type": "SYMBOL",
"name": "identifier"
}
}
]
},
"call_expression": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "_expression"},
{"type": "STRING", "value": "("},
{"type": "SYMBOL", "name": "_expression"},
{"type": "STRING", "value": ")"}
]
}
},
"member_expression": {
"type": "PREC_LEFT",
"value": 1,
"content": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "_expression"},
{"type": "STRING", "value": "."},
{
"type": "ALIAS",
"named": true,
"value": "property_name",
"content": {
"type": "SYMBOL",
"name": "identifier"
}
}
]
}
},
"identifier": {"type": "PATTERN", "value": "[a-z]+"}
}
}

16
test/fixtures/test_grammars/aliased_token_rules/grammar.js vendored Normal file
View file

@ -0,0 +1,16 @@
// This grammar shows that `ALIAS` rules can be applied directly to `TOKEN` and `IMMEDIATE_TOKEN`
// rules.
module.exports = grammar({
name: 'aliased_token_rules',
extras: $ => [/\s/],
rules: {
expression: $ => seq(
'a',
alias(token(seq('b', 'c')), $.X),
alias(token.immediate(seq('d', 'e')), $.Y),
),
}
});

61
test/fixtures/test_grammars/aliased_token_rules/grammar.json vendored
View file

@ -1,61 +0,0 @@
{
"name": "aliased_token_rules",
"extras": [
{"type": "PATTERN", "value": "\\s"}
],
"rules": {
"expression": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "a"
},
{
"type": "ALIAS",
"value": "X",
"named": true,
"content": {
"type": "TOKEN",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "b"
},
{
"type": "STRING",
"value": "c"
}
]
}
}
},
{
"type": "ALIAS",
"value": "Y",
"named": true,
"content": {
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "d"
},
{
"type": "STRING",
"value": "e"
}
]
}
}
}
]
}
}
}

1
test/fixtures/test_grammars/aliased_token_rules/readme.md vendored
View file

@ -1 +0,0 @@
This grammar shows that `ALIAS` rules can be applied directly to `TOKEN` and `IMMEDIATE_TOKEN` rules.

38
test/fixtures/test_grammars/aliased_unit_reductions/grammar.js vendored Normal file
View file

@ -0,0 +1,38 @@
// Normally, when there are invisible rules (rules whose names start with an `_`) that simply wrap
// another rule, there is an optimization at parser-generation time called *Unit Reduction
// Elimination* that avoids creating nodes for those rules at runtime. One case where this
// optimization must *not* be applied is when those invisible rules are going to be aliased within
// their parent rule. In that situation, eliminating the invisible node could cause the alias to be
// incorrectly applied to its child.
module.exports = grammar({
name: 'aliased_unit_reductions',
extras: $ => [/\s/],
rules: {
statement: $ => seq(
$._a,
// The `_b` rule is always aliased to `b_prime`, so it is internally treated
// as a simple alias.
alias($._b, $.b_prime),
// The `_c` rule is used without an alias in addition to being aliased to `c_prime`,
// so it is not a simple alias.
alias($._c, $.c_prime),
$._c,
';'
),
_a: $ => $._A,
_b: $ => $._B,
_c: $ => $._C,
_A: $ => $.identifier,
_B: $ => $.identifier,
_C: $ => $.identifier,
identifier: $ => /[a-z]+/,
}
});

81
test/fixtures/test_grammars/aliased_unit_reductions/grammar.json vendored
View file

@ -1,81 +0,0 @@
{
"name": "aliased_unit_reductions",
"extras": [
{"type": "PATTERN", "value": "\\s"}
],
"rules": {
"statement": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "_a"},
// The `_b` rule is always aliased to `b_prime`, so it is internally treated
// as a simple alias.
{
"type": "ALIAS",
"named": true,
"value": "b_prime",
"content": {
"type": "SYMBOL",
"name": "_b"
}
},
// The `_c` rule is used without an alias in addition to being aliased to `c_prime`,
// so it is not a simple alias.
{
"type": "ALIAS",
"named": true,
"value": "c_prime",
"content": {
"type": "SYMBOL",
"name": "_c"
}
},
{
"type": "SYMBOL",
"name": "_c"
},
{
"type": "STRING",
"value": ";"
}
]
},
"_a": {
"type": "SYMBOL",
"name": "_A"
},
"_A": {
"type": "SYMBOL",
"name": "identifier"
},
"_b": {
"type": "SYMBOL",
"name": "_B"
},
"_B": {
"type": "SYMBOL",
"name": "identifier"
},
"_c": {
"type": "SYMBOL",
"name": "_C"
},
"_C": {
"type": "SYMBOL",
"name": "identifier"
},
"identifier": {"type": "PATTERN", "value": "[a-z]+"}
}
}

5
test/fixtures/test_grammars/aliased_unit_reductions/readme.md vendored
View file

@ -1,5 +0,0 @@
Normally, when there are invisible rules (rules whose names start with an `_`) that simply
wrap another rule, there is an optimization at parser-generation time called *Unit Reduction Elimination* that avoids creating nodes for those rules at runtime. One case where this
optimization must *not* be applied is when those invisible rules are going to be aliased
within their parent rule. In that situation, eliminating the invisible node could cause
the alias to be incorrectly applied to its child.

17
test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.js vendored Normal file
View file

@ -0,0 +1,17 @@
// Every token in a grammar is given a name in the generated parser. Anonymous tokens (tokens
// specified directly in the body of some larger rule) are named according their content. So when
// tokens contains characters that aren't valid in a C string literal, we need to escape those
// characters. This grammar tests that this escaping works. The test is basically that the generated
// parser compiles succesfully.
module.exports = grammar({
name: "anonymous_tokens_with_escaped_chars",
rules: {
first_rule: $ => choice(
"\n",
"\r\n",
"'hello'",
/\d+/,
)
}
})

14
test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.json vendored
View file

@ -1,14 +0,0 @@
{
"name": "anonymous_tokens_with_escaped_chars",
"rules": {
"first_rule": {
"type": "CHOICE",
"members": [
{"type": "STRING", "value": "\n"},
{"type": "STRING", "value": "\r\n"},
{"type": "STRING", "value": "'hello'"},
{"type": "PATTERN", "value": "\\d+"}
]
}
}
}

1
test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/readme.md vendored
View file

@ -1 +0,0 @@
Every token in a grammar is given a name in the generated parser. Anonymous tokens (tokens specified directly in the body of some larger rule) are named according their content. So when tokens contains characters that aren't valid in a C string literal, we need to escape those characters. This grammar tests that this escaping works. The test is basically that the generated parser compiles succesfully.

18
test/fixtures/test_grammars/associativity_left/grammar.js vendored Normal file
View file

@ -0,0 +1,18 @@
module.exports = grammar({
name: 'associativity_left',
rules: {
expression: $ => choice(
$.math_operation,
$.identifier
),
math_operation: $ => prec.left(seq(
$.expression,
'+',
$.expression,
)),
identifier: $ => /[a-z]+/,
}
});

31
test/fixtures/test_grammars/associativity_left/grammar.json vendored
View file

@ -1,31 +0,0 @@
{
"name": "associativity_left",
"rules": {
"expression": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "math_operation"},
{"type": "SYMBOL", "name": "identifier"}
]
},
"math_operation": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "+"},
{"type": "SYMBOL", "name": "expression"}
]
}
},
"identifier": {
"type": "PATTERN",
"value": "[a-zA-Z]+"
}
}
}

18
test/fixtures/test_grammars/associativity_missing/grammar.js vendored Normal file
View file

@ -0,0 +1,18 @@
module.exports = grammar({
name: 'associativity_missing',
rules: {
expression: $ => choice(
$.math_operation,
$.identifier
),
math_operation: $ => seq(
$.expression,
'+',
$.expression,
),
identifier: $ => /[a-z]+/,
}
});

27
test/fixtures/test_grammars/associativity_missing/grammar.json vendored
View file

@ -1,27 +0,0 @@
{
"name": "associativity_missing",
"rules": {
"expression": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "math_operation"},
{"type": "SYMBOL", "name": "identifier"}
]
},
"math_operation": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "+"},
{"type": "SYMBOL", "name": "expression"}
]
},
"identifier": {
"type": "PATTERN",
"value": "[a-zA-Z]+"
}
}
}

18
test/fixtures/test_grammars/associativity_right/grammar.js vendored Normal file
View file

@ -0,0 +1,18 @@
module.exports = grammar({
name: 'associativity_right',
rules: {
expression: $ => choice(
$.math_operation,
$.identifier
),
math_operation: $ => prec.right(seq(
$.expression,
'+',
$.expression,
)),
identifier: $ => /[a-z]+/,
}
});

31
test/fixtures/test_grammars/associativity_right/grammar.json vendored
View file

@ -1,31 +0,0 @@
{
"name": "associativity_right",
"rules": {
"expression": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "math_operation"},
{"type": "SYMBOL", "name": "identifier"}
]
},
"math_operation": {
"type": "PREC_RIGHT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "+"},
{"type": "SYMBOL", "name": "expression"}
]
}
},
"identifier": {
"type": "PATTERN",
"value": "[a-zA-Z]+"
}
}
}

28
test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.js vendored Normal file
View file

@ -0,0 +1,28 @@
// This grammar has a conflict that involves *repeat rules*: auxiliary rules that are added by the
// parser generator in order to implement repetition. There is no way of referring to these rules in
// the grammar DSL, so these conflicts must be resolved by referring to their parent rules.
module.exports = grammar({
name: 'conflict_in_repeat_rule',
rules: {
statement: $ => choice(
seq($.array, ';'),
seq($.array_type, $.identifier, ';'),
),
array: $ => seq(
'[',
repeat(choice($.identifier, '0')),
']',
),
array_type: $ => seq(
'[',
repeat(choice($.identifier, 'void')),
']',
),
identifier: $ => /[a-z]+/
}
});

76
test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.json vendored
View file

@ -1,76 +0,0 @@
{
"name": "conflict_in_repeat_rule",
"rules": {
"statement": {
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "array"},
{"type": "STRING", "value": ";"}
]
},
{
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "array_type"},
{"type": "SYMBOL", "name": "identifier"},
{"type": "STRING", "value": ";"}
]
}
]
},
"array": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "["
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "identifier"},
{"type": "STRING", "value": "0"}
]
}
},
{
"type": "STRING",
"value": "]"
}
]
},
"array_type": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "["
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "identifier"},
{"type": "STRING", "value": "void"}
]
}
},
{
"type": "STRING",
"value": "]"
}
]
},
"identifier": {"type": "PATTERN", "value": "[a-z]+"}
}
}

1
test/fixtures/test_grammars/conflict_in_repeat_rule/readme.md vendored
View file

@ -1 +0,0 @@
This grammar has a conflict that involves *repeat rules*: auxiliary rules that are added by the parser generator in order to implement repetition. There is no way of referring to these rules in the grammar DSL, so these conflicts must be resolved by referring to their parent rules.

32
test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.js vendored Normal file
View file

@ -0,0 +1,32 @@
// This grammar is similar to the `conflict_in_repeat_rule` grammar, except that the conflict occurs
// after an external token is consumed. This tests that the logic for determining the repeat rule's
// "parent" rule works in the presence of external tokens.
module.exports = grammar({
name: 'conflict_in_repeat_rule_after_external_token',
externals: $ => [
$._program_start,
],
rules: {
statement: $ => choice(
seq($._program_start, $.array, ';'),
seq($._program_start, $.array_type, $.identifier, ';'),
),
array: $ => seq(
'[',
repeat(choice($.identifier, '0')),
']',
),
array_type: $ => seq(
'[',
repeat(choice($.identifier, 'void')),
']',
),
identifier: $ => /[a-z]+/
}
});

82
test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.json vendored
View file

@ -1,82 +0,0 @@
{
"name": "conflict_in_repeat_rule_after_external_token",
"externals": [
{"type": "SYMBOL", "name": "_program_start"}
],
"rules": {
"statement": {
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "_program_start"},
{"type": "SYMBOL", "name": "array"},
{"type": "STRING", "value": ";"}
]
},
{
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "_program_start"},
{"type": "SYMBOL", "name": "array_type"},
{"type": "SYMBOL", "name": "identifier"},
{"type": "STRING", "value": ";"}
]
}
]
},
"array": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "["
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "identifier"},
{"type": "STRING", "value": "0"}
]
}
},
{
"type": "STRING",
"value": "]"
}
]
},
"array_type": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "["
},
{
"type": "REPEAT",
"content": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "identifier"},
{"type": "STRING", "value": "void"}
]
}
},
{
"type": "STRING",
"value": "]"
}
]
},
"identifier": {"type": "PATTERN", "value": "[a-z]+"}
}
}

1
test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/readme.md vendored
View file

@ -1 +0,0 @@
This grammar is similar to the `conflict_in_repeat_rule` grammar, except that the conflict occurs after an external token is consumed. This tests that the logic for determining the repeat rule's "parent" rule works in the presence of external tokens.

16
test/fixtures/test_grammars/conflicting_precedence/grammar.js vendored Normal file
View file

@ -0,0 +1,16 @@
module.exports = grammar({
name: 'conflicting_precedence',
rules: {
expression: $ => choice(
$.sum,
$.product,
$.other_thing,
),
sum: $ => prec.left(0, seq($.expression, '+', $.expression)),
product: $ => prec.left(1, seq($.expression, '*', $.expression)),
other_thing: $ => prec.left(-1, seq($.expression, '*', '*')),
identifier: $ => /[a-zA-Z]+/
}
});

58
test/fixtures/test_grammars/conflicting_precedence/grammar.json vendored
View file

@ -1,58 +0,0 @@
{
"name": "conflicting_precedence",
"rules": {
"expression": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "sum"},
{"type": "SYMBOL", "name": "product"},
{"type": "SYMBOL", "name": "other_thing"}
]
},
"sum": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "+"},
{"type": "SYMBOL", "name": "expression"}
]
}
},
"product": {
"type": "PREC_LEFT",
"value": 1,
"content": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "*"},
{"type": "SYMBOL", "name": "expression"}
]
}
},
"other_thing": {
"type": "PREC_LEFT",
"value": -1,
"content": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "*"},
{"type": "STRING", "value": "*"}
]
}
},
"identifier": {
"type": "PATTERN",
"value": "[a-zA-Z]+"
}
}
}

4
test/fixtures/test_grammars/dynamic_precedence/corpus.txt vendored
View file

@ -2,7 +2,7 @@
Declarations
===============================
int * x
T * x
---
@ -14,7 +14,7 @@ int * x
Expressions
===============================
int * x * y
w * x * y
---

32
test/fixtures/test_grammars/dynamic_precedence/grammar.js vendored Normal file
View file

@ -0,0 +1,32 @@
module.exports = grammar({
name: 'dynamic_precedence',
extras: $ => [/\s/],
conflicts: $ => [[$.expression, $.type]],
rules: {
program: $ => choice(
$.declaration,
$.expression,
),
expression: $ => choice(
prec.left(seq($.expression, '*', $.expression)),
$.identifier
),
declaration: $ => seq(
$.type,
$.declarator,
),
declarator: $ => choice(
prec.dynamic(1, seq('*', $.identifier)),
$.identifier,
),
type: $ => $.identifier,
identifier: $ => /[a-z-A-Z]+/
}
});

73
test/fixtures/test_grammars/dynamic_precedence/grammar.json vendored
View file

@ -1,73 +0,0 @@
{
"name": "dynamic_precedence",
"conflicts": [
["expression", "type"]
],
"extras": [
{"type": "PATTERN", "value": "\\s"}
],
"rules": {
"program": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "declaration"},
{"type": "SYMBOL", "name": "expression"}
]
},
"expression": {
"type": "PREC_LEFT",
"value": 0,
"content": {
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "expression"},
{"type": "STRING", "value": "*"},
{"type": "SYMBOL", "name": "expression"}
]
},
{
"type": "SYMBOL",
"name": "identifier"
}
]
}
},
"declaration": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "type"},
{"type": "SYMBOL", "name": "declarator"}
]
},
"declarator": {
"type": "PREC_DYNAMIC",
"value": 1,
"content": {
"type": "SEQ",
"members": [
{"type": "STRING", "value": "*"},
{"type": "SYMBOL", "name": "identifier"}
]
}
},
"type": {
"type": "SYMBOL",
"name": "identifier"
},
"identifier": {
"type": "PATTERN",
"value": "[a-zA-Z]+"
}
}
}

10
test/fixtures/test_grammars/epsilon_external_tokens/grammar.js vendored Normal file
View file

@ -0,0 +1,10 @@
module.exports = grammar({
name: 'epsilon_external_tokens',
extras: $ => [/\s/],
externals: $ => [$.zero_width],
rules: {
document: $ => seq($.zero_width, 'hello'),
}
});

21
test/fixtures/test_grammars/epsilon_external_tokens/grammar.json vendored
View file

@ -1,21 +0,0 @@
{
"name": "epsilon_external_tokens",
"externals": [
{"type": "SYMBOL", "name": "zero_width"}
],
"extras": [
{"type": "PATTERN", "value": "\\s"}
],
"rules": {
"document": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "zero_width"},
{"type": "STRING", "value": "hello"}
]
}
}
}

11
test/fixtures/test_grammars/epsilon_rules/grammar.js vendored Normal file
View file

@ -0,0 +1,11 @@
module.exports = grammar({
name: 'epsilon_rules',
rules: {
rule_1: $ => $.rule_2,
rule_2: $ => optional($.rule_3),
rule_3: $ => 'x'
}
});

15
test/fixtures/test_grammars/epsilon_rules/grammar.json vendored
View file

@ -1,15 +0,0 @@
{
"name": "epsilon_rules",
"rules": {
"rule_1": {"type": "SYMBOL", "name": "rule_2"},
"rule_2": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "rule_1"},
{"type": "BLANK"}
]
}
}
}

28
test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.js vendored Normal file
View file

@ -0,0 +1,28 @@
module.exports = grammar({
name: 'external_and_internal_anonymous_tokens',
externals: $ => [
$.string,
'\n'
],
extras: $ => [/\s/],
rules: {
statement: $ => seq(
$._expression,
$._expression,
'\n'
),
_expression: $ => choice(
$.string,
$.variable,
$.number
),
variable: $ => /[a-z]+/,
number: $ => /\d+/
}
})

35
test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.json vendored
View file

@ -1,35 +0,0 @@
{
"name": "external_and_internal_anonymous_tokens",
"externals": [
{"type": "SYMBOL", "name": "string"},
{"type": "STRING", "value": "\n"}
],
"extras": [
{"type": "PATTERN", "value": "\\s"}
],
"rules": {
"statement": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "_expression"},
{"type": "SYMBOL", "name": "_expression"},
{"type": "STRING", "value": "\n"}
]
},
"_expression": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "string"},
{"type": "SYMBOL", "name": "variable"},
{"type": "SYMBOL", "name": "number"}
]
},
"variable": {"type": "PATTERN", "value": "[a-z]+"},
"number": {"type": "PATTERN", "value": "\\d+"}
}
}

32
test/fixtures/test_grammars/external_and_internal_tokens/grammar.js vendored Normal file
View file

@ -0,0 +1,32 @@
// This grammar has an external scanner whose `scan` method needs to be able to check for the
// validity of an *internal* token. This is done by including the names of that internal token
// (`line_break`) in the grammar's `externals` field.
module.exports = grammar({
name: 'external_and_internal_tokens',
externals: $ => [
$.string,
$.line_break,
],
extras: $ => [/\s/],
rules: {
statement: $ => seq(
$._expression,
$._expression,
$.line_break,
),
_expression: $ => choice(
$.string,
$.variable,
$.number,
),
variable: $ => /[a-z]+/,
number: $ => /\d+/,
line_break: $ => '\n',
}
});

36
test/fixtures/test_grammars/external_and_internal_tokens/grammar.json vendored
View file

@ -1,36 +0,0 @@
{
"name": "external_and_internal_tokens",
"externals": [
{"type": "SYMBOL", "name": "string"},
{"type": "SYMBOL", "name": "line_break"}
],
"extras": [
{"type": "PATTERN", "value": "\\s"}
],
"rules": {
"statement": {
"type": "SEQ",
"members": [
{"type": "SYMBOL", "name": "_expression"},
{"type": "SYMBOL", "name": "_expression"},
{"type": "SYMBOL", "name": "line_break"}
]
},
"_expression": {
"type": "CHOICE",
"members": [
{"type": "SYMBOL", "name": "string"},
{"type": "SYMBOL", "name": "variable"},
{"type": "SYMBOL", "name": "number"}
]
},
"variable": {"type": "PATTERN", "value": "[a-z]+"},
"number": {"type": "PATTERN", "value": "\\d+"},
"line_break": {"type": "STRING", "value": "\n"}
}
}

1
test/fixtures/test_grammars/external_and_internal_tokens/readme.md vendored
View file

@ -1 +0,0 @@
This grammar has an external scanner whose `scan` method needs to be able to check for the validity of an *internal* token. This is done by including the names of that internal token (`_line_break`) in the grammar's `externals` field.