From d05c665863e506c7494af378f2191b68f76ad86b Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 22 Oct 2021 18:47:13 -0600 Subject: [PATCH] Convert some of the fixture grammars from JSON to JS These tests are easier to write and maintain if the grammars are just JS, like grammars normally are. It doesn't slow the tests down significantly to shell out to `node` for each of these grammars. --- cli/src/generate/mod.rs | 2 +- cli/src/tests/corpus_test.rs | 7 +- .../aliased_inlined_rules/corpus.txt | 11 +-- .../aliased_inlined_rules/grammar.js | 28 +++++++ .../aliased_inlined_rules/grammar.json | 59 ------------- .../aliased_inlined_rules/readme.md | 1 - .../test_grammars/aliased_rules/grammar.js | 30 +++++++ .../test_grammars/aliased_rules/grammar.json | 71 ---------------- .../aliased_token_rules/grammar.js | 16 ++++ .../aliased_token_rules/grammar.json | 61 -------------- .../aliased_token_rules/readme.md | 1 - .../aliased_unit_reductions/grammar.js | 38 +++++++++ .../aliased_unit_reductions/grammar.json | 81 ------------------ .../aliased_unit_reductions/readme.md | 5 -- .../grammar.js | 17 ++++ .../grammar.json | 14 ---- .../readme.md | 1 - .../associativity_left/grammar.js | 18 ++++ .../associativity_left/grammar.json | 31 ------- .../associativity_missing/grammar.js | 18 ++++ .../associativity_missing/grammar.json | 27 ------ .../associativity_right/grammar.js | 18 ++++ .../associativity_right/grammar.json | 31 ------- .../conflict_in_repeat_rule/grammar.js | 28 +++++++ .../conflict_in_repeat_rule/grammar.json | 76 ----------------- .../conflict_in_repeat_rule/readme.md | 1 - .../grammar.js | 32 ++++++++ .../grammar.json | 82 ------------------- .../readme.md | 1 - .../conflicting_precedence/grammar.js | 16 ++++ .../conflicting_precedence/grammar.json | 58 ------------- .../dynamic_precedence/corpus.txt | 4 +- .../dynamic_precedence/grammar.js | 32 ++++++++ .../dynamic_precedence/grammar.json | 73 ----------------- .../epsilon_external_tokens/grammar.js | 10 +++ .../epsilon_external_tokens/grammar.json | 21 ----- .../test_grammars/epsilon_rules/grammar.js | 11 +++ .../test_grammars/epsilon_rules/grammar.json | 15 ---- .../grammar.js | 28 +++++++ .../grammar.json | 35 -------- .../external_and_internal_tokens/grammar.js | 32 ++++++++ .../external_and_internal_tokens/grammar.json | 36 -------- .../external_and_internal_tokens/readme.md | 1 - 43 files changed, 386 insertions(+), 792 deletions(-) create mode 100644 test/fixtures/test_grammars/aliased_inlined_rules/grammar.js delete mode 100644 test/fixtures/test_grammars/aliased_inlined_rules/grammar.json delete mode 100644 test/fixtures/test_grammars/aliased_inlined_rules/readme.md create mode 100644 test/fixtures/test_grammars/aliased_rules/grammar.js delete mode 100644 test/fixtures/test_grammars/aliased_rules/grammar.json create mode 100644 test/fixtures/test_grammars/aliased_token_rules/grammar.js delete mode 100644 test/fixtures/test_grammars/aliased_token_rules/grammar.json delete mode 100644 test/fixtures/test_grammars/aliased_token_rules/readme.md create mode 100644 test/fixtures/test_grammars/aliased_unit_reductions/grammar.js delete mode 100644 test/fixtures/test_grammars/aliased_unit_reductions/grammar.json delete mode 100644 test/fixtures/test_grammars/aliased_unit_reductions/readme.md create mode 100644 test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.js delete mode 100644 test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.json delete mode 100644 test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/readme.md create mode 100644 test/fixtures/test_grammars/associativity_left/grammar.js delete mode 100644 test/fixtures/test_grammars/associativity_left/grammar.json create mode 100644 test/fixtures/test_grammars/associativity_missing/grammar.js delete mode 100644 test/fixtures/test_grammars/associativity_missing/grammar.json create mode 100644 test/fixtures/test_grammars/associativity_right/grammar.js delete mode 100644 test/fixtures/test_grammars/associativity_right/grammar.json create mode 100644 test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.js delete mode 100644 test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.json delete mode 100644 test/fixtures/test_grammars/conflict_in_repeat_rule/readme.md create mode 100644 test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.js delete mode 100644 test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.json delete mode 100644 test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/readme.md create mode 100644 test/fixtures/test_grammars/conflicting_precedence/grammar.js delete mode 100644 test/fixtures/test_grammars/conflicting_precedence/grammar.json create mode 100644 test/fixtures/test_grammars/dynamic_precedence/grammar.js delete mode 100644 test/fixtures/test_grammars/dynamic_precedence/grammar.json create mode 100644 test/fixtures/test_grammars/epsilon_external_tokens/grammar.js delete mode 100644 test/fixtures/test_grammars/epsilon_external_tokens/grammar.json create mode 100644 test/fixtures/test_grammars/epsilon_rules/grammar.js delete mode 100644 test/fixtures/test_grammars/epsilon_rules/grammar.json create mode 100644 test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.js delete mode 100644 test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.json create mode 100644 test/fixtures/test_grammars/external_and_internal_tokens/grammar.js delete mode 100644 test/fixtures/test_grammars/external_and_internal_tokens/grammar.json diff --git a/cli/src/generate/mod.rs b/cli/src/generate/mod.rs index 141fdff0..d61a63f0 100644 --- a/cli/src/generate/mod.rs +++ b/cli/src/generate/mod.rs @@ -157,7 +157,7 @@ fn generate_parser_for_grammar_with_opts( }) } -fn load_grammar_file(grammar_path: &Path) -> Result { +pub fn load_grammar_file(grammar_path: &Path) -> Result { match grammar_path.extension().and_then(|e| e.to_str()) { Some("js") => Ok(load_js_grammar_file(grammar_path)?), Some("json") => Ok(fs::read_to_string(grammar_path)?), diff --git a/cli/src/tests/corpus_test.rs b/cli/src/tests/corpus_test.rs index 700e62a2..11c9992b 100644 --- a/cli/src/tests/corpus_test.rs +++ b/cli/src/tests/corpus_test.rs @@ -269,9 +269,12 @@ fn test_feature_corpus_files() { } let test_path = entry.path(); - let grammar_path = test_path.join("grammar.json"); + let mut grammar_path = test_path.join("grammar.js"); + if !grammar_path.exists() { + grammar_path = test_path.join("grammar.json"); + } let error_message_path = test_path.join("expected_error.txt"); - let grammar_json = fs::read_to_string(grammar_path).unwrap(); + let grammar_json = generate::load_grammar_file(&grammar_path).unwrap(); let generate_result = generate::generate_parser_for_grammar(&grammar_json); if error_message_path.exists() { diff --git a/test/fixtures/test_grammars/aliased_inlined_rules/corpus.txt b/test/fixtures/test_grammars/aliased_inlined_rules/corpus.txt index 49a91158..cffcbb86 100644 --- a/test/fixtures/test_grammars/aliased_inlined_rules/corpus.txt +++ b/test/fixtures/test_grammars/aliased_inlined_rules/corpus.txt @@ -2,12 +2,13 @@ OK ========================= -a.b.c +a.b.c; --- -(expression (member_expression - (expression (member_expression - (expression (variable_name)) +(statement + (member_expression + (member_expression + (variable_name) + (property_name)) (property_name))) - (property_name))) diff --git a/test/fixtures/test_grammars/aliased_inlined_rules/grammar.js b/test/fixtures/test_grammars/aliased_inlined_rules/grammar.js new file mode 100644 index 00000000..2f1091e7 --- /dev/null +++ b/test/fixtures/test_grammars/aliased_inlined_rules/grammar.js @@ -0,0 +1,28 @@ +// This grammar shows that `ALIAS` rules can *contain* a rule that is marked as `inline`. It also +// shows that you can alias a rule that would otherwise be anonymous, and it will then appear as a +// named node. + +module.exports = grammar({ + name: 'aliased_inlined_rules', + + extras: $ => [/\s/], + + inline: $ => [$.identifier], + + rules: { + statement: $ => seq($._expression, ';'), + + _expression: $ => choice( + $.member_expression, + alias($.identifier, $.variable_name), + ), + + member_expression: $ => prec.left(1, seq( + $._expression, + '.', + alias($.identifier, $.property_name) + )), + + identifier: $ => choice('a', 'b', 'c') + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/aliased_inlined_rules/grammar.json b/test/fixtures/test_grammars/aliased_inlined_rules/grammar.json deleted file mode 100644 index 02d76219..00000000 --- a/test/fixtures/test_grammars/aliased_inlined_rules/grammar.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "name": "aliased_inlined_rules", - - "extras": [ - {"type": "PATTERN", "value": "\\s"} - ], - - "inline": [ - "identifier" - ], - - "rules": { - "expression": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "member_expression"}, - { - "type": "ALIAS", - "value": "variable_name", - "named": true, - "content": { - "type": "SYMBOL", - "name": "identifier" - } - } - ] - }, - - "member_expression": { - "type": "PREC_LEFT", - "value": 1, - "content": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "expression"}, - {"type": "STRING", "value": "."}, - { - "type": "ALIAS", - "value": "property_name", - "named": true, - "content": { - "type": "SYMBOL", - "name": "identifier" - } - } - ] - } - }, - - "identifier": { - "type": "CHOICE", - "members": [ - {"type": "STRING", "value": "a"}, - {"type": "STRING", "value": "b"}, - {"type": "STRING", "value": "c"} - ] - } - } -} diff --git a/test/fixtures/test_grammars/aliased_inlined_rules/readme.md b/test/fixtures/test_grammars/aliased_inlined_rules/readme.md deleted file mode 100644 index 9775c62b..00000000 --- a/test/fixtures/test_grammars/aliased_inlined_rules/readme.md +++ /dev/null @@ -1 +0,0 @@ -This grammar shows that `ALIAS` rules can *contain* a rule that is marked as `inline`. It also shows that you can alias a rule that would otherwise be anonymous, and it will then appear as a named node. diff --git a/test/fixtures/test_grammars/aliased_rules/grammar.js b/test/fixtures/test_grammars/aliased_rules/grammar.js new file mode 100644 index 00000000..c3cb15cb --- /dev/null +++ b/test/fixtures/test_grammars/aliased_rules/grammar.js @@ -0,0 +1,30 @@ +module.exports = grammar({ + name: 'aliased_rules', + + extras: $ => [/\s/], + + rules: { + statement: $ => seq($._expression, ';'), + + _expression: $ => choice( + $.call_expression, + $.member_expression, + alias($.identifier, $.variable_name), + ), + + call_expression: $ => prec.left(seq( + $._expression, + '(', + $._expression, + ')' + )), + + member_expression: $ => prec.left(1, seq( + $._expression, + '.', + alias($.identifier, $.property_name) + )), + + identifier: $ => /[a-z]+/ + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/aliased_rules/grammar.json b/test/fixtures/test_grammars/aliased_rules/grammar.json deleted file mode 100644 index a66bfb78..00000000 --- a/test/fixtures/test_grammars/aliased_rules/grammar.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "name": "aliased_rules", - - "extras": [ - {"type": "PATTERN", "value": "\\s"} - ], - - "rules": { - "statement": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "_expression"}, - {"type": "STRING", "value": ";"} - ] - }, - - "_expression": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "call_expression"}, - {"type": "SYMBOL", "name": "member_expression"}, - { - "type": "ALIAS", - "named": true, - "value": "variable_name", - "content": { - "type": "SYMBOL", - "name": "identifier" - } - } - ] - }, - - "call_expression": { - "type": "PREC_LEFT", - "value": 0, - "content": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "_expression"}, - {"type": "STRING", "value": "("}, - {"type": "SYMBOL", "name": "_expression"}, - {"type": "STRING", "value": ")"} - ] - } - }, - - "member_expression": { - "type": "PREC_LEFT", - "value": 1, - "content": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "_expression"}, - {"type": "STRING", "value": "."}, - { - "type": "ALIAS", - "named": true, - "value": "property_name", - "content": { - "type": "SYMBOL", - "name": "identifier" - } - } - ] - } - }, - - "identifier": {"type": "PATTERN", "value": "[a-z]+"} - } -} diff --git a/test/fixtures/test_grammars/aliased_token_rules/grammar.js b/test/fixtures/test_grammars/aliased_token_rules/grammar.js new file mode 100644 index 00000000..704a2a34 --- /dev/null +++ b/test/fixtures/test_grammars/aliased_token_rules/grammar.js @@ -0,0 +1,16 @@ +// This grammar shows that `ALIAS` rules can be applied directly to `TOKEN` and `IMMEDIATE_TOKEN` +// rules. + +module.exports = grammar({ + name: 'aliased_token_rules', + + extras: $ => [/\s/], + + rules: { + expression: $ => seq( + 'a', + alias(token(seq('b', 'c')), $.X), + alias(token.immediate(seq('d', 'e')), $.Y), + ), + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/aliased_token_rules/grammar.json b/test/fixtures/test_grammars/aliased_token_rules/grammar.json deleted file mode 100644 index 76df46eb..00000000 --- a/test/fixtures/test_grammars/aliased_token_rules/grammar.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "name": "aliased_token_rules", - - "extras": [ - {"type": "PATTERN", "value": "\\s"} - ], - - "rules": { - "expression": { - "type": "SEQ", - "members": [ - { - "type": "STRING", - "value": "a" - }, - { - "type": "ALIAS", - "value": "X", - "named": true, - "content": { - "type": "TOKEN", - "content": { - "type": "SEQ", - "members": [ - { - "type": "STRING", - "value": "b" - }, - { - "type": "STRING", - "value": "c" - } - ] - } - } - }, - { - "type": "ALIAS", - "value": "Y", - "named": true, - "content": { - "type": "IMMEDIATE_TOKEN", - "content": { - "type": "SEQ", - "members": [ - { - "type": "STRING", - "value": "d" - }, - { - "type": "STRING", - "value": "e" - } - ] - } - } - } - ] - } - } -} diff --git a/test/fixtures/test_grammars/aliased_token_rules/readme.md b/test/fixtures/test_grammars/aliased_token_rules/readme.md deleted file mode 100644 index 03898a5f..00000000 --- a/test/fixtures/test_grammars/aliased_token_rules/readme.md +++ /dev/null @@ -1 +0,0 @@ -This grammar shows that `ALIAS` rules can be applied directly to `TOKEN` and `IMMEDIATE_TOKEN` rules. diff --git a/test/fixtures/test_grammars/aliased_unit_reductions/grammar.js b/test/fixtures/test_grammars/aliased_unit_reductions/grammar.js new file mode 100644 index 00000000..9b39de28 --- /dev/null +++ b/test/fixtures/test_grammars/aliased_unit_reductions/grammar.js @@ -0,0 +1,38 @@ +// Normally, when there are invisible rules (rules whose names start with an `_`) that simply wrap +// another rule, there is an optimization at parser-generation time called *Unit Reduction +// Elimination* that avoids creating nodes for those rules at runtime. One case where this +// optimization must *not* be applied is when those invisible rules are going to be aliased within +// their parent rule. In that situation, eliminating the invisible node could cause the alias to be +// incorrectly applied to its child. + +module.exports = grammar({ + name: 'aliased_unit_reductions', + + extras: $ => [/\s/], + + rules: { + statement: $ => seq( + $._a, + + // The `_b` rule is always aliased to `b_prime`, so it is internally treated + // as a simple alias. + alias($._b, $.b_prime), + + // The `_c` rule is used without an alias in addition to being aliased to `c_prime`, + // so it is not a simple alias. + alias($._c, $.c_prime), + + $._c, + ';' + ), + + _a: $ => $._A, + _b: $ => $._B, + _c: $ => $._C, + _A: $ => $.identifier, + _B: $ => $.identifier, + _C: $ => $.identifier, + + identifier: $ => /[a-z]+/, + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/aliased_unit_reductions/grammar.json b/test/fixtures/test_grammars/aliased_unit_reductions/grammar.json deleted file mode 100644 index 34080b7e..00000000 --- a/test/fixtures/test_grammars/aliased_unit_reductions/grammar.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "name": "aliased_unit_reductions", - - "extras": [ - {"type": "PATTERN", "value": "\\s"} - ], - - "rules": { - "statement": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "_a"}, - - // The `_b` rule is always aliased to `b_prime`, so it is internally treated - // as a simple alias. - { - "type": "ALIAS", - "named": true, - "value": "b_prime", - "content": { - "type": "SYMBOL", - "name": "_b" - } - }, - - // The `_c` rule is used without an alias in addition to being aliased to `c_prime`, - // so it is not a simple alias. - { - "type": "ALIAS", - "named": true, - "value": "c_prime", - "content": { - "type": "SYMBOL", - "name": "_c" - } - }, - { - "type": "SYMBOL", - "name": "_c" - }, - - { - "type": "STRING", - "value": ";" - } - ] - }, - - "_a": { - "type": "SYMBOL", - "name": "_A" - }, - - "_A": { - "type": "SYMBOL", - "name": "identifier" - }, - - "_b": { - "type": "SYMBOL", - "name": "_B" - }, - - "_B": { - "type": "SYMBOL", - "name": "identifier" - }, - - "_c": { - "type": "SYMBOL", - "name": "_C" - }, - - "_C": { - "type": "SYMBOL", - "name": "identifier" - }, - - "identifier": {"type": "PATTERN", "value": "[a-z]+"} - } -} diff --git a/test/fixtures/test_grammars/aliased_unit_reductions/readme.md b/test/fixtures/test_grammars/aliased_unit_reductions/readme.md deleted file mode 100644 index 4e193713..00000000 --- a/test/fixtures/test_grammars/aliased_unit_reductions/readme.md +++ /dev/null @@ -1,5 +0,0 @@ -Normally, when there are invisible rules (rules whose names start with an `_`) that simply -wrap another rule, there is an optimization at parser-generation time called *Unit Reduction Elimination* that avoids creating nodes for those rules at runtime. One case where this -optimization must *not* be applied is when those invisible rules are going to be aliased -within their parent rule. In that situation, eliminating the invisible node could cause -the alias to be incorrectly applied to its child. diff --git a/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.js b/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.js new file mode 100644 index 00000000..45b52968 --- /dev/null +++ b/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.js @@ -0,0 +1,17 @@ +// Every token in a grammar is given a name in the generated parser. Anonymous tokens (tokens +// specified directly in the body of some larger rule) are named according their content. So when +// tokens contains characters that aren't valid in a C string literal, we need to escape those +// characters. This grammar tests that this escaping works. The test is basically that the generated +// parser compiles succesfully. + +module.exports = grammar({ + name: "anonymous_tokens_with_escaped_chars", + rules: { + first_rule: $ => choice( + "\n", + "\r\n", + "'hello'", + /\d+/, + ) + } +}) diff --git a/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.json b/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.json deleted file mode 100644 index 38ada64c..00000000 --- a/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/grammar.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "anonymous_tokens_with_escaped_chars", - "rules": { - "first_rule": { - "type": "CHOICE", - "members": [ - {"type": "STRING", "value": "\n"}, - {"type": "STRING", "value": "\r\n"}, - {"type": "STRING", "value": "'hello'"}, - {"type": "PATTERN", "value": "\\d+"} - ] - } - } -} diff --git a/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/readme.md b/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/readme.md deleted file mode 100644 index 96233364..00000000 --- a/test/fixtures/test_grammars/anonymous_tokens_with_escaped_chars/readme.md +++ /dev/null @@ -1 +0,0 @@ -Every token in a grammar is given a name in the generated parser. Anonymous tokens (tokens specified directly in the body of some larger rule) are named according their content. So when tokens contains characters that aren't valid in a C string literal, we need to escape those characters. This grammar tests that this escaping works. The test is basically that the generated parser compiles succesfully. diff --git a/test/fixtures/test_grammars/associativity_left/grammar.js b/test/fixtures/test_grammars/associativity_left/grammar.js new file mode 100644 index 00000000..6dbc4671 --- /dev/null +++ b/test/fixtures/test_grammars/associativity_left/grammar.js @@ -0,0 +1,18 @@ +module.exports = grammar({ + name: 'associativity_left', + + rules: { + expression: $ => choice( + $.math_operation, + $.identifier + ), + + math_operation: $ => prec.left(seq( + $.expression, + '+', + $.expression, + )), + + identifier: $ => /[a-z]+/, + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/associativity_left/grammar.json b/test/fixtures/test_grammars/associativity_left/grammar.json deleted file mode 100644 index b1a25914..00000000 --- a/test/fixtures/test_grammars/associativity_left/grammar.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "associativity_left", - - "rules": { - "expression": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "math_operation"}, - {"type": "SYMBOL", "name": "identifier"} - ] - }, - - "math_operation": { - "type": "PREC_LEFT", - "value": 0, - "content": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "expression"}, - {"type": "STRING", "value": "+"}, - {"type": "SYMBOL", "name": "expression"} - ] - } - }, - - "identifier": { - "type": "PATTERN", - "value": "[a-zA-Z]+" - } - } -} \ No newline at end of file diff --git a/test/fixtures/test_grammars/associativity_missing/grammar.js b/test/fixtures/test_grammars/associativity_missing/grammar.js new file mode 100644 index 00000000..9c1ed980 --- /dev/null +++ b/test/fixtures/test_grammars/associativity_missing/grammar.js @@ -0,0 +1,18 @@ +module.exports = grammar({ + name: 'associativity_missing', + + rules: { + expression: $ => choice( + $.math_operation, + $.identifier + ), + + math_operation: $ => seq( + $.expression, + '+', + $.expression, + ), + + identifier: $ => /[a-z]+/, + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/associativity_missing/grammar.json b/test/fixtures/test_grammars/associativity_missing/grammar.json deleted file mode 100644 index e5bd9d83..00000000 --- a/test/fixtures/test_grammars/associativity_missing/grammar.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "associativity_missing", - - "rules": { - "expression": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "math_operation"}, - {"type": "SYMBOL", "name": "identifier"} - ] - }, - - "math_operation": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "expression"}, - {"type": "STRING", "value": "+"}, - {"type": "SYMBOL", "name": "expression"} - ] - }, - - "identifier": { - "type": "PATTERN", - "value": "[a-zA-Z]+" - } - } -} \ No newline at end of file diff --git a/test/fixtures/test_grammars/associativity_right/grammar.js b/test/fixtures/test_grammars/associativity_right/grammar.js new file mode 100644 index 00000000..69bfd065 --- /dev/null +++ b/test/fixtures/test_grammars/associativity_right/grammar.js @@ -0,0 +1,18 @@ +module.exports = grammar({ + name: 'associativity_right', + + rules: { + expression: $ => choice( + $.math_operation, + $.identifier + ), + + math_operation: $ => prec.right(seq( + $.expression, + '+', + $.expression, + )), + + identifier: $ => /[a-z]+/, + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/associativity_right/grammar.json b/test/fixtures/test_grammars/associativity_right/grammar.json deleted file mode 100644 index 80ce1ebb..00000000 --- a/test/fixtures/test_grammars/associativity_right/grammar.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "associativity_right", - - "rules": { - "expression": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "math_operation"}, - {"type": "SYMBOL", "name": "identifier"} - ] - }, - - "math_operation": { - "type": "PREC_RIGHT", - "value": 0, - "content": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "expression"}, - {"type": "STRING", "value": "+"}, - {"type": "SYMBOL", "name": "expression"} - ] - } - }, - - "identifier": { - "type": "PATTERN", - "value": "[a-zA-Z]+" - } - } -} \ No newline at end of file diff --git a/test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.js b/test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.js new file mode 100644 index 00000000..c23e8a7c --- /dev/null +++ b/test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.js @@ -0,0 +1,28 @@ +// This grammar has a conflict that involves *repeat rules*: auxiliary rules that are added by the +// parser generator in order to implement repetition. There is no way of referring to these rules in +// the grammar DSL, so these conflicts must be resolved by referring to their parent rules. + +module.exports = grammar({ + name: 'conflict_in_repeat_rule', + + rules: { + statement: $ => choice( + seq($.array, ';'), + seq($.array_type, $.identifier, ';'), + ), + + array: $ => seq( + '[', + repeat(choice($.identifier, '0')), + ']', + ), + + array_type: $ => seq( + '[', + repeat(choice($.identifier, 'void')), + ']', + ), + + identifier: $ => /[a-z]+/ + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.json b/test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.json deleted file mode 100644 index 45d0922a..00000000 --- a/test/fixtures/test_grammars/conflict_in_repeat_rule/grammar.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "name": "conflict_in_repeat_rule", - - "rules": { - "statement": { - "type": "CHOICE", - "members": [ - { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "array"}, - {"type": "STRING", "value": ";"} - ] - }, - { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "array_type"}, - {"type": "SYMBOL", "name": "identifier"}, - {"type": "STRING", "value": ";"} - ] - } - ] - }, - - "array": { - "type": "SEQ", - "members": [ - { - "type": "STRING", - "value": "[" - }, - { - "type": "REPEAT", - "content": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "identifier"}, - {"type": "STRING", "value": "0"} - ] - } - }, - { - "type": "STRING", - "value": "]" - } - ] - }, - - "array_type": { - "type": "SEQ", - "members": [ - { - "type": "STRING", - "value": "[" - }, - { - "type": "REPEAT", - "content": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "identifier"}, - {"type": "STRING", "value": "void"} - ] - } - }, - { - "type": "STRING", - "value": "]" - } - ] - }, - - "identifier": {"type": "PATTERN", "value": "[a-z]+"} - } -} diff --git a/test/fixtures/test_grammars/conflict_in_repeat_rule/readme.md b/test/fixtures/test_grammars/conflict_in_repeat_rule/readme.md deleted file mode 100644 index be4e65ed..00000000 --- a/test/fixtures/test_grammars/conflict_in_repeat_rule/readme.md +++ /dev/null @@ -1 +0,0 @@ -This grammar has a conflict that involves *repeat rules*: auxiliary rules that are added by the parser generator in order to implement repetition. There is no way of referring to these rules in the grammar DSL, so these conflicts must be resolved by referring to their parent rules. \ No newline at end of file diff --git a/test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.js b/test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.js new file mode 100644 index 00000000..27364b22 --- /dev/null +++ b/test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.js @@ -0,0 +1,32 @@ +// This grammar is similar to the `conflict_in_repeat_rule` grammar, except that the conflict occurs +// after an external token is consumed. This tests that the logic for determining the repeat rule's +// "parent" rule works in the presence of external tokens. + +module.exports = grammar({ + name: 'conflict_in_repeat_rule_after_external_token', + + externals: $ => [ + $._program_start, + ], + + rules: { + statement: $ => choice( + seq($._program_start, $.array, ';'), + seq($._program_start, $.array_type, $.identifier, ';'), + ), + + array: $ => seq( + '[', + repeat(choice($.identifier, '0')), + ']', + ), + + array_type: $ => seq( + '[', + repeat(choice($.identifier, 'void')), + ']', + ), + + identifier: $ => /[a-z]+/ + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.json b/test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.json deleted file mode 100644 index d97d9c9d..00000000 --- a/test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/grammar.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "name": "conflict_in_repeat_rule_after_external_token", - - "externals": [ - {"type": "SYMBOL", "name": "_program_start"} - ], - - "rules": { - "statement": { - "type": "CHOICE", - "members": [ - { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "_program_start"}, - {"type": "SYMBOL", "name": "array"}, - {"type": "STRING", "value": ";"} - ] - }, - { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "_program_start"}, - {"type": "SYMBOL", "name": "array_type"}, - {"type": "SYMBOL", "name": "identifier"}, - {"type": "STRING", "value": ";"} - ] - } - ] - }, - - "array": { - "type": "SEQ", - "members": [ - { - "type": "STRING", - "value": "[" - }, - { - "type": "REPEAT", - "content": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "identifier"}, - {"type": "STRING", "value": "0"} - ] - } - }, - { - "type": "STRING", - "value": "]" - } - ] - }, - - "array_type": { - "type": "SEQ", - "members": [ - { - "type": "STRING", - "value": "[" - }, - { - "type": "REPEAT", - "content": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "identifier"}, - {"type": "STRING", "value": "void"} - ] - } - }, - { - "type": "STRING", - "value": "]" - } - ] - }, - - "identifier": {"type": "PATTERN", "value": "[a-z]+"} - } -} diff --git a/test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/readme.md b/test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/readme.md deleted file mode 100644 index 32841cd0..00000000 --- a/test/fixtures/test_grammars/conflict_in_repeat_rule_after_external_token/readme.md +++ /dev/null @@ -1 +0,0 @@ -This grammar is similar to the `conflict_in_repeat_rule` grammar, except that the conflict occurs after an external token is consumed. This tests that the logic for determining the repeat rule's "parent" rule works in the presence of external tokens. diff --git a/test/fixtures/test_grammars/conflicting_precedence/grammar.js b/test/fixtures/test_grammars/conflicting_precedence/grammar.js new file mode 100644 index 00000000..8092f8e9 --- /dev/null +++ b/test/fixtures/test_grammars/conflicting_precedence/grammar.js @@ -0,0 +1,16 @@ +module.exports = grammar({ + name: 'conflicting_precedence', + + rules: { + expression: $ => choice( + $.sum, + $.product, + $.other_thing, + ), + + sum: $ => prec.left(0, seq($.expression, '+', $.expression)), + product: $ => prec.left(1, seq($.expression, '*', $.expression)), + other_thing: $ => prec.left(-1, seq($.expression, '*', '*')), + identifier: $ => /[a-zA-Z]+/ + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/conflicting_precedence/grammar.json b/test/fixtures/test_grammars/conflicting_precedence/grammar.json deleted file mode 100644 index 4e82de64..00000000 --- a/test/fixtures/test_grammars/conflicting_precedence/grammar.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "name": "conflicting_precedence", - - "rules": { - "expression": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "sum"}, - {"type": "SYMBOL", "name": "product"}, - {"type": "SYMBOL", "name": "other_thing"} - ] - }, - - "sum": { - "type": "PREC_LEFT", - "value": 0, - "content": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "expression"}, - {"type": "STRING", "value": "+"}, - {"type": "SYMBOL", "name": "expression"} - ] - } - }, - - "product": { - "type": "PREC_LEFT", - "value": 1, - "content": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "expression"}, - {"type": "STRING", "value": "*"}, - {"type": "SYMBOL", "name": "expression"} - ] - } - }, - - "other_thing": { - "type": "PREC_LEFT", - "value": -1, - "content": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "expression"}, - {"type": "STRING", "value": "*"}, - {"type": "STRING", "value": "*"} - ] - } - }, - - "identifier": { - "type": "PATTERN", - "value": "[a-zA-Z]+" - } - } -} \ No newline at end of file diff --git a/test/fixtures/test_grammars/dynamic_precedence/corpus.txt b/test/fixtures/test_grammars/dynamic_precedence/corpus.txt index 242efc14..4265a620 100644 --- a/test/fixtures/test_grammars/dynamic_precedence/corpus.txt +++ b/test/fixtures/test_grammars/dynamic_precedence/corpus.txt @@ -2,7 +2,7 @@ Declarations =============================== -int * x +T * x --- @@ -14,7 +14,7 @@ int * x Expressions =============================== -int * x * y +w * x * y --- diff --git a/test/fixtures/test_grammars/dynamic_precedence/grammar.js b/test/fixtures/test_grammars/dynamic_precedence/grammar.js new file mode 100644 index 00000000..321f1139 --- /dev/null +++ b/test/fixtures/test_grammars/dynamic_precedence/grammar.js @@ -0,0 +1,32 @@ +module.exports = grammar({ + name: 'dynamic_precedence', + + extras: $ => [/\s/], + + conflicts: $ => [[$.expression, $.type]], + + rules: { + program: $ => choice( + $.declaration, + $.expression, + ), + + expression: $ => choice( + prec.left(seq($.expression, '*', $.expression)), + $.identifier + ), + + declaration: $ => seq( + $.type, + $.declarator, + ), + + declarator: $ => choice( + prec.dynamic(1, seq('*', $.identifier)), + $.identifier, + ), + + type: $ => $.identifier, + identifier: $ => /[a-z-A-Z]+/ + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/dynamic_precedence/grammar.json b/test/fixtures/test_grammars/dynamic_precedence/grammar.json deleted file mode 100644 index 1a7e04ab..00000000 --- a/test/fixtures/test_grammars/dynamic_precedence/grammar.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "name": "dynamic_precedence", - - "conflicts": [ - ["expression", "type"] - ], - - "extras": [ - {"type": "PATTERN", "value": "\\s"} - ], - - "rules": { - "program": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "declaration"}, - {"type": "SYMBOL", "name": "expression"} - ] - }, - - "expression": { - "type": "PREC_LEFT", - "value": 0, - "content": { - "type": "CHOICE", - "members": [ - { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "expression"}, - {"type": "STRING", "value": "*"}, - {"type": "SYMBOL", "name": "expression"} - ] - }, - { - "type": "SYMBOL", - "name": "identifier" - } - ] - } - }, - - "declaration": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "type"}, - {"type": "SYMBOL", "name": "declarator"} - ] - }, - - "declarator": { - "type": "PREC_DYNAMIC", - "value": 1, - "content": { - "type": "SEQ", - "members": [ - {"type": "STRING", "value": "*"}, - {"type": "SYMBOL", "name": "identifier"} - ] - } - }, - - "type": { - "type": "SYMBOL", - "name": "identifier" - }, - - "identifier": { - "type": "PATTERN", - "value": "[a-zA-Z]+" - } - } -} diff --git a/test/fixtures/test_grammars/epsilon_external_tokens/grammar.js b/test/fixtures/test_grammars/epsilon_external_tokens/grammar.js new file mode 100644 index 00000000..27deef47 --- /dev/null +++ b/test/fixtures/test_grammars/epsilon_external_tokens/grammar.js @@ -0,0 +1,10 @@ +module.exports = grammar({ + name: 'epsilon_external_tokens', + + extras: $ => [/\s/], + externals: $ => [$.zero_width], + + rules: { + document: $ => seq($.zero_width, 'hello'), + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/epsilon_external_tokens/grammar.json b/test/fixtures/test_grammars/epsilon_external_tokens/grammar.json deleted file mode 100644 index 89be042a..00000000 --- a/test/fixtures/test_grammars/epsilon_external_tokens/grammar.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "epsilon_external_tokens", - - "externals": [ - {"type": "SYMBOL", "name": "zero_width"} - ], - - "extras": [ - {"type": "PATTERN", "value": "\\s"} - ], - - "rules": { - "document": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "zero_width"}, - {"type": "STRING", "value": "hello"} - ] - } - } -} diff --git a/test/fixtures/test_grammars/epsilon_rules/grammar.js b/test/fixtures/test_grammars/epsilon_rules/grammar.js new file mode 100644 index 00000000..8cba729b --- /dev/null +++ b/test/fixtures/test_grammars/epsilon_rules/grammar.js @@ -0,0 +1,11 @@ +module.exports = grammar({ + name: 'epsilon_rules', + + rules: { + rule_1: $ => $.rule_2, + + rule_2: $ => optional($.rule_3), + + rule_3: $ => 'x' + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/epsilon_rules/grammar.json b/test/fixtures/test_grammars/epsilon_rules/grammar.json deleted file mode 100644 index 5be5b983..00000000 --- a/test/fixtures/test_grammars/epsilon_rules/grammar.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "epsilon_rules", - - "rules": { - "rule_1": {"type": "SYMBOL", "name": "rule_2"}, - - "rule_2": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "rule_1"}, - {"type": "BLANK"} - ] - } - } -} \ No newline at end of file diff --git a/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.js b/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.js new file mode 100644 index 00000000..e289f5ad --- /dev/null +++ b/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.js @@ -0,0 +1,28 @@ +module.exports = grammar({ + name: 'external_and_internal_anonymous_tokens', + + externals: $ => [ + $.string, + '\n' + ], + + extras: $ => [/\s/], + + rules: { + statement: $ => seq( + $._expression, + $._expression, + '\n' + ), + + _expression: $ => choice( + $.string, + $.variable, + $.number + ), + + variable: $ => /[a-z]+/, + + number: $ => /\d+/ + } +}) \ No newline at end of file diff --git a/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.json b/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.json deleted file mode 100644 index 057bdbf0..00000000 --- a/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/grammar.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "external_and_internal_anonymous_tokens", - - "externals": [ - {"type": "SYMBOL", "name": "string"}, - {"type": "STRING", "value": "\n"} - ], - - "extras": [ - {"type": "PATTERN", "value": "\\s"} - ], - - "rules": { - "statement": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "_expression"}, - {"type": "SYMBOL", "name": "_expression"}, - {"type": "STRING", "value": "\n"} - ] - }, - - "_expression": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "string"}, - {"type": "SYMBOL", "name": "variable"}, - {"type": "SYMBOL", "name": "number"} - ] - }, - - "variable": {"type": "PATTERN", "value": "[a-z]+"}, - "number": {"type": "PATTERN", "value": "\\d+"} - } -} diff --git a/test/fixtures/test_grammars/external_and_internal_tokens/grammar.js b/test/fixtures/test_grammars/external_and_internal_tokens/grammar.js new file mode 100644 index 00000000..6b70c98a --- /dev/null +++ b/test/fixtures/test_grammars/external_and_internal_tokens/grammar.js @@ -0,0 +1,32 @@ +// This grammar has an external scanner whose `scan` method needs to be able to check for the +// validity of an *internal* token. This is done by including the names of that internal token +// (`line_break`) in the grammar's `externals` field. + +module.exports = grammar({ + name: 'external_and_internal_tokens', + + externals: $ => [ + $.string, + $.line_break, + ], + + extras: $ => [/\s/], + + rules: { + statement: $ => seq( + $._expression, + $._expression, + $.line_break, + ), + + _expression: $ => choice( + $.string, + $.variable, + $.number, + ), + + variable: $ => /[a-z]+/, + number: $ => /\d+/, + line_break: $ => '\n', + } +}); \ No newline at end of file diff --git a/test/fixtures/test_grammars/external_and_internal_tokens/grammar.json b/test/fixtures/test_grammars/external_and_internal_tokens/grammar.json deleted file mode 100644 index 62d77383..00000000 --- a/test/fixtures/test_grammars/external_and_internal_tokens/grammar.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "external_and_internal_tokens", - - "externals": [ - {"type": "SYMBOL", "name": "string"}, - {"type": "SYMBOL", "name": "line_break"} - ], - - "extras": [ - {"type": "PATTERN", "value": "\\s"} - ], - - "rules": { - "statement": { - "type": "SEQ", - "members": [ - {"type": "SYMBOL", "name": "_expression"}, - {"type": "SYMBOL", "name": "_expression"}, - {"type": "SYMBOL", "name": "line_break"} - ] - }, - - "_expression": { - "type": "CHOICE", - "members": [ - {"type": "SYMBOL", "name": "string"}, - {"type": "SYMBOL", "name": "variable"}, - {"type": "SYMBOL", "name": "number"} - ] - }, - - "variable": {"type": "PATTERN", "value": "[a-z]+"}, - "number": {"type": "PATTERN", "value": "\\d+"}, - "line_break": {"type": "STRING", "value": "\n"} - } -} diff --git a/test/fixtures/test_grammars/external_and_internal_tokens/readme.md b/test/fixtures/test_grammars/external_and_internal_tokens/readme.md index 14ae934f..e69de29b 100644 --- a/test/fixtures/test_grammars/external_and_internal_tokens/readme.md +++ b/test/fixtures/test_grammars/external_and_internal_tokens/readme.md @@ -1 +0,0 @@ -This grammar has an external scanner whose `scan` method needs to be able to check for the validity of an *internal* token. This is done by including the names of that internal token (`_line_break`) in the grammar's `externals` field. \ No newline at end of file