Rework javascript fixture grammar

This commit is contained in:
Max Brunsfeld 2014-10-21 08:49:16 -07:00
parent 8134b64d00
commit 91cf35b72c
7 changed files with 55530 additions and 90492 deletions

View file

@ -5,7 +5,7 @@ namespace tree_sitter_examples {
using namespace tree_sitter::rules;
rule_ptr comma_sep1(rule_ptr element) {
return seq({ element, repeat(seq({ str(","), element })) });
return seq({ element, repeat(seq({ keyword(","), element })) });
}
rule_ptr comma_sep(rule_ptr element) {

View file

@ -12,207 +12,328 @@ static rule_ptr terminated(rule_ptr rule) {
str(";") }) });
}
enum {
PREC_COMMA = -1,
PREC_ASSIGN,
PREC_TERNARY,
PREC_OR,
PREC_AND,
PREC_REL,
PREC_ADD,
PREC_MULT,
PREC_TYPE,
PREC_NOT,
PREC_SIGN,
PREC_INC,
PREC_CALL,
PREC_CONSTRUCT,
PREC_MEMBER,
};
extern const Grammar javascript = Grammar({
{ "program", repeat(sym("statement")) },
// Statements
/*
* Statements
*/
{ "statement", choice({
sym("expression_statement"),
sym("var_declaration"),
sym("statement_block"),
sym("if_statement"),
sym("try_statement"),
sym("switch_statement"),
sym("while_statement"),
sym("for_statement"),
sym("while_statement"),
sym("for_in_statement"),
sym("break_statement"),
sym("var_declaration"),
sym("throw_statement"),
sym("do_statement"),
sym("try_statement"),
sym("return_statement"),
sym("delete_statement"),
sym("expression_statement") }) },
{ "statement_block", in_braces(err(repeat(sym("statement")))) },
{ "for_statement", seq({
keyword("for"),
in_parens(err(seq({
choice({
sym("var_declaration"),
sym("expression_statement") }),
sym("expression_statement"),
sym("expression") }))),
sym("statement") }) },
{ "for_in_statement", seq({
keyword("for"),
in_parens(err(seq({
optional(keyword("var")),
sym("identifier"),
keyword("in"),
sym("expression") }))),
sym("statement") }) },
{ "throw_statement", terminated(seq({
keyword("throw"),
sym("expression") })) },
{ "if_statement", seq({
keyword("if"),
in_parens(err(sym("expression"))),
sym("statement"),
optional(prec(1, seq({
keyword("else"),
sym("statement") }))) }) },
{ "while_statement", seq({
keyword("while"),
in_parens(err(sym("expression"))),
sym("statement") }) },
{ "try_statement", seq({
keyword("try"),
sym("statement"),
optional(sym("catch_clause")),
optional(sym("finally_clause")) }) },
{ "catch_clause", seq({
keyword("catch"),
in_parens(err(sym("identifier"))),
sym("statement") }) },
{ "finally_clause", seq({
keyword("finally"),
sym("statement") }) },
{ "switch_statement", seq({
keyword("switch"),
in_parens(err(sym("expression"))),
in_braces(repeat(sym("switch_case"))) }) },
{ "switch_case", seq({
choice({
seq({
keyword("case"),
sym("expression") }),
keyword("default") }),
str(":"),
repeat(sym("statement")) }) },
{ "break_statement", terminated(keyword("break")) },
sym("break_statement"),
sym("throw_statement"),
sym("delete_statement") }) },
{ "expression_statement", terminated(err(sym("expression"))) },
{ "var_declaration", terminated(seq({
keyword("var"),
comma_sep1(err(choice({
sym("identifier"),
sym("var_assignment") }))) })) },
{ "statement_block", in_braces(err(repeat(sym("statement")))) },
{ "if_statement", seq({
keyword("if"),
sym("_paren_expression"),
sym("statement"),
optional(seq({
keyword("else"),
sym("statement") })) }) },
{ "switch_statement", seq({
keyword("switch"),
sym("_paren_expression"),
str("{"),
repeat(choice({ sym("case"), sym("default") })),
str("}") }) },
{ "for_statement", seq({
keyword("for"),
str("("),
choice({
sym("var_declaration"),
seq({ sym("expression"), str(";") }),
str(";") }),
optional(err(sym("expression"))), str(";"),
optional(err(sym("expression"))),
str(")"),
sym("statement") }) },
{ "for_in_statement", seq({
keyword("for"),
str("("),
optional(keyword("var")),
sym("identifier"),
keyword("in"),
sym("expression"),
str(")"),
sym("statement") }) },
{ "while_statement", seq({
keyword("while"),
sym("_paren_expression"),
sym("statement") }) },
{ "do_statement", seq({
keyword("do"),
sym("statement"),
keyword("while"),
sym("_paren_expression") })},
{ "try_statement", seq({
keyword("try"),
sym("statement_block"),
optional(sym("catch")),
optional(sym("finally")) }) },
{ "return_statement", terminated(seq({
keyword("return"),
optional(sym("expression")) })) },
{ "throw_statement", terminated(seq({
keyword("throw"),
sym("expression") })) },
{ "break_statement", terminated(keyword("break")) },
{ "delete_statement", terminated(seq({
keyword("delete"),
choice({ sym("member_access"), sym("subscript_access") }) })) },
/*
* Statement components
*/
{ "case", seq({
keyword("case"),
sym("expression"),
str(":"),
repeat(sym("statement")) }) },
{ "default", seq({
keyword("default"),
str(":"),
repeat(sym("statement")) }) },
{ "catch", seq({
keyword("catch"),
str("("),
err(sym("identifier")),
str(")"),
sym("statement_block") }) },
{ "finally", seq({
keyword("finally"),
sym("statement_block") }) },
{ "var_assignment", seq({
sym("identifier"),
str("="),
sym("expression") }) },
{ "expression_statement", terminated(err(sym("expression"))) },
{ "return_statement", terminated(seq({
keyword("return"),
optional(sym("expression")) })) },
{ "delete_statement", terminated(seq({
keyword("delete"),
sym("property_access") })) },
// Expressions
{ "_paren_expression", in_parens(err(sym("expression"))) },
/*
* Expressions
*/
{ "expression", choice({
sym("object"),
sym("array"),
sym("function_expression"),
sym("function_call"),
sym("constructor_call"),
sym("property_access"),
sym("member_access"),
sym("subscript_access"),
sym("assignment"),
sym("math_assignment"),
sym("ternary"),
sym("math_op"),
sym("bool_op"),
sym("object"),
sym("array"),
sym("math_op"),
sym("comma_op"),
sym("bitwise_op"),
sym("rel_op"),
sym("type_op"),
sym("null"),
sym("number"),
sym("undefined"),
sym("regex"),
sym("string"),
sym("number"),
sym("true"),
sym("false"),
sym("null"),
sym("identifier"),
sym("in_expression"),
sym("instanceof_expression"),
sym("typeof_expression"),
in_parens(sym("expression")) }) },
{ "in_expression", infix_op("in", "expression", 3) },
{ "instanceof_expression", infix_op("instanceof", "expression", 3) },
{ "typeof_expression", prefix_op("typeof", "expression", 3) },
{ "math_op", choice({
prefix_op("++", "expression", 3),
prefix_op("--", "expression", 3),
postfix_op("++", "expression", 3),
postfix_op("--", "expression", 3),
prefix_op("+", "expression", 3),
prefix_op("-", "expression", 3),
infix_op("*", "expression", 2),
infix_op("/", "expression", 2),
infix_op("&", "expression", 2),
infix_op("|", "expression", 2),
infix_op("^", "expression", 2),
infix_op("+", "expression", 1),
infix_op("-", "expression", 1) }) },
{ "bool_op", choice({
infix_op("||", "expression", 1),
infix_op("&&", "expression", 2),
infix_op("===", "expression", 3),
infix_op("==", "expression", 3),
infix_op("!==", "expression", 3),
infix_op("!=", "expression", 3),
infix_op("<=", "expression", 3),
infix_op("<", "expression", 3),
infix_op(">=", "expression", 3),
infix_op(">", "expression", 3),
prefix_op("!", "expression", 4) }) },
{ "ternary", seq({
sym("true"),
sym("_paren_expression") }) },
{ "object", in_braces(comma_sep(err(sym("pair")))) },
{ "array", in_brackets(comma_sep(err(sym("expression")))) },
{ "function_expression", seq({
keyword("function"),
optional(sym("identifier")),
str("("),
optional(sym("formal_parameters")),
str(")"),
sym("statement_block") }) },
{ "function_call", prec(PREC_CALL, seq({
sym("expression"),
str("("),
optional(err(sym("arguments"))),
str(")") })) },
{ "constructor_call", prec(PREC_CONSTRUCT, seq({
keyword("new"),
sym("expression"),
optional(seq({
str("("),
err(optional(sym("arguments"))),
str(")") })) })) },
{ "member_access", prec(PREC_MEMBER, seq({
sym("expression"),
str("."),
sym("identifier") })) },
{ "subscript_access", prec(PREC_MEMBER, seq({
sym("expression"),
str("["),
err(sym("expression")),
str("]") })) },
{ "assignment", prec(PREC_ASSIGN, seq({
choice({
sym("identifier"),
sym("member_access"),
sym("subscript_access") }),
str("="),
sym("expression") })) },
{ "math_assignment", prec(PREC_ASSIGN, seq({
choice({
sym("identifier"),
sym("member_access"),
sym("subscript_access") }),
choice({ str("+="), str("-="), str("*="), str("/=") }),
sym("expression") })) },
{ "ternary", prec(PREC_TERNARY, seq({
sym("expression"),
str("?"),
sym("expression"),
str(":"),
sym("expression") }) },
{ "assignment", prec(-1, seq({
choice({
sym("identifier"),
sym("property_access") }),
choice({
str("="),
str("+="),
str("-="),
str("*="),
str("/=") }),
sym("expression") })) },
{ "function_expression", seq({
keyword("function"),
optional(sym("identifier")),
in_parens(sym("formal_parameters")),
sym("statement_block") }) },
{ "function_call", seq({
sym("expression"),
in_parens(comma_sep(err(sym("expression")))) }) },
{ "constructor_call", seq({
keyword("new"),
sym("function_call") }) },
{ "property_access", seq({
sym("expression"),
prec(10, choice({
seq({ str("."), sym("identifier") }),
in_brackets(sym("expression")) })) }) },
{ "formal_parameters", comma_sep(sym("identifier")) },
// Literals
{ "bool_op", choice({
infix_op("||", "expression", PREC_OR),
infix_op("&&", "expression", PREC_AND),
prefix_op("!", "expression", PREC_NOT) }) },
{ "comma_op", infix_op(",", "expression", PREC_COMMA) },
{ "math_op", choice({
prefix_op("+", "expression", PREC_SIGN),
prefix_op("-", "expression", PREC_SIGN),
postfix_op("++", "expression", PREC_INC),
postfix_op("--", "expression", PREC_INC),
infix_op("*", "expression", PREC_MULT),
infix_op("/", "expression", PREC_MULT),
infix_op("+", "expression", PREC_ADD),
infix_op("-", "expression", PREC_ADD) }) },
{ "bitwise_op", choice({
infix_op("&", "expression", PREC_MULT),
infix_op("|", "expression", PREC_MULT),
infix_op("<<", "expression", PREC_MULT),
infix_op(">>", "expression", PREC_MULT) }) },
{ "rel_op", choice({
infix_op("==", "expression", PREC_REL),
infix_op("!=", "expression", PREC_REL),
infix_op("<=", "expression", PREC_REL),
infix_op(">=", "expression", PREC_REL),
infix_op("===", "expression", PREC_REL),
infix_op("!==", "expression", PREC_REL),
infix_op("<", "expression", PREC_REL),
infix_op(">", "expression", PREC_REL) }) },
{ "type_op", choice({
infix_op("in", "expression", PREC_REL),
infix_op("instanceof", "expression", PREC_REL),
prefix_op("typeof", "expression", PREC_TYPE) }) },
/*
* Primitives
*/
{ "comment", token(choice({
seq({
str("/*"),
repeat(pattern("[^*]|(*[^/])")),
str("*/") }),
pattern("//[^\n]*") })) },
{ "object", in_braces(comma_sep(err(seq({
choice({ sym("string"), sym("identifier") }),
str(":"),
sym("expression") })))) },
{ "array", in_brackets(comma_sep(err(sym("expression")))) },
{ "regex", token(seq({ delimited("/"), optional(str("g")) })) },
{ "string", token(choice({
delimited("\""),
delimited("'") })) },
{ "_line_break", str("\n") },
{ "identifier", pattern("[\\a_$][\\w_$]*") },
{ "regex", token(seq({ delimited("/"), optional(str("g")) })) },
{ "number", pattern("\\d+(\\.\\d+)?") },
{ "identifier", pattern("[\\a_$][\\w_$]*") },
{ "null", keyword("null") },
{ "undefined", keyword("undefined") },
{ "true", keyword("true") },
{ "false", keyword("false") },
{ "_line_break", str("\n") },
/*
* Expression components
*/
{ "formal_parameters", comma_sep1(sym("identifier")) },
{ "arguments", comma_sep1(err(sym("expression"))) },
{ "pair", seq({
choice({ sym("string"), sym("identifier") }),
str(":"),
sym("expression") }) },
}).ubiquitous_tokens({
sym("comment"),
sym("_line_break"),

File diff suppressed because it is too large Load diff

View file

@ -27,7 +27,7 @@ if (isReady()) {
---
(if_statement (function_call (identifier))
(statement_block (expression_statement (function_call (property_access (identifier) (identifier)) (identifier)))))
(statement_block (expression_statement (function_call (member_access (identifier) (identifier)) (identifier)))))
==========================================
if-else statements
@ -78,7 +78,7 @@ for (var i = 1; someCondition(i); i = next()) {
(for_statement
(var_declaration (var_assignment (identifier) (number)))
(expression_statement (function_call (identifier) (identifier)))
(function_call (identifier) (identifier))
(assignment (identifier) (function_call (identifier)))
(statement_block (expression_statement (function_call (identifier)))))
@ -135,11 +135,11 @@ try {
(program
(try_statement
(statement_block (expression_statement (function_call (identifier))))
(catch_clause (identifier)
(catch (identifier)
(statement_block (expression_statement (function_call (identifier) (identifier))))))
(try_statement
(statement_block (expression_statement (function_call (identifier))))
(finally_clause
(finally
(statement_block (expression_statement (function_call (identifier)))))))
===========================================
@ -150,7 +150,7 @@ throw new Error("wtf");
---
(throw_statement (constructor_call (function_call (identifier) (string))))
(throw_statement (constructor_call (identifier) (string)))
===========================================
indented code after blocks
@ -163,7 +163,7 @@ indented code after blocks
(program
(expression_statement
(function_expression (identifier) (formal_parameters) (statement_block)))
(function_expression (identifier) (statement_block)))
(return_statement (identifier)))
===========================================
@ -184,13 +184,13 @@ switch(x) {
---
(switch_statement (identifier)
(switch_case
(case
(string)
(expression_statement (function_call (identifier) (string)))
(break_statement))
(switch_case
(case
(function_call (identifier))
(expression_statement (function_call (identifier) (string)))
(break_statement))
(switch_case
(default
(expression_statement (function_call (identifier) (string)))))

View file

@ -7,7 +7,7 @@ theFunction(/regex1/, /regex2/g);
---
(expression_statement (function_call (identifier)
(regex) (regex)))
(arguments (regex) (regex))))
==========================================
numbers
@ -18,7 +18,7 @@ theFunction(100.0, 200);
---
(expression_statement (function_call (identifier)
(number) (number)))
(arguments (number) (number))))
==========================================
strings
@ -29,7 +29,7 @@ theFunction('', "", 'single-quoted-string', "double-quoted-string");
---
(expression_statement (function_call (identifier)
(string) (string) (string) (string)))
(arguments (string) (string) (string) (string))))
==========================================
function expressions
@ -45,10 +45,10 @@ var x = {
(var_declaration (var_assignment
(identifier)
(object (identifier) (function_expression
(object (pair (identifier) (function_expression
(formal_parameters (identifier) (identifier))
(statement_block
(var_declaration (var_assignment (identifier) (identifier))))))))
(var_declaration (var_assignment (identifier) (identifier)))))))))
==========================================
comments
@ -75,11 +75,11 @@ var thing = {
(object
(comment)
(comment)
(identifier) (function_expression
(pair (identifier) (function_expression
(formal_parameters (identifier) (comment))
(statement_block
(comment)
(expression_statement (function_call (identifier))))))))
(expression_statement (function_call (identifier)))))))))
==========================================
comments within expressions

View file

@ -7,8 +7,8 @@ x.theMethod(5, 6);
---
(expression_statement (function_call
(property_access (identifier) (identifier))
(number) (number)))
(member_access (identifier) (identifier))
(arguments (number) (number))))
==========================================
constructor calls
@ -20,9 +20,9 @@ var x = new Node(5, new Node(3, null));
(var_declaration (var_assignment
(identifier)
(constructor_call (function_call (identifier)
(constructor_call (identifier) (arguments
(number)
(constructor_call (function_call (identifier)
(constructor_call (identifier) (arguments
(number)
(null)))))))
@ -37,11 +37,11 @@ print(object.property);
(program
(expression_statement (assignment
(property_access (identifier) (identifier))
(member_access (identifier) (identifier))
(string)))
(expression_statement (function_call
(identifier)
(property_access (identifier) (identifier)))))
(member_access (identifier) (identifier)))))
==========================================
property access across lines
@ -54,8 +54,8 @@ object
---
(expression_statement
(property_access
(property_access (identifier) (identifier))
(member_access
(member_access (identifier) (identifier))
(identifier)))
===========================================
@ -68,11 +68,11 @@ print(object[propertyName()]);
---
(program
(expression_statement (assignment
(property_access (identifier) (function_call (identifier)))
(subscript_access (identifier) (function_call (identifier)))
(function_call (identifier))))
(expression_statement (function_call
(identifier)
(property_access (identifier) (function_call (identifier))))))
(subscript_access (identifier) (function_call (identifier))))))
==========================================
ternary expressions
@ -91,7 +91,7 @@ print(isDone() ? stuff : otherStuff);
mathematical operators
==========================================
-a + b * c - d / +e
a++ + b * c - d / e--
---
(expression_statement
@ -119,14 +119,14 @@ boolean operators
type operators
===========================================
print((x instanceof Array) || (typeof x == "string"))
print((x instanceof Array) || (typeof x === "string"))
---
(expression_statement (function_call (identifier)
(bool_op
(expression (instanceof_expression (identifier) (identifier)))
(expression (typeof_expression (bool_op (identifier) (string)))))))
(expression (type_op (identifier) (identifier)))
(expression (rel_op (type_op (identifier)) (string))))))
============================================
the 'in' operator
@ -137,7 +137,7 @@ print(x in y)
---
(expression_statement (function_call
(identifier)
(in_expression (identifier) (identifier))))
(type_op (identifier) (identifier))))
============================================
assignment operators
@ -151,10 +151,10 @@ x /= 2;
---
(program
(expression_statement (assignment (identifier) (number)))
(expression_statement (assignment (identifier) (number)))
(expression_statement (assignment (identifier) (number)))
(expression_statement (assignment (identifier) (number))))
(expression_statement (math_assignment (identifier) (number)))
(expression_statement (math_assignment (identifier) (number)))
(expression_statement (math_assignment (identifier) (number)))
(expression_statement (math_assignment (identifier) (number))))
============================================
property access and operators
@ -166,5 +166,5 @@ print(x.y.z && a.b.c)
(expression_statement (function_call (identifier)
(bool_op
(property_access (property_access (identifier) (identifier)) (identifier))
(property_access (property_access (identifier) (identifier)) (identifier)))))
(member_access (member_access (identifier) (identifier)) (identifier))
(member_access (member_access (identifier) (identifier)) (identifier)))))

View file

@ -193,7 +193,7 @@ describe("Parser", [&]() {
AssertThat(ts_node_string(root), Equals(
"(DOCUMENT "
"(expression_statement (function_call "
"(property_access (function_call (identifier)) (identifier)))))"));
"(member_access (function_call (identifier)) (identifier)))))"));
});
});
@ -208,7 +208,7 @@ describe("Parser", [&]() {
AssertThat(ts_node_string(root), Equals(
"(DOCUMENT "
"(expression_statement (function_call "
"(property_access (function_call (identifier)) "
"(member_access (function_call (identifier)) "
"(comment) "
"(identifier)))))"));
});
@ -423,14 +423,14 @@ describe("Parser", [&]() {
set_text("{ x: (b.c) };");
AssertThat(ts_node_string(root), Equals(
"(DOCUMENT (expression_statement (object "
"(identifier) (expression (property_access (identifier) (identifier))))))"));
"(DOCUMENT (expression_statement (object (pair "
"(identifier) (expression (member_access (identifier) (identifier)))))))"));
replace_text(strlen("{ x: "), strlen("(b.c)"), "b.c");
AssertThat(ts_node_string(root), Equals(
"(DOCUMENT (expression_statement (object "
"(identifier) (property_access (identifier) (identifier)))))"));
"(DOCUMENT (expression_statement (object (pair "
"(identifier) (member_access (identifier) (identifier))))))"));
});
});
});