traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add regex postfix flags to javascript grammar

Browse source 
- Refactor statement terminators in javascript grammar
- Reorganize javascript language tests
This commit is contained in:
Max Brunsfeld 2014-06-11 16:43:03 -07:00
parent 082560dd6e
commit bb4d83ce47
9 changed files with 32747 additions and 32747 deletions
Download patch file Download diff file Expand all files Collapse all files

32
examples/grammars/javascript.cc
View file

@ -5,6 +5,12 @@ namespace tree_sitter_examples {
using tree_sitter::Grammar;
using namespace tree_sitter::rules;
static rule_ptr terminated(rule_ptr rule) {
return seq({ rule, prec(-1, token(choice({
str("\n"),
str(";") }))) });
}
extern const Grammar javascript = Grammar({
{ "program", repeat(sym("statement")) },
@ -66,26 +72,19 @@ namespace tree_sitter_examples {
keyword("default") }),
str(":"),
repeat(sym("statement")) }) },
{ "break_statement", seq({
keyword("break"),
sym("_terminator") }) },
{ "var_declaration", seq({
{ "break_statement", terminated(keyword("break")) },
{ "var_declaration", terminated(seq({
keyword("var"),
comma_sep(err(choice({
sym("assignment"),
sym("identifier") }))),
sym("_terminator") }) },
{ "expression_statement", seq({
err(sym("expression")),
sym("_terminator") }) },
{ "return_statement", seq({
sym("identifier") }))) })) },
{ "expression_statement", terminated(err(sym("expression"))) },
{ "return_statement", terminated(seq({
keyword("return"),
optional(sym("expression")),
sym("_terminator") }) },
{ "delete_statement", seq({
optional(sym("expression")) })) },
{ "delete_statement", terminated(seq({
keyword("delete"),
sym("property_access"),
sym("_terminator") }) },
sym("property_access") })) },
// Expressions
{ "expression", choice({
@ -183,8 +182,7 @@ namespace tree_sitter_examples {
str(":"),
sym("expression") })))) },
{ "array", in_brackets(comma_sep(err(sym("expression")))) },
{ "_terminator", pattern("[;\n]") },
{ "regex", token(delimited("/")) },
{ "regex", token(seq({ delimited("/"), optional(str("g")) })) },
{ "string", token(choice({
delimited("\""),
delimited("'") })) },

65093
examples/parsers/javascript.c
View file

File diff suppressed because it is too large Load diff

83
spec/runtime/languages/javascript/literals.txt Normal file
View file

@ -0,0 +1,83 @@
==========================================
parses regexes
==========================================
theFunction(/regex1/, /regex2/g);
---
(program (expression_statement (function_call (identifier)
(regex) (regex))))
==========================================
parses numbers
==========================================
theFunction(100.0, 200);
---
(program (expression_statement (function_call (identifier)
(number) (number))))
==========================================
parses strings
==========================================
theFunction('', "", 'single-quoted-string', "double-quoted-string");
---
(program (expression_statement (function_call (identifier)
(string) (string) (string) (string))))
==========================================
parses multiple statements
==========================================
var x = {}, z, i = 0;
firstFunction(x)
secondFunction(x);
---
(program
(var_declaration
(assignment (identifier) (object))
(identifier)
(assignment (identifier) (number)))
(expression_statement (function_call (identifier) (identifier)))
(expression_statement (function_call (identifier) (identifier))))
==========================================
parses function expressions
==========================================
var x = {
theMethod: function(argA, argB) {
var x = argA;
}
};
---
(program
(var_declaration (assignment
(identifier)
(object (identifier) (function_expression
(formal_parameters (identifier) (identifier))
(statement_block (var_declaration (assignment (identifier) (identifier)))))))))
==========================================
parses comments
==========================================
// this is the beginning of the script.
// here we go.
var thing = {
// this is a property.
// its value is a function.
key: function(x /* this is a parameter */) {
// this is a statement
doStuff();
}
};
---
(program
(comment)
(comment)
(var_declaration (assignment (identifier) (object
(comment)
(comment)
(identifier) (function_expression
(formal_parameters (identifier) (comment))
(statement_block
(comment)
(expression_statement (function_call (identifier)))))))))

180
spec/runtime/languages/javascript/main.txt
View file

@ -1,180 +0,0 @@
==========================================
parses literals
==========================================
theFunction(
100.0,
200,
/the-regex/,
'',
"",
'the-single-quoted-string',
"the-double-quoted-string"
);
---
(program (expression_statement (function_call
(identifier)
(number)
(number)
(regex)
(string)
(string)
(string)
(string))))
==========================================
parses constructor calls
==========================================
var x = new Node(5, new Node(3, null));
---
(program (var_declaration
(assignment (identifier)
(constructor_call (function_call (identifier)
(number)
(constructor_call (function_call (identifier)
(number)
(null))))))))
==========================================
parses multiple statements
==========================================
var x = {}, z, i = 0;
firstFunction(x)
secondFunction(x);
---
(program
(var_declaration
(assignment (identifier) (object))
(identifier)
(assignment (identifier) (number)))
(expression_statement (function_call (identifier) (identifier)))
(expression_statement (function_call (identifier) (identifier))))
==========================================
parses function expressions and calls
==========================================
var x = {
theMethod: function(argA, argB) {
var x = argA;
}
};
x.theMethod(5, 6);
---
(program
(var_declaration (assignment
(identifier)
(object (identifier) (function_expression
(formal_parameters (identifier) (identifier))
(statement_block (var_declaration (assignment (identifier) (identifier))))))))
(expression_statement (function_call
(property_access (identifier) (identifier))
(number) (number))))
==========================================
parses property access with dot notation
==========================================
object.property = "the-value";
print(object.property);
---
(program
(expression_statement (assignment
(property_access (identifier) (identifier))
(string)))
(expression_statement (function_call
(identifier)
(property_access (identifier) (identifier)))))
===========================================
parses dynamic property access
==========================================
object[propertName()] = propertyValue();
print(object[propertyName()]);
---
(program
(expression_statement (assignment
(property_access (identifier) (function_call (identifier)))
(function_call (identifier))))
(expression_statement (function_call
(identifier)
(property_access (identifier) (function_call (identifier))))))
==========================================
parses comments
==========================================
// this is the beginning of the script.
// here we go.
var thing = {
// this is a property.
// its value is a function.
key: function(x /* this is a parameter */) {
// this is a statement
doStuff();
}
};
---
(program
(comment)
(comment)
(var_declaration (assignment (identifier) (object
(comment)
(comment)
(identifier) (function_expression
(formal_parameters (identifier) (comment))
(statement_block
(comment)
(expression_statement (function_call (identifier)))))))))
======================================
parses real code
=====================================
_.each = _.forEach = function(obj, iterator, context) {
if (obj == null) return obj;
if (obj.length === obj.length) {
for (var i = 0; i < length; i++) {
if (iterator.call(context, obj[i], i, obj) === breaker) return;
}
} else {
var keys = _.keys(obj);
for (var i = 0; i < length; i++) {
if (iterator.call(context, obj[keys[i]], keys[i], obj) === breaker) return;
}
}
return obj;
};
---
(program (expression_statement
(assignment
(property_access (identifier) (identifier))
(assignment (property_access (identifier) (identifier))
(function_expression
(formal_parameters (identifier) (identifier) (identifier))
(statement_block
(if_statement (bool_op (identifier) (null)) (return_statement (identifier)))
(if_statement
(bool_op (property_access (identifier) (identifier)) (property_access (identifier) (identifier)))
(statement_block
(for_statement
(var_declaration (assignment (identifier) (number)))
(expression_statement (bool_op (identifier) (identifier)))
(math_op (identifier))
(statement_block
(if_statement
(bool_op (function_call (property_access (identifier) (identifier)) (identifier) (property_access (identifier) (identifier)) (identifier) (identifier)) (identifier))
(return_statement)))))
(statement_block
(var_declaration (assignment (identifier) (function_call (property_access (identifier) (identifier)) (identifier))))
(for_statement
(var_declaration (assignment (identifier) (number)))
(expression_statement (bool_op (identifier) (identifier)))
(math_op (identifier))
(statement_block
(if_statement
(bool_op (function_call
(property_access (identifier) (identifier))
(identifier)
(property_access (identifier) (property_access (identifier) (identifier)))
(property_access (identifier) (identifier)) (identifier)) (identifier))
(return_statement))))))
(return_statement (identifier))))))))

50
spec/runtime/languages/javascript/operators.txt
View file

@ -1,3 +1,53 @@
==========================================
parses function calls
==========================================
x.theMethod(5, 6);
---
(program (expression_statement (function_call
(property_access (identifier) (identifier))
(number) (number))))
==========================================
parses constructor calls
==========================================
var x = new Node(5, new Node(3, null));
---
(program (var_declaration
(assignment (identifier)
(constructor_call (function_call (identifier)
(number)
(constructor_call (function_call (identifier)
(number)
(null))))))))
==========================================
parses property access with dot notation
==========================================
object.property = "the-value";
print(object.property);
---
(program
(expression_statement (assignment
(property_access (identifier) (identifier))
(string)))
(expression_statement (function_call
(identifier)
(property_access (identifier) (identifier)))))
===========================================
parses dynamic property access
==========================================
object[propertName()] = propertyValue();
print(object[propertyName()]);
---
(program
(expression_statement (assignment
(property_access (identifier) (function_call (identifier)))
(function_call (identifier))))
(expression_statement (function_call
(identifier)
(property_access (identifier) (function_call (identifier))))))
==========================================
parses ternary expressions
==========================================

8
src/compiler/generate_code/c_code.cc
View file

@ -17,7 +17,7 @@ namespace tree_sitter {
using std::pair;
using util::join;
using util::indent;
using util::character_code;
using util::escape_char;
namespace generate_code {
string _switch(string condition, string body) {
@ -154,10 +154,10 @@ namespace tree_sitter {
string condition_for_character_range(const rules::CharacterRange &range) {
string lookahead("lookahead");
if (range.min == range.max) {
return lookahead + " == '" + character_code(range.min) + "'";
return lookahead + " == '" + escape_char(range.min) + "'";
} else {
return string("'") + character_code(range.min) + string("' <= ") + lookahead +
" && " + lookahead + " <= '" + character_code(range.max) + "'";
return string("'") + escape_char(range.min) + string("' <= ") + lookahead +
" && " + lookahead + " <= '" + escape_char(range.max) + "'";
}
}

4
src/compiler/prepare_grammar/token_description.cc
View file

@ -14,11 +14,11 @@ namespace tree_sitter {
namespace prepare_grammar {
class TokenDescription : public rules::RuleFn<string> {
string apply_to(const rules::Pattern *rule) {
return "/" + rule->value + "/";
return "/" + util::escape_string(rule->value) + "/";
}
string apply_to(const rules::String *rule) {
return "'" + rule->value + "'";
return "'" + util::escape_string(rule->value) + "'";
}
string apply_to(const rules::Metadata *rule) {

42
src/compiler/util/string_helpers.cc
View file

@ -23,6 +23,27 @@ namespace tree_sitter {
str_replace(&input, "\n", "\\n");
return input;
}
string escape_char(char character) {
switch (character) {
case '\0':
return "\\0";
case '"':
return "\\\"";
case '\'':
return "\\'";
case '\n':
return "\\n";
case '\r':
return "\\r";
case '\t':
return "\\t";
case '\\':
return "\\\\";
default:
return string() + character;
}
}
string join(vector<string> lines, string separator) {
string result;
@ -44,26 +65,5 @@ namespace tree_sitter {
util::str_replace(&input, "\n", "\n" + tab);
return tab + input;
}
string character_code(char character) {
switch (character) {
case '\0':
return "\\0";
case '"':
return "\\\"";
case '\'':
return "\\'";
case '\n':
return "\\n";
case '\r':
return "\\r";
case '\t':
return "\\t";
case '\\':
return "\\\\";
default:
return string() + character;
}
}
}
}

2
src/compiler/util/string_helpers.h
View file

@ -9,10 +9,10 @@ namespace tree_sitter {
namespace util {
void str_replace(std::string *input, const std::string &search, const std::string &replace);
std::string escape_string(std::string input);
std::string escape_char(char character);
std::string indent(std::string input);
std::string join(std::vector<std::string> lines, std::string separator);
std::string join(std::vector<std::string> lines);
std::string character_code(char character);
}
}