🔥 keyword and keypattern functions

Just make strings have higher precedence than regexps.

include/tree_sitter/compiler.h

@@ -21,8 +21,6 @@ rule_ptr seq(const std::vector<rule_ptr> &);
 rule_ptr sym(const std::string &);
 rule_ptr pattern(const std::string &);
 rule_ptr str(const std::string &);
-rule_ptr keyword(const std::string &);
-rule_ptr keypattern(const std::string &);
 rule_ptr err(const rule_ptr &);
 rule_ptr left_assoc(int precedence, const rule_ptr &);
 rule_ptr right_assoc(int precedence, const rule_ptr &);

spec/compiler/prepare_grammar/expand_tokens_spec.cc

@@ -24,7 +24,7 @@ describe("expand_tokens", []() {
       AssertThat(result.first.rules, Equals(rule_list({
           { "rule_A", seq({
               i_sym(10),
-              seq({ character({ 'x' }), character({ 'y' }), character({ 'z' }) }),
+              token(left_assoc(1, seq({ character({ 'x' }), character({ 'y' }), character({ 'z' }) }))),
               i_sym(11) }) },
       })));
     });
@@ -38,10 +38,10 @@ describe("expand_tokens", []() {
       auto result = expand_tokens(grammar);
 
       AssertThat(result.first.rules, Equals(rule_list({
-          { "rule_A", seq({
+          { "rule_A", token(left_assoc(1, seq({
               character({ 945 }),
               character({ ' ' }),
-              character({ 946 }) }) }
+              character({ 946 }) }))) }
       })));
     });
   });

spec/fixtures/grammars/golang.cc

@@ -18,10 +18,10 @@ extern const Grammar golang = Grammar({
         repeat(sym("imports_block")),
         repeat(sym("declaration")) }) },
     { "package_directive", seq({
-        keyword("package"),
+        str("package"),
         sym("package_name") }) },
     { "imports_block", seq({
-        keyword("import"),
+        str("import"),
         choice({
             in_parens(err(repeat(sym("package_import")))),
             sym("package_import") }) }) },
@@ -33,11 +33,11 @@ extern const Grammar golang = Grammar({
 
     // Declarations
     { "type_declaration", terminated(seq({
-        keyword("type"),
+        str("type"),
         sym("type_name"),
         sym("type_expression") })) },
     { "var_declaration", terminated(seq({
-        keyword("var"),
+        str("var"),
         sym("var_name"),
         choice({
             seq({
@@ -46,7 +46,7 @@ extern const Grammar golang = Grammar({
                 sym("expression") }),
             sym("type_expression") }) })) },
     { "func_declaration", terminated(seq({
-        keyword("func"),
+        str("func"),
         sym("var_name"),
         sym("_func_signature"),
         sym("block_statement") })) },
@@ -61,22 +61,22 @@ extern const Grammar golang = Grammar({
 
     // Type expressions
     { "pointer_type", seq({
-        keyword("*"),
+        str("*"),
         sym("type_expression") }) },
     { "map_type", seq({
-        keyword("map"),
+        str("map"),
         in_brackets(sym("type_expression")),
         sym("type_expression") }) },
     { "slice_type", seq({
         in_brackets(blank()),
         sym("type_expression") }) },
     { "struct_type", seq({
-        keyword("struct"),
+        str("struct"),
         in_braces(repeat(seq({
             sym("var_name"),
             sym("type_expression") }))) }) },
     { "interface_type", seq({
-        keyword("interface"),
+        str("interface"),
         in_braces(repeat(seq({
             sym("var_name"),
             sym("_func_signature") }))) }) },
@@ -89,7 +89,7 @@ extern const Grammar golang = Grammar({
         sym("range_statement"),
         sym("if_statement") }) },
     { "return_statement", terminated(seq({
-        keyword("return"),
+        str("return"),
         comma_sep(sym("expression")) })) },
     { "declaration_statement", choice({
         sym("var_declaration"),
@@ -98,19 +98,19 @@ extern const Grammar golang = Grammar({
             str(":="),
             sym("expression") })) }) },
     { "range_statement", seq({
-        keyword("for"),
+        str("for"),
         sym("var_name"),
-        optional(seq({ keyword(","), sym("var_name") })),
+        optional(seq({ str(","), sym("var_name") })),
         str(":="),
-        keyword("range"),
+        str("range"),
         sym("expression"),
         sym("block_statement") }) },
     { "if_statement", seq({
-        keyword("if"),
+        str("if"),
         sym("expression"),
         sym("block_statement"),
         optional(seq({
-            keyword("else"),
+            str("else"),
             choice({
                 sym("if_statement"),
                 sym("block_statement") }) })) }) },
@@ -165,7 +165,7 @@ extern const Grammar golang = Grammar({
     { "type_name", sym("_identifier") },
     { "_identifier", pattern("\\a[\\w_]*") },
     { "number", pattern("\\d+(\\.\\d+)?") },
-    { "comment", keypattern("//[^\n]*") },
+    { "comment", pattern("//[^\n]*") },
 }).ubiquitous_tokens({
     sym("comment"),
     sym("_line_break"),

spec/fixtures/grammars/helpers.cc

@@ -5,7 +5,7 @@ namespace tree_sitter_examples {
 using namespace tree_sitter::rules;
 
 rule_ptr comma_sep1(rule_ptr element) {
-  return seq({ element, repeat(seq({ keyword(","), element })) });
+  return seq({ element, repeat(seq({ str(","), element })) });
 }
 
 rule_ptr comma_sep(rule_ptr element) {
@@ -31,20 +31,20 @@ rule_ptr in_brackets(rule_ptr rule) {
 rule_ptr infix_op(std::string op, std::string rule_name, int precedence) {
   return left_assoc(precedence, seq({
       sym(rule_name),
-      keyword(op),
+      str(op),
       sym(rule_name) }));
 }
 
 rule_ptr prefix_op(std::string op, std::string rule_name, int precedence) {
   return right_assoc(precedence, seq({
-      keyword(op),
+      str(op),
       sym(rule_name) }));
 }
 
 rule_ptr postfix_op(std::string op, std::string rule_name, int precedence) {
   return left_assoc(precedence, seq({
       sym(rule_name),
-      keyword(op) }));
+      str(op) }));
 }
 
 rule_ptr delimited(std::string delimiter) {

spec/fixtures/grammars/javascript.cc

@@ -60,7 +60,7 @@ extern const Grammar javascript = Grammar({
     { "expression_statement", terminated(err(sym("expression"))) },
 
     { "var_declaration", terminated(seq({
-        keyword("var"),
+        str("var"),
         comma_sep1(err(choice({
             sym("identifier"),
             sym("var_assignment") }))) })) },
@@ -69,22 +69,22 @@ extern const Grammar javascript = Grammar({
         in_braces(err(repeat(sym("statement"))))) },
 
     { "if_statement", right_assoc(0, seq({
-        keyword("if"),
+        str("if"),
         sym("_paren_expression"),
         sym("statement"),
         optional(seq({
-            keyword("else"),
+            str("else"),
             sym("statement") })) })) },
 
     { "switch_statement", seq({
-        keyword("switch"),
+        str("switch"),
         sym("_paren_expression"),
         str("{"),
         repeat(choice({ sym("case"), sym("default") })),
         str("}") }) },
 
     { "for_statement", seq({
-        keyword("for"),
+        str("for"),
         str("("),
         choice({
             sym("var_declaration"),
@@ -96,45 +96,45 @@ extern const Grammar javascript = Grammar({
         sym("statement") }) },
 
     { "for_in_statement", seq({
-        keyword("for"),
+        str("for"),
         str("("),
-        optional(keyword("var")),
+        optional(str("var")),
         left_assoc(PREC_REL, seq({
           sym("identifier"),
-          keyword("in"),
+          str("in"),
           sym("expression") })),
         str(")"),
         sym("statement") }) },
 
     { "while_statement", seq({
-        keyword("while"),
+        str("while"),
         sym("_paren_expression"),
         sym("statement") }) },
 
     // { "do_statement", seq({
-        // keyword("do"),
+        // str("do"),
         // sym("statement"),
-        // keyword("while"),
+        // str("while"),
         // sym("_paren_expression") })},
 
     { "try_statement", seq({
-        keyword("try"),
+        str("try"),
         sym("statement_block"),
         optional(sym("catch")),
         optional(sym("finally")) }) },
 
     { "return_statement", terminated(seq({
-        keyword("return"),
+        str("return"),
         optional(sym("expression")) })) },
 
     { "throw_statement", terminated(seq({
-        keyword("throw"),
+        str("throw"),
         sym("expression") })) },
 
-    { "break_statement", terminated(keyword("break")) },
+    { "break_statement", terminated(str("break")) },
 
     { "delete_statement", terminated(seq({
-        keyword("delete"),
+        str("delete"),
         choice({ sym("member_access"), sym("subscript_access") }) })) },
 
     /*
@@ -142,25 +142,25 @@ extern const Grammar javascript = Grammar({
      */
 
     { "case", seq({
-        keyword("case"),
+        str("case"),
         sym("expression"),
         str(":"),
         repeat(sym("statement")) }) },
 
     { "default", seq({
-        keyword("default"),
+        str("default"),
         str(":"),
         repeat(sym("statement")) }) },
 
     { "catch", seq({
-        keyword("catch"),
+        str("catch"),
         str("("),
         err(sym("identifier")),
         str(")"),
         sym("statement_block") }) },
 
     { "finally", seq({
-        keyword("finally"),
+        str("finally"),
         sym("statement_block") }) },
 
     { "var_assignment", seq({
@@ -208,7 +208,7 @@ extern const Grammar javascript = Grammar({
     { "array", in_brackets(comma_sep(err(sym("expression")))) },
 
     { "function_expression", seq({
-        keyword("function"),
+        str("function"),
         optional(sym("identifier")),
         str("("),
         optional(sym("formal_parameters")),
@@ -223,10 +223,10 @@ extern const Grammar javascript = Grammar({
 
     { "constructor_call", choice({
         left_assoc(PREC_SHORT_NEW, seq({
-          keyword("new"),
+          str("new"),
           sym("expression") })),
         left_assoc(PREC_FULL_NEW, seq({
-          keyword("new"),
+          str("new"),
           sym("expression"),
           str("("),
           err(optional(sym("arguments"))),
@@ -306,7 +306,7 @@ extern const Grammar javascript = Grammar({
     { "type_op", choice({
         left_assoc(PREC_REL, seq({
           choice({ sym("expression"), sym("identifier") }),
-          keyword("in"),
+          str("in"),
           sym("expression") })),
         infix_op("instanceof", "expression", PREC_REL),
         prefix_op("typeof", "expression", PREC_TYPE) }) },
@@ -332,10 +332,10 @@ extern const Grammar javascript = Grammar({
 
     { "identifier", pattern("[\\a_$][\\w_$]*") },
 
-    { "null", keyword("null") },
-    { "undefined", keyword("undefined") },
-    { "true", keyword("true") },
-    { "false", keyword("false") },
+    { "null", str("null") },
+    { "undefined", str("undefined") },
+    { "true", str("true") },
+    { "false", str("false") },
     { "_line_break", str("\n") },
 
     /*

spec/fixtures/grammars/json.cc

@@ -22,9 +22,9 @@ extern const Grammar json = Grammar({
     { "array", in_brackets(comma_sep(err(sym("value")))) },
     { "string", pattern("\"([^\"]|\\\\\")*\"") },
     { "number", pattern("\\d+(\\.\\d+)?") },
-    { "null", keyword("null") },
-    { "true", keyword("true") },
-    { "false", keyword("false") },
+    { "null", str("null") },
+    { "true", str("true") },
+    { "false", str("false") },
 }).ubiquitous_tokens({
     pattern("\\s"),
 });

src/compiler/rules/rules.cc

@@ -24,8 +24,6 @@ using std::set;
 using std::vector;
 using std::map;
 
-static const int KEYWORD_PRECEDENCE = 100;
-
 static rule_ptr metadata(rule_ptr rule, map<MetadataKey, int> values) {
   return std::make_shared<Metadata>(rule, values);
 }
@@ -44,14 +42,8 @@ rule_ptr sym(const string &name) { return make_shared<NamedSymbol>(name); }
 
 rule_ptr pattern(const string &value) { return make_shared<Pattern>(value); }
 
-rule_ptr str(const string &value) { return make_shared<String>(value); }
-
-rule_ptr keyword(const string &value) {
-  return token(left_assoc(KEYWORD_PRECEDENCE, str(value)));
-}
-
-rule_ptr keypattern(const string &value) {
-  return token(left_assoc(KEYWORD_PRECEDENCE, pattern(value)));
+rule_ptr str(const string &value) {
+  return token(left_assoc(1, make_shared<String>(value)));
 }
 
 rule_ptr err(const rule_ptr &rule) {