Add token helper for building token rules

Now you can specify the structure of tokens using
all of the rule functions, not just `str` and `pattern`

src/compiler/rules/metadata.cc

@@ -26,7 +26,14 @@ namespace tree_sitter  {
         }
 
         rule_ptr Metadata::copy() const {
-            return make_shared<Metadata>(rule, value);
+            return make_shared<Metadata>(rule->copy(), value);
+        }
+
+        int Metadata::value_for(MetadataKey key) const {
+            auto pair = value.find(key);
+            return (pair != value.end()) ?
+                pair->second :
+                0;
         }
 
         std::string Metadata::to_string() const {

src/compiler/rules/metadata.h

@@ -9,7 +9,8 @@ namespace tree_sitter  {
     namespace rules {
         typedef enum {
             START_TOKEN,
-            PRECEDENCE
+            PRECEDENCE,
+            IS_TOKEN,
         } MetadataKey;
 
         class Metadata : public Rule {
@@ -21,6 +22,7 @@ namespace tree_sitter  {
             rule_ptr copy() const;
             std::string to_string() const;
             void accept(Visitor *visitor) const;
+            int value_for(MetadataKey key) const;
 
             const rule_ptr rule;
             const std::map<MetadataKey, int> value;

src/compiler/rules/rules.cc

@@ -23,6 +23,10 @@ namespace tree_sitter {
     using std::map;
 
     namespace rules {
+        static rule_ptr metadata(rule_ptr rule, map<MetadataKey, int> values) {
+            return std::make_shared<Metadata>(rule, values);
+        }
+
         rule_ptr blank() {
             return make_shared<Blank>();
         }
@@ -56,9 +60,11 @@ namespace tree_sitter {
         }
 
         rule_ptr prec(int precedence, rule_ptr rule) {
-            return std::make_shared<Metadata>(rule, map<MetadataKey, int>({
-                { PRECEDENCE, precedence }
-            }));
+            return metadata(rule, { { PRECEDENCE, precedence } });
+        }
+
+        rule_ptr token(rule_ptr rule) {
+            return metadata(rule, { { IS_TOKEN, 1 } });
         }
     }
 }