Unify ubiquitous tokens and lexical separators in API

examples/grammars/arithmetic.cc

@@ -28,6 +28,9 @@ extern const Grammar arithmetic = Grammar({
     { "variable", pattern("\\a[\\w_]*") },
 
     { "comment", pattern("#.*") },
-}).ubiquitous_tokens({ "comment" });
+}).ubiquitous_tokens({
+    sym("comment"),
+    pattern("\\s"),
+});
 
 }  // namespace tree_sitter_examples

examples/grammars/golang.cc

@@ -166,8 +166,10 @@ extern const Grammar golang = Grammar({
     { "_identifier", pattern("\\a[\\w_]*") },
     { "number", pattern("\\d+(\\.\\d+)?") },
     { "comment", keypattern("//[^\n]*") },
-})
-    .ubiquitous_tokens({ "comment", "_line_break" })
-    .separators({ ' ', '\t', '\r' });
+}).ubiquitous_tokens({
+    sym("comment"),
+    sym("_line_break"),
+    pattern("[ \t\r]"),
+});
 
 }  // namespace tree_sitter_examples

examples/grammars/javascript.cc

@@ -213,8 +213,10 @@ extern const Grammar javascript = Grammar({
     { "null", keyword("null") },
     { "true", keyword("true") },
     { "false", keyword("false") },
-})
-    .ubiquitous_tokens({ "comment", "_line_break" })
-    .separators({ ' ', '\t', '\r' });
+}).ubiquitous_tokens({
+    sym("comment"),
+    sym("_line_break"),
+    pattern("[ \t\r]"),
+});
 
 }  // namespace tree_sitter_examples

examples/grammars/json.cc

@@ -6,7 +6,7 @@ namespace tree_sitter_examples {
 using tree_sitter::Grammar;
 using namespace tree_sitter::rules;
 
-extern const Grammar json({
+extern const Grammar json = Grammar({
     { "value", choice({
         sym("object"),
         sym("array"),
@@ -25,6 +25,8 @@ extern const Grammar json({
     { "null", keyword("null") },
     { "true", keyword("true") },
     { "false", keyword("false") },
+}).ubiquitous_tokens({
+    pattern("\\s"),
 });
 
 }  // namespace tree_sitter_examples