From 49ad910474cc956549f28990655b5018031de2d2 Mon Sep 17 00:00:00 2001
From: Max Brunsfeld <maxbrunsfeld@gmail.com>
Date: Wed, 19 Feb 2014 12:48:38 -0800
Subject: [PATCH] Skip whitespace by default in lexer

---
 include/tree_sitter/parser.h     | 7 +++++++
 spec/runtime/arithmetic_spec.cpp | 8 ++++----
 spec/runtime/json_spec.cpp       | 6 +++---
 todo.md                          | 1 -
 4 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h
index 2369123e..930f387b 100644
--- a/include/tree_sitter/parser.h
+++ b/include/tree_sitter/parser.h
@@ -8,6 +8,7 @@ extern "C" {
 #include "./runtime.h"
 #include <stdio.h>
 #include <string.h>
+#include <ctype.h>
 
 //#define TS_DEBUG_PARSE
 //#define TS_DEBUG_LEX
@@ -147,6 +148,11 @@ static void TSParserAcceptInput(TSParser *parser) {
     DEBUG_PARSE("accept \n");
 }
 
+static void TSParserSkipWhitespace(TSParser *parser) {
+    while (isspace(parser->input[parser->position]))
+        parser->position++;
+}
+
 #pragma mark - DSL
 
 #define START_PARSER() \
@@ -154,6 +160,7 @@ TSParser p = TSParserMake(input), *parser = &p; \
 next_state:
 
 #define START_LEXER() \
+TSParserSkipWhitespace(parser); \
 next_state:
 
 #define LOOKAHEAD_SYM() \
diff --git a/spec/runtime/arithmetic_spec.cpp b/spec/runtime/arithmetic_spec.cpp
index 561f1e77..c68096c0 100644
--- a/spec/runtime/arithmetic_spec.cpp
+++ b/spec/runtime/arithmetic_spec.cpp
@@ -25,21 +25,21 @@ describe("arithmetic", []() {
     });
 
     it("parses products of variables", [&]() {
-        TSDocumentSetText(document, "x+y");
+        TSDocumentSetText(document, "x + y");
         AssertThat(string(TSDocumentToString(document)), Equals(
             "(expression (term (factor (variable))) (plus) (term (factor (variable))))"));
         
-        TSDocumentSetText(document, "x*y");
+        TSDocumentSetText(document, "x * y");
         AssertThat(string(TSDocumentToString(document)), Equals(
             "(expression (term (factor (variable)) (times) (factor (variable))))"));
     });
     
     it("parses complex trees", [&]() {
-        TSDocumentSetText(document, "x*y+z*a");
+        TSDocumentSetText(document, "x * y + z * a");
         AssertThat(string(TSDocumentToString(document)), Equals(
             "(expression (term (factor (variable)) (times) (factor (variable))) (plus) (term (factor (variable)) (times) (factor (variable))))"));
 
-        TSDocumentSetText(document, "x*(y+z)");
+        TSDocumentSetText(document, "x * (y + z)");
         AssertThat(string(TSDocumentToString(document)), Equals(
             "(expression (term (factor (variable)) (times) (factor (expression (term (factor (variable))) (plus) (term (factor (variable)))))))"));
     });
diff --git a/spec/runtime/json_spec.cpp b/spec/runtime/json_spec.cpp
index 6860d81c..8788f3d5 100644
--- a/spec/runtime/json_spec.cpp
+++ b/spec/runtime/json_spec.cpp
@@ -27,10 +27,10 @@ describe("json", []() {
         TSDocumentSetText(document, "{}");
         AssertThat(string(TSDocumentToString(document)), Equals("(value (object))"));
 
-        TSDocumentSetText(document, "{\"key1\":1}");
+        TSDocumentSetText(document, "{ \"key1\": 1 }");
         AssertThat(string(TSDocumentToString(document)), Equals("(value (object (string) (value (number))))"));
 
-        TSDocumentSetText(document, "{\"key1\":1,\"key2\":2}");
+        TSDocumentSetText(document, "{\"key1\": 1, \"key2\": 2 }");
         AssertThat(string(TSDocumentToString(document)), Equals("(value (object (string) (value (number)) (string) (value (number))))"));
     });
     
@@ -41,7 +41,7 @@ describe("json", []() {
         TSDocumentSetText(document, "[5]");
         AssertThat(string(TSDocumentToString(document)), Equals("(value (array (value (number))))"));
 
-        TSDocumentSetText(document, "[1,2,3]");
+        TSDocumentSetText(document, "[1, 2, 3]");
         AssertThat(string(TSDocumentToString(document)), Equals("(value (array (value (number)) (value (number)) (value (number))))"));
     });
 });
diff --git a/todo.md b/todo.md
index 95e85e5e..caf610a8 100644
--- a/todo.md
+++ b/todo.md
@@ -2,7 +2,6 @@ TODO
 ====
 
 ## batch parsing
-- make it easy to make whitespace-separated sequence rules
 - add comments to generated C code describing the generated tokens (regexp pattern)
 - fix any memory leaks
 - add special lexical behavior for indentation-aware languages