traxys/tree-sitter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Skip whitespace by default in lexer

Browse source
This commit is contained in:
Max Brunsfeld 2014-02-19 12:48:38 -08:00
parent 19804367b3
commit 49ad910474
4 changed files with 14 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

7
include/tree_sitter/parser.h
View file

@ -8,6 +8,7 @@ extern "C" {
#include "./runtime.h"
#include <stdio.h>
#include <string.h>
#include <ctype.h>
//#define TS_DEBUG_PARSE
//#define TS_DEBUG_LEX
@ -147,6 +148,11 @@ static void TSParserAcceptInput(TSParser *parser) {
DEBUG_PARSE("accept \n");
}
static void TSParserSkipWhitespace(TSParser *parser) {
while (isspace(parser->input[parser->position]))
parser->position++;
}
#pragma mark - DSL
#define START_PARSER() \
@ -154,6 +160,7 @@ TSParser p = TSParserMake(input), *parser = &p; \
next_state:
#define START_LEXER() \
TSParserSkipWhitespace(parser); \
next_state:
#define LOOKAHEAD_SYM() \

8
spec/runtime/arithmetic_spec.cpp
View file

@ -25,21 +25,21 @@ describe("arithmetic", []() {
});
it("parses products of variables", [&]() {
TSDocumentSetText(document, "x+y");
TSDocumentSetText(document, "x + y");
AssertThat(string(TSDocumentToString(document)), Equals(
"(expression (term (factor (variable))) (plus) (term (factor (variable))))"));
TSDocumentSetText(document, "x*y");
TSDocumentSetText(document, "x * y");
AssertThat(string(TSDocumentToString(document)), Equals(
"(expression (term (factor (variable)) (times) (factor (variable))))"));
});
it("parses complex trees", [&]() {
TSDocumentSetText(document, "x*y+z*a");
TSDocumentSetText(document, "x * y + z * a");
AssertThat(string(TSDocumentToString(document)), Equals(
"(expression (term (factor (variable)) (times) (factor (variable))) (plus) (term (factor (variable)) (times) (factor (variable))))"));
TSDocumentSetText(document, "x*(y+z)");
TSDocumentSetText(document, "x * (y + z)");
AssertThat(string(TSDocumentToString(document)), Equals(
"(expression (term (factor (variable)) (times) (factor (expression (term (factor (variable))) (plus) (term (factor (variable)))))))"));
});

6
spec/runtime/json_spec.cpp
View file

@ -27,10 +27,10 @@ describe("json", []() {
TSDocumentSetText(document, "{}");
AssertThat(string(TSDocumentToString(document)), Equals("(value (object))"));
TSDocumentSetText(document, "{\"key1\":1}");
TSDocumentSetText(document, "{ \"key1\": 1 }");
AssertThat(string(TSDocumentToString(document)), Equals("(value (object (string) (value (number))))"));
TSDocumentSetText(document, "{\"key1\":1,\"key2\":2}");
TSDocumentSetText(document, "{\"key1\": 1, \"key2\": 2 }");
AssertThat(string(TSDocumentToString(document)), Equals("(value (object (string) (value (number)) (string) (value (number))))"));
});
@ -41,7 +41,7 @@ describe("json", []() {
TSDocumentSetText(document, "[5]");
AssertThat(string(TSDocumentToString(document)), Equals("(value (array (value (number))))"));
TSDocumentSetText(document, "[1,2,3]");
TSDocumentSetText(document, "[1, 2, 3]");
AssertThat(string(TSDocumentToString(document)), Equals("(value (array (value (number)) (value (number)) (value (number))))"));
});
});

1
todo.md
View file

@ -2,7 +2,6 @@ TODO
====
## batch parsing
- make it easy to make whitespace-separated sequence rules
- add comments to generated C code describing the generated tokens (regexp pattern)
- fix any memory leaks
- add special lexical behavior for indentation-aware languages