Revert "Remove the separator characters construct"
This reverts commit 5cd07648fd.
The separators construct is useful as an optimization. It turns out that
constructing a node for every chunk of whitespace in a document causes a
significant performance regression.
Conflicts:
src/compiler/build_tables/build_lex_table.cc
src/compiler/grammar.cc
src/runtime/parser.c
This commit is contained in:
parent
e941f8c175
commit
545e575508
43 changed files with 9065 additions and 11203 deletions
|
|
@ -208,13 +208,13 @@ extern const Grammar javascript = Grammar({
|
|||
delimited("\""),
|
||||
delimited("'") })) },
|
||||
{ "_line_break", str("\n") },
|
||||
{ "_whitespace", pattern("\\s+") },
|
||||
{ "identifier", pattern("[\\a_$][\\w_$]*") },
|
||||
{ "number", pattern("\\d+(\\.\\d+)?") },
|
||||
{ "null", keyword("null") },
|
||||
{ "true", keyword("true") },
|
||||
{ "false", keyword("false") },
|
||||
})
|
||||
.ubiquitous_tokens({ "comment", "_whitespace", "_line_break" });
|
||||
.ubiquitous_tokens({ "comment", "_line_break" })
|
||||
.separators({ ' ', '\t', '\r' });
|
||||
|
||||
} // namespace tree_sitter_examples
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue