From 45c40c874289e07bec1eecc224453120378b5e5b Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 17 Jul 2017 17:46:46 -0700 Subject: [PATCH] Update test grammars to use new serialization API --- .../scanner.c | 25 ++++++++++++---- .../external_and_internal_tokens/scanner.c | 29 ++++++++++++++----- .../external_extra_tokens/scanner.c | 27 ++++++++++++----- .../test_grammars/external_tokens/scanner.c | 12 ++++++-- 4 files changed, 70 insertions(+), 23 deletions(-) diff --git a/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c b/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c index c940be1a..eb999d15 100644 --- a/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_and_internal_anonymous_tokens/scanner.c @@ -1,14 +1,27 @@ #include "../external_and_internal_tokens/scanner.c" -void *tree_sitter_external_and_internal_anonymous_tokens_external_scanner_create() { return NULL; } +void *tree_sitter_external_and_internal_anonymous_tokens_external_scanner_create() { + return NULL; +} -void tree_sitter_external_and_internal_anonymous_tokens_external_scanner_destroy(void *payload) {} +void tree_sitter_external_and_internal_anonymous_tokens_external_scanner_destroy( + void *payload +) {} -void tree_sitter_external_and_internal_anonymous_tokens_external_scanner_reset(void *payload) {} +void tree_sitter_external_and_internal_anonymous_tokens_external_scanner_reset( + void *payload +) {} -bool tree_sitter_external_and_internal_anonymous_tokens_external_scanner_serialize(void *payload, TSExternalTokenState state) { return true; } +unsigned tree_sitter_external_and_internal_anonymous_tokens_external_scanner_serialize( + void *payload, + char *buffer +) { return 0; } -void tree_sitter_external_and_internal_anonymous_tokens_external_scanner_deserialize(void *payload, TSExternalTokenState state) {} +void tree_sitter_external_and_internal_anonymous_tokens_external_scanner_deserialize( + void *payload, + const char *buffer, + unsigned length +) {} bool tree_sitter_external_and_internal_anonymous_tokens_external_scanner_scan( void *payload, @@ -20,4 +33,4 @@ bool tree_sitter_external_and_internal_anonymous_tokens_external_scanner_scan( lexer, whitelist ); -} \ No newline at end of file +} diff --git a/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c b/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c index 4d0acd0a..837a1d1a 100644 --- a/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_and_internal_tokens/scanner.c @@ -5,19 +5,34 @@ enum { LINE_BREAK }; -void *tree_sitter_external_and_internal_tokens_external_scanner_create() { return NULL; } +void *tree_sitter_external_and_internal_tokens_external_scanner_create() { + return NULL; +} -void tree_sitter_external_and_internal_tokens_external_scanner_destroy(void *payload) {} +void tree_sitter_external_and_internal_tokens_external_scanner_destroy( + void *payload +) {} -void tree_sitter_external_and_internal_tokens_external_scanner_reset(void *payload) {} +void tree_sitter_external_and_internal_tokens_external_scanner_reset( + void *payload +) {} -bool tree_sitter_external_and_internal_tokens_external_scanner_serialize(void *payload, TSExternalTokenState state) { return true; } +unsigned tree_sitter_external_and_internal_tokens_external_scanner_serialize( + void *payload, + char *buffer +) { return 0; } -void tree_sitter_external_and_internal_tokens_external_scanner_deserialize(void *payload, TSExternalTokenState state) {} +void tree_sitter_external_and_internal_tokens_external_scanner_deserialize( + void *payload, + const char *buffer, + unsigned length +) {} bool tree_sitter_external_and_internal_tokens_external_scanner_scan( - void *payload, TSLexer *lexer, const bool *whitelist) { - + void *payload, + TSLexer *lexer, + const bool *whitelist +) { // If a line-break is a valid lookahead token, only skip spaces. if (whitelist[LINE_BREAK]) { while (lexer->lookahead == ' ') { diff --git a/test/fixtures/test_grammars/external_extra_tokens/scanner.c b/test/fixtures/test_grammars/external_extra_tokens/scanner.c index 4bd3e22e..ac6d8407 100644 --- a/test/fixtures/test_grammars/external_extra_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_extra_tokens/scanner.c @@ -4,18 +4,32 @@ enum { COMMENT, }; -void *tree_sitter_external_extra_tokens_external_scanner_create() { return NULL; } +void *tree_sitter_external_extra_tokens_external_scanner_create() { + return NULL; +} -void tree_sitter_external_extra_tokens_external_scanner_destroy(void *payload) {} +void tree_sitter_external_extra_tokens_external_scanner_destroy( + void *payload) {} -void tree_sitter_external_extra_tokens_external_scanner_reset(void *payload) {} +void tree_sitter_external_extra_tokens_external_scanner_reset( + void *payload) {} -bool tree_sitter_external_extra_tokens_external_scanner_serialize(void *payload, TSExternalTokenState state) { return true; } +unsigned tree_sitter_external_extra_tokens_external_scanner_serialize( + void *payload, + char *buffer +) { return 0; } -void tree_sitter_external_extra_tokens_external_scanner_deserialize(void *payload, TSExternalTokenState state) {} +void tree_sitter_external_extra_tokens_external_scanner_deserialize( + void *payload, + const char *buffer, + unsigned length +) {} bool tree_sitter_external_extra_tokens_external_scanner_scan( - void *payload, TSLexer *lexer, const bool *whitelist) { + void *payload, + TSLexer *lexer, + const bool *valid_symbols +) { while (lexer->lookahead == ' ') { lexer->advance(lexer, true); @@ -33,4 +47,3 @@ bool tree_sitter_external_extra_tokens_external_scanner_scan( return false; } - diff --git a/test/fixtures/test_grammars/external_tokens/scanner.c b/test/fixtures/test_grammars/external_tokens/scanner.c index 7622e74d..3fea84b3 100644 --- a/test/fixtures/test_grammars/external_tokens/scanner.c +++ b/test/fixtures/test_grammars/external_tokens/scanner.c @@ -28,9 +28,16 @@ void tree_sitter_external_tokens_external_scanner_destroy(void *payload) { void tree_sitter_external_tokens_external_scanner_reset(void *payload) {} -bool tree_sitter_external_tokens_external_scanner_serialize(void *payload, TSExternalTokenState state) { return true; } +unsigned tree_sitter_external_tokens_external_scanner_serialize( + void *payload, + char *buffer +) { return true; } -void tree_sitter_external_tokens_external_scanner_deserialize(void *payload, TSExternalTokenState state) {} +void tree_sitter_external_tokens_external_scanner_deserialize( + void *payload, + const char *buffer, + unsigned length +) {} bool tree_sitter_external_tokens_external_scanner_scan( void *payload, TSLexer *lexer, const bool *whitelist) { @@ -111,4 +118,3 @@ bool tree_sitter_external_tokens_external_scanner_scan( return false; } -