From a62b7a70f3bc8686f0a83f620737b82a8580dd29 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 30 Oct 2019 17:11:15 -0700 Subject: [PATCH 1/5] Lexer: track EOF state without relying on null character as lookahead --- cli/src/tests/parser_test.rs | 38 ++++- lib/src/lexer.c | 319 +++++++++++++++++++++-------------- 2 files changed, 225 insertions(+), 132 deletions(-) diff --git a/cli/src/tests/parser_test.rs b/cli/src/tests/parser_test.rs index c9249eb8..694cbd1e 100644 --- a/cli/src/tests/parser_test.rs +++ b/cli/src/tests/parser_test.rs @@ -258,6 +258,36 @@ fn test_parsing_text_with_byte_order_mark() { assert_eq!(tree.root_node().start_byte(), 3); } +#[test] +fn test_parsing_invalid_chars_at_eof() { + let mut parser = Parser::new(); + parser.set_language(get_language("json")).unwrap(); + let tree = parser.parse(b"\xdf", None).unwrap(); + assert_eq!(tree.root_node().to_sexp(), "(ERROR (UNEXPECTED INVALID))"); +} + +#[test] +fn test_parsing_ends_when_input_callback_returns_empty() { + let mut parser = Parser::new(); + parser.set_language(get_language("javascript")).unwrap(); + let mut i = 0; + let source = b"abcdefghijklmnoqrs"; + let tree = parser + .parse_with( + &mut |offset, _| { + i += 1; + if offset >= 6 { + b"" + } else { + &source[offset..usize::min(source.len(), offset + 3)] + } + }, + None, + ) + .unwrap(); + assert_eq!(tree.root_node().end_byte(), 6); +} + // Incremental parsing #[test] @@ -928,10 +958,10 @@ fn test_parsing_with_a_newly_included_range() { assert_eq!( tree.changed_ranges(&first_tree).collect::>(), vec![Range { - start_byte: first_code_end_index + 1, - end_byte: second_code_end_index + 1, - start_point: Point::new(0, first_code_end_index + 1), - end_point: Point::new(0, second_code_end_index + 1), + start_byte: first_code_end_index, + end_byte: second_code_end_index, + start_point: Point::new(0, first_code_end_index), + end_point: Point::new(0, second_code_end_index), }] ); } diff --git a/lib/src/lexer.c b/lib/src/lexer.c index a6d99546..8257ff4e 100644 --- a/lib/src/lexer.c +++ b/lib/src/lexer.c @@ -4,23 +4,56 @@ #include "./length.h" #include "./unicode.h" -#define LOG(...) \ - if (self->logger.log) { \ - snprintf(self->debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, __VA_ARGS__); \ - self->logger.log(self->logger.payload, TSLogTypeLex, self->debug_buffer); \ +#define LOG(message, character) \ + if (self->logger.log) { \ + snprintf( \ + self->debug_buffer, \ + TREE_SITTER_SERIALIZATION_BUFFER_SIZE, \ + 32 <= character && character < 127 ? \ + message " character:'%c'" : \ + message " character:%d", \ + character \ + ); \ + self->logger.log( \ + self->logger.payload, \ + TSLogTypeLex, \ + self->debug_buffer \ + ); \ } -#define LOG_CHARACTER(message, character) \ - LOG( \ - 32 <= character && character < 127 ? \ - message " character:'%c'" : \ - message " character:%d", character \ - ) - -static const char empty_chunk[3] = { 0, 0 }; - static const int32_t BYTE_ORDER_MARK = 0xFEFF; +static const TSRange DEFAULT_RANGE = { + .start_point = { + .row = 0, + .column = 0, + }, + .end_point = { + .row = UINT32_MAX, + .column = UINT32_MAX, + }, + .start_byte = 0, + .end_byte = UINT32_MAX +}; + +// Check if the lexer has reached EOF. This state is stored +// by setting the lexer's `current_included_range_index` such that +// it has consumed all of its available ranges. +static bool ts_lexer__eof(const TSLexer *_self) { + Lexer *self = (Lexer *)_self; + return self->current_included_range_index == self->included_range_count; +} + +// Clear the currently stored chunk of source code, because the lexer's +// position has changed. +static void ts_lexer__clear_chunk(Lexer *self) { + self->chunk = NULL; + self->chunk_size = 0; + self->chunk_start = 0; +} + +// Call the lexer's input callback to obtain a new chunk of source code +// for the current position. static void ts_lexer__get_chunk(Lexer *self) { self->chunk_start = self->current_position.bytes; self->chunk = self->input.read( @@ -29,9 +62,15 @@ static void ts_lexer__get_chunk(Lexer *self) { self->current_position.extent, &self->chunk_size ); - if (!self->chunk_size) self->chunk = empty_chunk; + if (!self->chunk_size) { + self->current_included_range_index = self->included_range_count; + self->chunk = NULL; + } } +// Decode the next unicode character in the current chunk of source code. +// This assumes that the lexer has already retrieved a chunk of source +// code that spans the current position. static void ts_lexer__get_lookahead(Lexer *self) { uint32_t position_in_chunk = self->current_position.bytes - self->chunk_start; const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk; @@ -63,10 +102,17 @@ static void ts_lexer__get_lookahead(Lexer *self) { } } -static void ts_lexer__advance(TSLexer *payload, bool skip) { - Lexer *self = (Lexer *)payload; - if (self->chunk == empty_chunk) - return; +// Advance to the next character in the source code, retrieving a new +// chunk of source code if needed. +static void ts_lexer__advance(TSLexer *_self, bool skip) { + Lexer *self = (Lexer *)_self; + if (!self->chunk) return; + + if (skip) { + LOG("skip", self->data.lookahead); + } else { + LOG("consume", self->data.lookahead); + } if (self->lookahead_size) { self->current_position.bytes += self->lookahead_size; @@ -78,53 +124,65 @@ static void ts_lexer__advance(TSLexer *payload, bool skip) { } } - TSRange *current_range = &self->included_ranges[self->current_included_range_index]; - if (self->current_position.bytes == current_range->end_byte) { - self->current_included_range_index++; - if (self->current_included_range_index == self->included_range_count) { - self->data.lookahead = '\0'; - self->lookahead_size = 1; - return; - } else { - current_range++; - self->current_position = (Length) { - current_range->start_byte, - current_range->start_point, - }; + const TSRange *current_range = NULL; + if (self->current_included_range_index < self->included_range_count) { + current_range = &self->included_ranges[self->current_included_range_index]; + if (self->current_position.bytes == current_range->end_byte) { + self->current_included_range_index++; + if (self->current_included_range_index < self->included_range_count) { + current_range++; + self->current_position = (Length) { + current_range->start_byte, + current_range->start_point, + }; + } else { + current_range = NULL; + } } } - if (skip) { - LOG_CHARACTER("skip", self->data.lookahead); - self->token_start_position = self->current_position; + if (skip) self->token_start_position = self->current_position; + + if (current_range) { + if (self->current_position.bytes >= self->chunk_start + self->chunk_size) { + ts_lexer__get_chunk(self); + } + ts_lexer__get_lookahead(self); } else { - LOG_CHARACTER("consume", self->data.lookahead); - } - - if (self->current_position.bytes >= self->chunk_start + self->chunk_size) { - ts_lexer__get_chunk(self); - } - - ts_lexer__get_lookahead(self); -} - -static void ts_lexer__mark_end(TSLexer *payload) { - Lexer *self = (Lexer *)payload; - TSRange *current_included_range = &self->included_ranges[self->current_included_range_index]; - if (self->current_included_range_index > 0 && - self->current_position.bytes == current_included_range->start_byte) { - TSRange *previous_included_range = current_included_range - 1; - self->token_end_position = (Length) { - previous_included_range->end_byte, - previous_included_range->end_point, - }; - } else { - self->token_end_position = self->current_position; + ts_lexer__clear_chunk(self); + self->data.lookahead = '\0'; + self->lookahead_size = 1; } } -static uint32_t ts_lexer__get_column(TSLexer *payload) { - Lexer *self = (Lexer *)payload; +// Mark that a token match has completed. This can be called multiple +// times if a longer match is found later. +static void ts_lexer__mark_end(TSLexer *_self) { + Lexer *self = (Lexer *)_self; + if (!ts_lexer__eof(&self->data)) { + // If the lexer is right at the beginning of included range, + // then the token should be considered to end at the *end* of the + // previous included range, rather than here. + TSRange *current_included_range = &self->included_ranges[ + self->current_included_range_index + ]; + if ( + self->current_included_range_index > 0 && + self->current_position.bytes == current_included_range->start_byte + ) { + TSRange *previous_included_range = current_included_range - 1; + self->token_end_position = (Length) { + previous_included_range->end_byte, + previous_included_range->end_point, + }; + return; + } + } + self->token_end_position = self->current_position; +} + +static uint32_t ts_lexer__get_column(TSLexer *_self) { + Lexer *self = (Lexer *)_self; uint32_t goal_byte = self->current_position.bytes; self->current_position.bytes -= self->current_position.extent.column; @@ -136,25 +194,32 @@ static uint32_t ts_lexer__get_column(TSLexer *payload) { uint32_t result = 0; while (self->current_position.bytes < goal_byte) { - ts_lexer__advance(payload, false); + ts_lexer__advance(&self->data, false); result++; } return result; } -static bool ts_lexer__is_at_included_range_start(TSLexer *payload) { - const Lexer *self = (const Lexer *)payload; - TSRange *current_range = &self->included_ranges[self->current_included_range_index]; - return self->current_position.bytes == current_range->start_byte; +// Is the lexer at a boundary between two disjoint included ranges of +// source code? This is exposed as an API because some languages' external +// scanners need to perform custom actions at these bounaries. +static bool ts_lexer__is_at_included_range_start(const TSLexer *_self) { + const Lexer *self = (const Lexer *)_self; + if (self->current_included_range_index < self->included_range_count) { + TSRange *current_range = &self->included_ranges[self->current_included_range_index]; + return self->current_position.bytes == current_range->start_byte; + } else { + return false; + } } -// The lexer's methods are stored as a struct field so that generated -// parsers can call them without needing to be linked against this library. - void ts_lexer_init(Lexer *self) { *self = (Lexer) { .data = { + // The lexer's methods are stored as struct fields so that generated + // parsers can call them without needing to be linked against this + // library. .advance = ts_lexer__advance, .mark_end = ts_lexer__mark_end, .get_column = ts_lexer__get_column, @@ -163,40 +228,34 @@ void ts_lexer_init(Lexer *self) { .result_symbol = 0, }, .chunk = NULL, + .chunk_size = 0, .chunk_start = 0, - .current_position = {UINT32_MAX, {0, 0}}, + .current_position = {0, {0, 0}}, .logger = { .payload = NULL, .log = NULL }, + .included_ranges = NULL, + .included_range_count = 0, .current_included_range_index = 0, }; - - self->included_ranges = NULL; ts_lexer_set_included_ranges(self, NULL, 0); - ts_lexer_reset(self, length_zero()); } void ts_lexer_delete(Lexer *self) { ts_free(self->included_ranges); } -void ts_lexer_set_input(Lexer *self, TSInput input) { - self->input = input; - self->data.lookahead = 0; - self->lookahead_size = 0; - self->chunk = 0; - self->chunk_start = 0; - self->chunk_size = 0; -} - static void ts_lexer_goto(Lexer *self, Length position) { + self->current_position = position; bool found_included_range = false; + + // Move to the first valid position at or after the given position. for (unsigned i = 0; i < self->included_range_count; i++) { TSRange *included_range = &self->included_ranges[i]; if (included_range->end_byte > position.bytes) { if (included_range->start_byte > position.bytes) { - position = (Length) { + self->current_position = (Length) { .bytes = included_range->start_byte, .extent = included_range->start_point, }; @@ -208,46 +267,61 @@ static void ts_lexer_goto(Lexer *self, Length position) { } } - if (!found_included_range) { + if (found_included_range) { + // If the current position is outside of the current chunk of text, + // then clear out the current chunk of text. + if (self->chunk && ( + position.bytes < self->chunk_start || + position.bytes >= self->chunk_start + self->chunk_size + )) { + ts_lexer__clear_chunk(self); + } + + self->lookahead_size = 0; + self->data.lookahead = '\0'; + } + + // If the given position is beyond any of included ranges, move to the EOF + // state - past the end of the included ranges. + else { + self->current_included_range_index = self->included_range_count; TSRange *last_included_range = &self->included_ranges[self->included_range_count - 1]; - position = (Length) { + self->current_position = (Length) { .bytes = last_included_range->end_byte, .extent = last_included_range->end_point, }; - self->chunk = empty_chunk; - self->chunk_start = position.bytes; - self->chunk_size = 2; + ts_lexer__clear_chunk(self); + self->lookahead_size = 1; + self->data.lookahead = '\0'; } - - self->token_start_position = position; - self->token_end_position = LENGTH_UNDEFINED; - self->current_position = position; - - if (self->chunk && (position.bytes < self->chunk_start || - position.bytes >= self->chunk_start + self->chunk_size)) { - self->chunk = 0; - self->chunk_start = 0; - self->chunk_size = 0; - } - - self->lookahead_size = 0; - self->data.lookahead = 0; } +void ts_lexer_set_input(Lexer *self, TSInput input) { + self->input = input; + ts_lexer__clear_chunk(self); + ts_lexer_goto(self, self->current_position); +} + +// Move the lexer to the given position. This doesn't do any work +// if the parser is already at the given position. void ts_lexer_reset(Lexer *self, Length position) { - if (position.bytes != self->current_position.bytes) ts_lexer_goto(self, position); + if (position.bytes != self->current_position.bytes) { + ts_lexer_goto(self, position); + } } void ts_lexer_start(Lexer *self) { self->token_start_position = self->current_position; self->token_end_position = LENGTH_UNDEFINED; self->data.result_symbol = 0; - if (!self->chunk) ts_lexer__get_chunk(self); - if (!self->lookahead_size) ts_lexer__get_lookahead(self); - if ( - self->current_position.bytes == 0 && - self->data.lookahead == BYTE_ORDER_MARK - ) ts_lexer__advance((TSLexer *)self, true); + if (!ts_lexer__eof(&self->data)) { + if (!self->chunk_size) ts_lexer__get_chunk(self); + if (!self->lookahead_size) ts_lexer__get_lookahead(self); + if ( + self->current_position.bytes == 0 && + self->data.lookahead == BYTE_ORDER_MARK + ) ts_lexer__advance(&self->data, true); + } } void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) { @@ -271,8 +345,8 @@ void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) { } void ts_lexer_advance_to_end(Lexer *self) { - while (self->data.lookahead != 0) { - ts_lexer__advance((TSLexer *)self, false); + while (self->chunk) { + ts_lexer__advance(&self->data, false); } } @@ -280,30 +354,19 @@ void ts_lexer_mark_end(Lexer *self) { ts_lexer__mark_end(&self->data); } -static const TSRange DEFAULT_RANGES[] = { - { - .start_point = { - .row = 0, - .column = 0, - }, - .end_point = { - .row = UINT32_MAX, - .column = UINT32_MAX, - }, - .start_byte = 0, - .end_byte = UINT32_MAX - } -}; - -void ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, uint32_t count) { +void ts_lexer_set_included_ranges( + Lexer *self, + const TSRange *ranges, + uint32_t count +) { if (count == 0 || !ranges) { - ranges = DEFAULT_RANGES; + ranges = &DEFAULT_RANGE; count = 1; } - size_t sz = count * sizeof(TSRange); - self->included_ranges = ts_realloc(self->included_ranges, sz); - memcpy(self->included_ranges, ranges, sz); + size_t size = count * sizeof(TSRange); + self->included_ranges = ts_realloc(self->included_ranges, size); + memcpy(self->included_ranges, ranges, size); self->included_range_count = count; ts_lexer_goto(self, self->current_position); } From d3b7caa565286dd69c3d138574c9a660f3c74da5 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 30 Oct 2019 17:11:15 -0700 Subject: [PATCH 2/5] Add a TSLexer.eof() API, use it in generated parsers --- .../generate/build_tables/build_lex_table.rs | 20 +++++++++++-------- cli/src/generate/render.rs | 4 ++++ cli/src/generate/tables.rs | 1 + lib/include/tree_sitter/parser.h | 7 +++++-- lib/src/lexer.c | 1 + lib/src/lexer.h | 2 +- 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/cli/src/generate/build_tables/build_lex_table.rs b/cli/src/generate/build_tables/build_lex_table.rs index 21594253..b365feb1 100644 --- a/cli/src/generate/build_tables/build_lex_table.rs +++ b/cli/src/generate/build_tables/build_lex_table.rs @@ -2,7 +2,7 @@ use super::coincident_tokens::CoincidentTokenIndex; use super::token_conflicts::TokenConflictMap; use crate::generate::dedup::split_state_id_groups; use crate::generate::grammars::{LexicalGrammar, SyntaxGrammar}; -use crate::generate::nfa::{CharacterSet, NfaCursor}; +use crate::generate::nfa::NfaCursor; use crate::generate::rules::{Symbol, TokenSet}; use crate::generate::tables::{AdvanceAction, LexState, LexTable, ParseStateId, ParseTable}; use log::info; @@ -189,13 +189,10 @@ impl<'a> LexTableBuilder<'a> { // character that leads to the empty set of NFA states. if eof_valid { let (next_state_id, _) = self.add_state(Vec::new(), false); - self.table.states[state_id].advance_actions.push(( - CharacterSet::empty().add_char('\0'), - AdvanceAction { - state: next_state_id, - in_main_token: true, - }, - )); + self.table.states[state_id].eof_action = Some(AdvanceAction { + state: next_state_id, + in_main_token: true, + }); } for transition in transitions { @@ -273,6 +270,7 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) { let signature = ( i == 0, state.accept_action, + state.eof_action.is_some(), state .advance_actions .iter() @@ -320,6 +318,9 @@ fn minimize_lex_table(table: &mut LexTable, parse_table: &mut ParseTable) { for (_, advance_action) in new_state.advance_actions.iter_mut() { advance_action.state = group_ids_by_state_id[advance_action.state]; } + if let Some(eof_action) = &mut new_state.eof_action { + eof_action.state = group_ids_by_state_id[eof_action.state]; + } new_states.push(new_state); } @@ -364,6 +365,9 @@ fn sort_states(table: &mut LexTable, parse_table: &mut ParseTable) { for (_, advance_action) in state.advance_actions.iter_mut() { advance_action.state = new_ids_by_old_id[advance_action.state]; } + if let Some(eof_action) = &mut state.eof_action { + eof_action.state = new_ids_by_old_id[eof_action.state]; + } state }) .collect(); diff --git a/cli/src/generate/render.rs b/cli/src/generate/render.rs index e2afa893..be4b5681 100644 --- a/cli/src/generate/render.rs +++ b/cli/src/generate/render.rs @@ -540,6 +540,10 @@ impl Generator { add_line!(self, "ACCEPT_TOKEN({});", self.symbol_ids[&accept_action]); } + if let Some(eof_action) = state.eof_action { + add_line!(self, "if (eof) ADVANCE({});", eof_action.state); + } + let mut ruled_out_characters = HashSet::new(); for (characters, action) in state.advance_actions { let previous_length = self.buffer.len(); diff --git a/cli/src/generate/tables.rs b/cli/src/generate/tables.rs index 6f4c34a6..15b18a97 100644 --- a/cli/src/generate/tables.rs +++ b/cli/src/generate/tables.rs @@ -77,6 +77,7 @@ pub(crate) struct AdvanceAction { #[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] pub(crate) struct LexState { pub accept_action: Option, + pub eof_action: Option, pub advance_actions: Vec<(CharacterSet, AdvanceAction)>, } diff --git a/lib/include/tree_sitter/parser.h b/lib/include/tree_sitter/parser.h index 974a7ca5..73e7889c 100644 --- a/lib/include/tree_sitter/parser.h +++ b/lib/include/tree_sitter/parser.h @@ -45,7 +45,8 @@ struct TSLexer { void (*advance)(TSLexer *, bool); void (*mark_end)(TSLexer *); uint32_t (*get_column)(TSLexer *); - bool (*is_at_included_range_start)(TSLexer *); + bool (*is_at_included_range_start)(const TSLexer *); + bool (*eof)(const TSLexer *); }; typedef enum { @@ -126,13 +127,15 @@ struct TSLanguage { #define START_LEXER() \ bool result = false; \ bool skip = false; \ + bool eof = false; \ int32_t lookahead; \ goto start; \ next_state: \ lexer->advance(lexer, skip); \ start: \ skip = false; \ - lookahead = lexer->lookahead; + lookahead = lexer->lookahead; \ + eof = lexer->eof(lexer); #define ADVANCE(state_value) \ { \ diff --git a/lib/src/lexer.c b/lib/src/lexer.c index 8257ff4e..e2ca8519 100644 --- a/lib/src/lexer.c +++ b/lib/src/lexer.c @@ -224,6 +224,7 @@ void ts_lexer_init(Lexer *self) { .mark_end = ts_lexer__mark_end, .get_column = ts_lexer__get_column, .is_at_included_range_start = ts_lexer__is_at_included_range_start, + .eof = ts_lexer__eof, .lookahead = 0, .result_symbol = 0, }, diff --git a/lib/src/lexer.h b/lib/src/lexer.h index f523d88f..8cd9c267 100644 --- a/lib/src/lexer.h +++ b/lib/src/lexer.h @@ -16,7 +16,7 @@ typedef struct { Length token_start_position; Length token_end_position; - TSRange * included_ranges; + TSRange *included_ranges; size_t included_range_count; size_t current_included_range_index; From d765332c612c9b2cc38717439bca66dd463802c1 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 31 Oct 2019 14:32:10 -0700 Subject: [PATCH 3/5] Don't rely on new eof ABI in parsers unless --next-abi is passed --- cli/src/generate/render.rs | 7 +++++++ lib/include/tree_sitter/parser.h | 3 +-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cli/src/generate/render.rs b/cli/src/generate/render.rs index be4b5681..7b663560 100644 --- a/cli/src/generate/render.rs +++ b/cli/src/generate/render.rs @@ -513,6 +513,13 @@ impl Generator { ); indent!(self); add_line!(self, "START_LEXER();"); + + if self.next_abi { + add_line!(self, "eof = lexer->eof(lexer);"); + } else { + add_line!(self, "eof = lookahead == 0;"); + } + add_line!(self, "switch (state) {{"); indent!(self); diff --git a/lib/include/tree_sitter/parser.h b/lib/include/tree_sitter/parser.h index 73e7889c..c28e9d5c 100644 --- a/lib/include/tree_sitter/parser.h +++ b/lib/include/tree_sitter/parser.h @@ -134,8 +134,7 @@ struct TSLanguage { lexer->advance(lexer, skip); \ start: \ skip = false; \ - lookahead = lexer->lookahead; \ - eof = lexer->eof(lexer); + lookahead = lexer->lookahead; #define ADVANCE(state_value) \ { \ From 39ceff1908e67862fcb01a69424fb95fcac80b09 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 31 Oct 2019 14:22:42 -0700 Subject: [PATCH 4/5] Restore json corpus tests, which accidentally weren't running --- cli/src/tests/corpus_test.rs | 1 + test/fixtures/error_corpus/json_errors.txt | 23 +++++++++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/cli/src/tests/corpus_test.rs b/cli/src/tests/corpus_test.rs index a8adce5f..5fed8607 100644 --- a/cli/src/tests/corpus_test.rs +++ b/cli/src/tests/corpus_test.rs @@ -21,6 +21,7 @@ const LANGUAGES: &'static [&'static str] = &[ "go", "html", "javascript", + "json", "python", ]; diff --git a/test/fixtures/error_corpus/json_errors.txt b/test/fixtures/error_corpus/json_errors.txt index 80a8d407..b26aaf1d 100644 --- a/test/fixtures/error_corpus/json_errors.txt +++ b/test/fixtures/error_corpus/json_errors.txt @@ -26,7 +26,7 @@ errors inside arrays --- -(value (array +(document (array (number) (ERROR) (number))) @@ -39,8 +39,8 @@ errors inside objects --- -(value (object - (pair (string) (number)) +(document (object + (pair (string (string_content)) (number)) (ERROR (UNEXPECTED 'o')))) ========================================== @@ -51,9 +51,18 @@ errors inside nested objects --- -(value (object - (pair (string) (object - (pair (string) (number)) +(document (object + (pair (string (string_content)) (object + (pair (string (string_content)) (number)) (ERROR (number)))) - (pair (string) (number)) + (pair (string (string_content)) (number)) (ERROR))) + +=============================== +incomplete tokens at EOF +======================== + +nul +--- + +(ERROR (UNEXPECTED EOF)) From 32621cde2b01813a6b51823442ed486add61ded5 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 31 Oct 2019 14:22:52 -0700 Subject: [PATCH 5/5] Remove .gitmodules file --- .gitmodules | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .gitmodules diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29b..00000000