diff --git a/include/tree_sitter/parser.h b/include/tree_sitter/parser.h index e03ba757..2b9fdec4 100644 --- a/include/tree_sitter/parser.h +++ b/include/tree_sitter/parser.h @@ -13,14 +13,21 @@ extern "C" { #define TS_DEBUG_BUFFER_SIZE 512 typedef struct TSTree TSTree; + typedef unsigned short TSStateId; +typedef enum { + TSNodeTypeNormal, + TSNodeTypeConcrete, + TSNodeTypeHidden, +} TSNodeType; + typedef struct TSLexer { // Public void (*start_fn)(struct TSLexer *, TSStateId); void (*start_token_fn)(struct TSLexer *); bool (*advance_fn)(struct TSLexer *, TSStateId); - TSTree *(*accept_fn)(struct TSLexer *, TSSymbol, int, const char *); + TSTree *(*accept_fn)(struct TSLexer *, TSSymbol, TSNodeType, const char *); // Private const char *chunk; @@ -63,7 +70,7 @@ typedef struct { struct TSLanguage { size_t symbol_count; const char **symbol_names; - const int *hidden_symbol_flags; + const TSNodeType *node_types; const TSParseAction **parse_table; const TSStateId *lex_states; TSTree *(*lex_fn)(TSLexer *, TSStateId); @@ -94,8 +101,8 @@ struct TSLanguage { GO_TO_STATE(state_index); \ } -#define ACCEPT_TOKEN(symbol) \ - return lexer->accept_fn(lexer, symbol, ts_hidden_symbol_flags[symbol], \ +#define ACCEPT_TOKEN(symbol) \ + return lexer->accept_fn(lexer, symbol, ts_node_types[symbol], \ ts_symbol_names[symbol]); #define LEX_ERROR() \ @@ -150,7 +157,7 @@ struct TSLanguage { #define EXPORT_LANGUAGE(language_name) \ static TSLanguage language = { \ .symbol_count = SYMBOL_COUNT, \ - .hidden_symbol_flags = ts_hidden_symbol_flags, \ + .node_types = ts_node_types, \ .parse_table = (const TSParseAction **)ts_parse_actions, \ .lex_states = ts_lex_states, \ .symbol_names = ts_symbol_names, \ diff --git a/spec/fixtures/parsers/arithmetic.c b/spec/fixtures/parsers/arithmetic.c index 76dbe16f..20b12e63 100644 --- a/spec/fixtures/parsers/arithmetic.c +++ b/spec/fixtures/parsers/arithmetic.c @@ -47,15 +47,27 @@ static const char *ts_symbol_names[] = { [sym_comment] = "comment", }; -static const int ts_hidden_symbol_flags[SYMBOL_COUNT] = { - [sym__expression] = 1, - [aux_sym_PLUS] = 1, - [aux_sym_DASH] = 1, - [aux_sym_STAR] = 1, - [aux_sym_SLASH] = 1, - [aux_sym_CARET] = 1, - [aux_sym_LPAREN] = 1, - [aux_sym_RPAREN] = 1, +static const TSNodeType ts_node_types[SYMBOL_COUNT] = { + [sym_program] = TSNodeTypeNormal, + [sym__expression] = TSNodeTypeHidden, + [sym_sum] = TSNodeTypeNormal, + [sym_difference] = TSNodeTypeNormal, + [sym_product] = TSNodeTypeNormal, + [sym_quotient] = TSNodeTypeNormal, + [sym_exponent] = TSNodeTypeNormal, + [sym_group] = TSNodeTypeNormal, + [ts_builtin_sym_error] = TSNodeTypeNormal, + [ts_builtin_sym_end] = TSNodeTypeHidden, + [aux_sym_PLUS] = TSNodeTypeConcrete, + [aux_sym_DASH] = TSNodeTypeConcrete, + [aux_sym_STAR] = TSNodeTypeConcrete, + [aux_sym_SLASH] = TSNodeTypeConcrete, + [aux_sym_CARET] = TSNodeTypeConcrete, + [aux_sym_LPAREN] = TSNodeTypeConcrete, + [aux_sym_RPAREN] = TSNodeTypeConcrete, + [sym_number] = TSNodeTypeNormal, + [sym_variable] = TSNodeTypeNormal, + [sym_comment] = TSNodeTypeNormal, }; static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { diff --git a/spec/fixtures/parsers/c.c b/spec/fixtures/parsers/c.c index aa2f2ffa..56eee6f6 100644 --- a/spec/fixtures/parsers/c.c +++ b/spec/fixtures/parsers/c.c @@ -137,45 +137,72 @@ static const char *ts_symbol_names[] = { [sym_comment] = "comment", }; -static const int ts_hidden_symbol_flags[SYMBOL_COUNT] = { - [sym__type_specifier] = 1, - [sym__init_declarator] = 1, - [sym__direct_declarator] = 1, - [aux_sym_function_definition_repeat1] = 1, - [aux_sym_declaration_specifiers_repeat1] = 1, - [aux_sym_type_name_repeat1] = 1, - [aux_sym_struct_specifier_repeat1] = 1, - [aux_sym_declaration_repeat1] = 1, - [aux_sym_designation_repeat1] = 1, - [aux_sym_declarator_repeat1] = 1, - [aux_sym__direct_declarator_repeat1] = 1, - [aux_sym_compound_statement_repeat1] = 1, - [aux_sym_call_expression_repeat1] = 1, - [aux_sym_typedef] = 1, - [aux_sym_extern] = 1, - [aux_sym_static] = 1, - [aux_sym_auto] = 1, - [aux_sym_register] = 1, - [aux_sym_signed] = 1, - [aux_sym_unsigned] = 1, - [aux_sym_long] = 1, - [aux_sym_short] = 1, - [aux_sym_struct] = 1, - [aux_sym_LBRACE] = 1, - [aux_sym_RBRACE] = 1, - [aux_sym_COMMA] = 1, - [aux_sym_SEMI] = 1, - [aux_sym_EQ] = 1, - [aux_sym_LBRACK] = 1, - [aux_sym_RBRACK] = 1, - [aux_sym_DOT] = 1, - [aux_sym_LPAREN] = 1, - [aux_sym_RPAREN] = 1, - [aux_sym_const] = 1, - [aux_sym_restrict] = 1, - [aux_sym_volatile] = 1, - [aux_sym_STAR] = 1, - [aux_sym_PLUS] = 1, +static const TSNodeType ts_node_types[SYMBOL_COUNT] = { + [sym_program] = TSNodeTypeNormal, + [sym_function_definition] = TSNodeTypeNormal, + [sym_declaration_specifiers] = TSNodeTypeNormal, + [sym_storage_class_specifier] = TSNodeTypeNormal, + [sym__type_specifier] = TSNodeTypeHidden, + [sym_type_name] = TSNodeTypeNormal, + [sym_struct_specifier] = TSNodeTypeNormal, + [sym_struct_declaration] = TSNodeTypeNormal, + [sym_parameter_declaration] = TSNodeTypeNormal, + [sym_declaration] = TSNodeTypeNormal, + [sym__init_declarator] = TSNodeTypeHidden, + [sym_initializer] = TSNodeTypeNormal, + [sym_initializer_list] = TSNodeTypeNormal, + [sym_designation] = TSNodeTypeNormal, + [sym_declarator] = TSNodeTypeNormal, + [sym__direct_declarator] = TSNodeTypeHidden, + [sym_type_qualifier] = TSNodeTypeNormal, + [sym_pointer] = TSNodeTypeNormal, + [sym_compound_statement] = TSNodeTypeNormal, + [sym_expression] = TSNodeTypeNormal, + [sym_math_expression] = TSNodeTypeNormal, + [sym_call_expression] = TSNodeTypeNormal, + [sym_statement] = TSNodeTypeNormal, + [sym_expression_statement] = TSNodeTypeNormal, + [aux_sym_function_definition_repeat1] = TSNodeTypeHidden, + [aux_sym_declaration_specifiers_repeat1] = TSNodeTypeHidden, + [aux_sym_type_name_repeat1] = TSNodeTypeHidden, + [aux_sym_struct_specifier_repeat1] = TSNodeTypeHidden, + [aux_sym_declaration_repeat1] = TSNodeTypeHidden, + [aux_sym_designation_repeat1] = TSNodeTypeHidden, + [aux_sym_declarator_repeat1] = TSNodeTypeHidden, + [aux_sym__direct_declarator_repeat1] = TSNodeTypeHidden, + [aux_sym_compound_statement_repeat1] = TSNodeTypeHidden, + [aux_sym_call_expression_repeat1] = TSNodeTypeHidden, + [ts_builtin_sym_error] = TSNodeTypeNormal, + [ts_builtin_sym_end] = TSNodeTypeHidden, + [aux_sym_typedef] = TSNodeTypeConcrete, + [aux_sym_extern] = TSNodeTypeConcrete, + [aux_sym_static] = TSNodeTypeConcrete, + [aux_sym_auto] = TSNodeTypeConcrete, + [aux_sym_register] = TSNodeTypeConcrete, + [aux_sym_signed] = TSNodeTypeConcrete, + [aux_sym_unsigned] = TSNodeTypeConcrete, + [aux_sym_long] = TSNodeTypeConcrete, + [aux_sym_short] = TSNodeTypeConcrete, + [aux_sym_struct] = TSNodeTypeConcrete, + [aux_sym_LBRACE] = TSNodeTypeConcrete, + [aux_sym_RBRACE] = TSNodeTypeConcrete, + [aux_sym_COMMA] = TSNodeTypeConcrete, + [aux_sym_SEMI] = TSNodeTypeConcrete, + [aux_sym_EQ] = TSNodeTypeConcrete, + [aux_sym_LBRACK] = TSNodeTypeConcrete, + [aux_sym_RBRACK] = TSNodeTypeConcrete, + [aux_sym_DOT] = TSNodeTypeConcrete, + [aux_sym_LPAREN] = TSNodeTypeConcrete, + [aux_sym_RPAREN] = TSNodeTypeConcrete, + [aux_sym_const] = TSNodeTypeConcrete, + [aux_sym_restrict] = TSNodeTypeConcrete, + [aux_sym_volatile] = TSNodeTypeConcrete, + [aux_sym_STAR] = TSNodeTypeConcrete, + [aux_sym_PLUS] = TSNodeTypeConcrete, + [sym_string] = TSNodeTypeNormal, + [sym_identifier] = TSNodeTypeNormal, + [sym_number] = TSNodeTypeNormal, + [sym_comment] = TSNodeTypeNormal, }; static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { diff --git a/spec/fixtures/parsers/golang.c b/spec/fixtures/parsers/golang.c index 6974ec3d..6bc85a6b 100644 --- a/spec/fixtures/parsers/golang.c +++ b/spec/fixtures/parsers/golang.c @@ -175,61 +175,91 @@ static const char *ts_symbol_names[] = { [sym_comment] = "comment", }; -static const int ts_hidden_symbol_flags[SYMBOL_COUNT] = { - [sym__declaration] = 1, - [sym__type_expression] = 1, - [sym__statement] = 1, - [sym__expression] = 1, - [sym__func_signature] = 1, - [aux_sym_program_repeat1] = 1, - [aux_sym_program_repeat2] = 1, - [aux_sym_imports_block_repeat1] = 1, - [aux_sym_block_statement_repeat1] = 1, - [aux_sym_struct_type_repeat1] = 1, - [aux_sym_interface_type_repeat1] = 1, - [aux_sym_return_statement_repeat1] = 1, - [aux_sym_short_var_declaration_repeat1] = 1, - [aux_sym__func_signature_repeat1] = 1, - [aux_sym__func_signature_repeat2] = 1, - [aux_sym__func_signature_repeat3] = 1, - [aux_sym_package] = 1, - [aux_sym_import] = 1, - [aux_sym_LPAREN] = 1, - [aux_sym_RPAREN] = 1, - [aux_sym_type] = 1, - [aux_sym_SEMI] = 1, - [aux_sym_var] = 1, - [aux_sym_EQ] = 1, - [aux_sym_func] = 1, - [aux_sym_LBRACE] = 1, - [aux_sym_RBRACE] = 1, - [aux_sym_STAR] = 1, - [aux_sym_map] = 1, - [aux_sym_LBRACK] = 1, - [aux_sym_RBRACK] = 1, - [aux_sym_struct] = 1, - [aux_sym_interface] = 1, - [aux_sym_return] = 1, - [aux_sym_COMMA] = 1, - [aux_sym_COLON_EQ] = 1, - [aux_sym_for] = 1, - [aux_sym_range] = 1, - [aux_sym_if] = 1, - [aux_sym_else] = 1, - [aux_sym_DOT] = 1, - [aux_sym_SLASH] = 1, - [aux_sym_PLUS] = 1, - [aux_sym_DASH] = 1, - [aux_sym_PIPE_PIPE] = 1, - [aux_sym_AMP_AMP] = 1, - [aux_sym_EQ_EQ] = 1, - [aux_sym_LT_EQ] = 1, - [aux_sym_LT] = 1, - [aux_sym_GT_EQ] = 1, - [aux_sym_GT] = 1, - [aux_sym_BANG] = 1, - [sym__line_break] = 1, - [sym__identifier] = 1, +static const TSNodeType ts_node_types[SYMBOL_COUNT] = { + [sym_program] = TSNodeTypeNormal, + [sym_package_directive] = TSNodeTypeNormal, + [sym_imports_block] = TSNodeTypeNormal, + [sym_package_import] = TSNodeTypeNormal, + [sym__declaration] = TSNodeTypeHidden, + [sym_type_declaration] = TSNodeTypeNormal, + [sym_var_declaration] = TSNodeTypeNormal, + [sym_func_declaration] = TSNodeTypeNormal, + [sym_block_statement] = TSNodeTypeNormal, + [sym__type_expression] = TSNodeTypeHidden, + [sym_pointer_type] = TSNodeTypeNormal, + [sym_map_type] = TSNodeTypeNormal, + [sym_slice_type] = TSNodeTypeNormal, + [sym_struct_type] = TSNodeTypeNormal, + [sym_interface_type] = TSNodeTypeNormal, + [sym__statement] = TSNodeTypeHidden, + [sym_return_statement] = TSNodeTypeNormal, + [sym_short_var_declaration] = TSNodeTypeNormal, + [sym_range_statement] = TSNodeTypeNormal, + [sym_if_statement] = TSNodeTypeNormal, + [sym_expression_statement] = TSNodeTypeNormal, + [sym__expression] = TSNodeTypeHidden, + [sym_call_expression] = TSNodeTypeNormal, + [sym_selector_expression] = TSNodeTypeNormal, + [sym_math_op] = TSNodeTypeNormal, + [sym_bool_op] = TSNodeTypeNormal, + [sym__func_signature] = TSNodeTypeHidden, + [sym_package_name] = TSNodeTypeNormal, + [sym_var_name] = TSNodeTypeNormal, + [sym_type_name] = TSNodeTypeNormal, + [aux_sym_program_repeat1] = TSNodeTypeHidden, + [aux_sym_program_repeat2] = TSNodeTypeHidden, + [aux_sym_imports_block_repeat1] = TSNodeTypeHidden, + [aux_sym_block_statement_repeat1] = TSNodeTypeHidden, + [aux_sym_struct_type_repeat1] = TSNodeTypeHidden, + [aux_sym_interface_type_repeat1] = TSNodeTypeHidden, + [aux_sym_return_statement_repeat1] = TSNodeTypeHidden, + [aux_sym_short_var_declaration_repeat1] = TSNodeTypeHidden, + [aux_sym__func_signature_repeat1] = TSNodeTypeHidden, + [aux_sym__func_signature_repeat2] = TSNodeTypeHidden, + [aux_sym__func_signature_repeat3] = TSNodeTypeHidden, + [ts_builtin_sym_error] = TSNodeTypeNormal, + [ts_builtin_sym_end] = TSNodeTypeHidden, + [aux_sym_package] = TSNodeTypeConcrete, + [aux_sym_import] = TSNodeTypeConcrete, + [aux_sym_LPAREN] = TSNodeTypeConcrete, + [aux_sym_RPAREN] = TSNodeTypeConcrete, + [aux_sym_type] = TSNodeTypeConcrete, + [aux_sym_SEMI] = TSNodeTypeConcrete, + [aux_sym_var] = TSNodeTypeConcrete, + [aux_sym_EQ] = TSNodeTypeConcrete, + [aux_sym_func] = TSNodeTypeConcrete, + [aux_sym_LBRACE] = TSNodeTypeConcrete, + [aux_sym_RBRACE] = TSNodeTypeConcrete, + [aux_sym_STAR] = TSNodeTypeConcrete, + [aux_sym_map] = TSNodeTypeConcrete, + [aux_sym_LBRACK] = TSNodeTypeConcrete, + [aux_sym_RBRACK] = TSNodeTypeConcrete, + [aux_sym_struct] = TSNodeTypeConcrete, + [aux_sym_interface] = TSNodeTypeConcrete, + [aux_sym_return] = TSNodeTypeConcrete, + [aux_sym_COMMA] = TSNodeTypeConcrete, + [aux_sym_COLON_EQ] = TSNodeTypeConcrete, + [aux_sym_for] = TSNodeTypeConcrete, + [aux_sym_range] = TSNodeTypeConcrete, + [aux_sym_if] = TSNodeTypeConcrete, + [aux_sym_else] = TSNodeTypeConcrete, + [aux_sym_DOT] = TSNodeTypeConcrete, + [aux_sym_SLASH] = TSNodeTypeConcrete, + [aux_sym_PLUS] = TSNodeTypeConcrete, + [aux_sym_DASH] = TSNodeTypeConcrete, + [aux_sym_PIPE_PIPE] = TSNodeTypeConcrete, + [aux_sym_AMP_AMP] = TSNodeTypeConcrete, + [aux_sym_EQ_EQ] = TSNodeTypeConcrete, + [aux_sym_LT_EQ] = TSNodeTypeConcrete, + [aux_sym_LT] = TSNodeTypeConcrete, + [aux_sym_GT_EQ] = TSNodeTypeConcrete, + [aux_sym_GT] = TSNodeTypeConcrete, + [aux_sym_BANG] = TSNodeTypeConcrete, + [sym__line_break] = TSNodeTypeHidden, + [sym_string] = TSNodeTypeNormal, + [sym__identifier] = TSNodeTypeHidden, + [sym_number] = TSNodeTypeNormal, + [sym_comment] = TSNodeTypeNormal, }; static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { diff --git a/spec/fixtures/parsers/javascript.c b/spec/fixtures/parsers/javascript.c index 8494204c..76490fb8 100644 --- a/spec/fixtures/parsers/javascript.c +++ b/spec/fixtures/parsers/javascript.c @@ -219,66 +219,113 @@ static const char *ts_symbol_names[] = { [sym__line_break] = "_line_break", }; -static const int ts_hidden_symbol_flags[SYMBOL_COUNT] = { - [sym__statement] = 1, - [sym__paren_expression] = 1, - [sym__expression] = 1, - [aux_sym_program_repeat1] = 1, - [aux_sym_var_declaration_repeat1] = 1, - [aux_sym_switch_statement_repeat1] = 1, - [aux_sym_object_repeat1] = 1, - [aux_sym_array_repeat1] = 1, - [aux_sym_formal_parameters_repeat1] = 1, - [aux_sym_SEMI] = 1, - [aux_sym_var] = 1, - [aux_sym_COMMA] = 1, - [aux_sym_LBRACE] = 1, - [aux_sym_RBRACE] = 1, - [aux_sym_if] = 1, - [aux_sym_else] = 1, - [aux_sym_switch] = 1, - [aux_sym_for] = 1, - [aux_sym_LPAREN] = 1, - [aux_sym_RPAREN] = 1, - [aux_sym_in] = 1, - [aux_sym_while] = 1, - [aux_sym_try] = 1, - [aux_sym_return] = 1, - [aux_sym_throw] = 1, - [aux_sym_break] = 1, - [aux_sym_delete] = 1, - [aux_sym_case] = 1, - [aux_sym_COLON] = 1, - [aux_sym_default] = 1, - [aux_sym_catch] = 1, - [aux_sym_finally] = 1, - [aux_sym_EQ] = 1, - [aux_sym_LBRACK] = 1, - [aux_sym_RBRACK] = 1, - [aux_sym_function] = 1, - [aux_sym_new] = 1, - [aux_sym_DOT] = 1, - [aux_sym_PLUS_EQ] = 1, - [aux_sym_DASH_EQ] = 1, - [aux_sym_STAR_EQ] = 1, - [aux_sym_SLASH_EQ] = 1, - [aux_sym_QMARK] = 1, - [aux_sym_PIPE_PIPE] = 1, - [aux_sym_AMP_AMP] = 1, - [aux_sym_BANG] = 1, - [aux_sym_PLUS_PLUS] = 1, - [aux_sym_DASH_DASH] = 1, - [aux_sym_STAR] = 1, - [aux_sym_SLASH] = 1, - [aux_sym_PLUS] = 1, - [aux_sym_DASH] = 1, - [aux_sym_EQ_EQ_EQ] = 1, - [aux_sym_BANG_EQ_EQ] = 1, - [aux_sym_LT] = 1, - [aux_sym_GT] = 1, - [aux_sym_instanceof] = 1, - [aux_sym_typeof] = 1, - [sym__line_break] = 1, +static const TSNodeType ts_node_types[SYMBOL_COUNT] = { + [sym_program] = TSNodeTypeNormal, + [sym__statement] = TSNodeTypeHidden, + [sym_expression_statement] = TSNodeTypeNormal, + [sym_var_declaration] = TSNodeTypeNormal, + [sym_statement_block] = TSNodeTypeNormal, + [sym_if_statement] = TSNodeTypeNormal, + [sym_switch_statement] = TSNodeTypeNormal, + [sym_for_statement] = TSNodeTypeNormal, + [sym_for_in_statement] = TSNodeTypeNormal, + [sym_while_statement] = TSNodeTypeNormal, + [sym_try_statement] = TSNodeTypeNormal, + [sym_return_statement] = TSNodeTypeNormal, + [sym_throw_statement] = TSNodeTypeNormal, + [sym_break_statement] = TSNodeTypeNormal, + [sym_delete_statement] = TSNodeTypeNormal, + [sym_case] = TSNodeTypeNormal, + [sym_default] = TSNodeTypeNormal, + [sym_catch] = TSNodeTypeNormal, + [sym_finally] = TSNodeTypeNormal, + [sym_var_assignment] = TSNodeTypeNormal, + [sym__paren_expression] = TSNodeTypeHidden, + [sym__expression] = TSNodeTypeHidden, + [sym_object] = TSNodeTypeNormal, + [sym_array] = TSNodeTypeNormal, + [sym_function_expression] = TSNodeTypeNormal, + [sym_function_call] = TSNodeTypeNormal, + [sym_constructor_call] = TSNodeTypeNormal, + [sym_member_access] = TSNodeTypeNormal, + [sym_subscript_access] = TSNodeTypeNormal, + [sym_assignment] = TSNodeTypeNormal, + [sym_math_assignment] = TSNodeTypeNormal, + [sym_ternary] = TSNodeTypeNormal, + [sym_bool_op] = TSNodeTypeNormal, + [sym_math_op] = TSNodeTypeNormal, + [sym_rel_op] = TSNodeTypeNormal, + [sym_type_op] = TSNodeTypeNormal, + [sym_formal_parameters] = TSNodeTypeNormal, + [sym_arguments] = TSNodeTypeNormal, + [sym_pair] = TSNodeTypeNormal, + [aux_sym_program_repeat1] = TSNodeTypeHidden, + [aux_sym_var_declaration_repeat1] = TSNodeTypeHidden, + [aux_sym_switch_statement_repeat1] = TSNodeTypeHidden, + [aux_sym_object_repeat1] = TSNodeTypeHidden, + [aux_sym_array_repeat1] = TSNodeTypeHidden, + [aux_sym_formal_parameters_repeat1] = TSNodeTypeHidden, + [ts_builtin_sym_error] = TSNodeTypeNormal, + [ts_builtin_sym_end] = TSNodeTypeHidden, + [aux_sym_SEMI] = TSNodeTypeConcrete, + [aux_sym_var] = TSNodeTypeConcrete, + [aux_sym_COMMA] = TSNodeTypeConcrete, + [aux_sym_LBRACE] = TSNodeTypeConcrete, + [aux_sym_RBRACE] = TSNodeTypeConcrete, + [aux_sym_if] = TSNodeTypeConcrete, + [aux_sym_else] = TSNodeTypeConcrete, + [aux_sym_switch] = TSNodeTypeConcrete, + [aux_sym_for] = TSNodeTypeConcrete, + [aux_sym_LPAREN] = TSNodeTypeConcrete, + [aux_sym_RPAREN] = TSNodeTypeConcrete, + [aux_sym_in] = TSNodeTypeConcrete, + [aux_sym_while] = TSNodeTypeConcrete, + [aux_sym_try] = TSNodeTypeConcrete, + [aux_sym_return] = TSNodeTypeConcrete, + [aux_sym_throw] = TSNodeTypeConcrete, + [aux_sym_break] = TSNodeTypeConcrete, + [aux_sym_delete] = TSNodeTypeConcrete, + [aux_sym_case] = TSNodeTypeConcrete, + [aux_sym_COLON] = TSNodeTypeConcrete, + [aux_sym_default] = TSNodeTypeConcrete, + [aux_sym_catch] = TSNodeTypeConcrete, + [aux_sym_finally] = TSNodeTypeConcrete, + [aux_sym_EQ] = TSNodeTypeConcrete, + [aux_sym_LBRACK] = TSNodeTypeConcrete, + [aux_sym_RBRACK] = TSNodeTypeConcrete, + [aux_sym_function] = TSNodeTypeConcrete, + [aux_sym_new] = TSNodeTypeConcrete, + [aux_sym_DOT] = TSNodeTypeConcrete, + [aux_sym_PLUS_EQ] = TSNodeTypeConcrete, + [aux_sym_DASH_EQ] = TSNodeTypeConcrete, + [aux_sym_STAR_EQ] = TSNodeTypeConcrete, + [aux_sym_SLASH_EQ] = TSNodeTypeConcrete, + [aux_sym_QMARK] = TSNodeTypeConcrete, + [aux_sym_PIPE_PIPE] = TSNodeTypeConcrete, + [aux_sym_AMP_AMP] = TSNodeTypeConcrete, + [aux_sym_BANG] = TSNodeTypeConcrete, + [aux_sym_PLUS_PLUS] = TSNodeTypeConcrete, + [aux_sym_DASH_DASH] = TSNodeTypeConcrete, + [aux_sym_STAR] = TSNodeTypeConcrete, + [aux_sym_SLASH] = TSNodeTypeConcrete, + [aux_sym_PLUS] = TSNodeTypeConcrete, + [aux_sym_DASH] = TSNodeTypeConcrete, + [aux_sym_EQ_EQ_EQ] = TSNodeTypeConcrete, + [aux_sym_BANG_EQ_EQ] = TSNodeTypeConcrete, + [aux_sym_LT] = TSNodeTypeConcrete, + [aux_sym_GT] = TSNodeTypeConcrete, + [aux_sym_instanceof] = TSNodeTypeConcrete, + [aux_sym_typeof] = TSNodeTypeConcrete, + [sym_comment] = TSNodeTypeNormal, + [sym_string] = TSNodeTypeNormal, + [sym_regex] = TSNodeTypeNormal, + [sym_number] = TSNodeTypeNormal, + [sym_identifier] = TSNodeTypeNormal, + [sym_null] = TSNodeTypeNormal, + [sym_undefined] = TSNodeTypeNormal, + [sym_true] = TSNodeTypeNormal, + [sym_false] = TSNodeTypeNormal, + [sym__line_break] = TSNodeTypeHidden, }; static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { diff --git a/spec/fixtures/parsers/json.c b/spec/fixtures/parsers/json.c index ab6c1c6c..c4efdd76 100644 --- a/spec/fixtures/parsers/json.c +++ b/spec/fixtures/parsers/json.c @@ -43,16 +43,25 @@ static const char *ts_symbol_names[] = { [sym_false] = "false", }; -static const int ts_hidden_symbol_flags[SYMBOL_COUNT] = { - [sym__value] = 1, - [aux_sym_object_repeat1] = 1, - [aux_sym_array_repeat1] = 1, - [aux_sym_LBRACE] = 1, - [aux_sym_COLON] = 1, - [aux_sym_COMMA] = 1, - [aux_sym_RBRACE] = 1, - [aux_sym_LBRACK] = 1, - [aux_sym_RBRACK] = 1, +static const TSNodeType ts_node_types[SYMBOL_COUNT] = { + [sym__value] = TSNodeTypeHidden, + [sym_object] = TSNodeTypeNormal, + [sym_array] = TSNodeTypeNormal, + [aux_sym_object_repeat1] = TSNodeTypeHidden, + [aux_sym_array_repeat1] = TSNodeTypeHidden, + [ts_builtin_sym_error] = TSNodeTypeNormal, + [ts_builtin_sym_end] = TSNodeTypeHidden, + [aux_sym_LBRACE] = TSNodeTypeConcrete, + [aux_sym_COLON] = TSNodeTypeConcrete, + [aux_sym_COMMA] = TSNodeTypeConcrete, + [aux_sym_RBRACE] = TSNodeTypeConcrete, + [aux_sym_LBRACK] = TSNodeTypeConcrete, + [aux_sym_RBRACK] = TSNodeTypeConcrete, + [sym_string] = TSNodeTypeNormal, + [sym_number] = TSNodeTypeNormal, + [sym_null] = TSNodeTypeNormal, + [sym_true] = TSNodeTypeNormal, + [sym_false] = TSNodeTypeNormal, }; static TSTree *ts_lex(TSLexer *lexer, TSStateId lex_state) { diff --git a/spec/runtime/parse_stack_spec.cc b/spec/runtime/parse_stack_spec.cc index f6810718..2ed527ec 100644 --- a/spec/runtime/parse_stack_spec.cc +++ b/spec/runtime/parse_stack_spec.cc @@ -43,7 +43,7 @@ describe("ParseStack", [&]() { TSLength len = ts_length_make(2, 2); for (size_t i = 0; i < tree_count; i++) - trees[i] = ts_tree_make_leaf(ts_builtin_sym_start + i, len, len, false); + trees[i] = ts_tree_make_leaf(ts_builtin_sym_start + i, len, len, TSNodeTypeNormal); }); after_each([&]() { diff --git a/spec/runtime/stack_spec.cc b/spec/runtime/stack_spec.cc index 4d5c1e72..0b8380fc 100644 --- a/spec/runtime/stack_spec.cc +++ b/spec/runtime/stack_spec.cc @@ -32,7 +32,7 @@ describe("stacks", [&]() { sym1, ts_length_make(0, 0), ts_length_make(1, 1), - 0); + TSNodeTypeNormal); ts_stack_push(&stack, 5, node1); }); diff --git a/spec/runtime/tree_spec.cc b/spec/runtime/tree_spec.cc index 17b99b6d..d0351c3b 100644 --- a/spec/runtime/tree_spec.cc +++ b/spec/runtime/tree_spec.cc @@ -26,19 +26,19 @@ describe("Tree", []() { cat, ts_length_make(5, 4), ts_length_make(2, 1), - false); + TSNodeTypeNormal); tree2 = ts_tree_make_leaf( cat, ts_length_make(3, 3), ts_length_make(1, 1), - false); + TSNodeTypeNormal); parent1 = ts_tree_make_node( dog, 2, tree_array({ tree1, tree2, }), - false); + TSNodeTypeNormal); }); after_each([&]() { @@ -88,7 +88,7 @@ describe("Tree", []() { parent = ts_tree_make_node(pig, 2, tree_array({ tree1, tree2, - }), 0); + }), TSNodeTypeNormal); }); after_each([&]() { @@ -109,7 +109,7 @@ describe("Tree", []() { parent = ts_tree_make_node(pig, 2, tree_array({ tree1, tree2, - }), 0); + }), TSNodeTypeNormal); }); after_each([&]() { @@ -130,7 +130,7 @@ describe("Tree", []() { parent = ts_tree_make_node(pig, 2, tree_array({ tree1, tree2, - }), 0); + }), TSNodeTypeNormal); }); after_each([&]() { @@ -150,7 +150,7 @@ describe("Tree", []() { cat, ts_length_make(5, 4), ts_length_make(2, 1), - 0); + TSNodeTypeNormal); AssertThat(ts_tree_eq(tree1, tree1_copy), IsTrue()); @@ -158,13 +158,13 @@ describe("Tree", []() { cat, ts_length_make(3, 3), ts_length_make(1, 1), - 0); + TSNodeTypeNormal); AssertThat(ts_tree_eq(tree2, tree2_copy), IsTrue()); TSTree *parent2 = ts_tree_make_node(dog, 2, tree_array({ tree1_copy, tree2_copy, - }), 0); + }), TSNodeTypeNormal); AssertThat(ts_tree_eq(parent1, parent2), IsTrue()); @@ -178,7 +178,7 @@ describe("Tree", []() { tree1->symbol + 1, tree1->size, tree1->padding, - 0); + TSNodeTypeNormal); AssertThat(ts_tree_eq(tree1, different_tree), IsFalse()); ts_tree_release(different_tree); @@ -189,11 +189,11 @@ describe("Tree", []() { tree1->symbol + 1, tree1->size, tree1->padding, - 0); + TSNodeTypeNormal); TSTree *different_parent = ts_tree_make_node(dog, 2, tree_array({ different_tree, different_tree, - }), 0); + }), TSNodeTypeNormal); AssertThat(ts_tree_eq(different_parent, parent1), IsFalse()); AssertThat(ts_tree_eq(parent1, different_parent), IsFalse()); @@ -215,7 +215,7 @@ describe("Tree", []() { }); it("hides invisible nodes", [&]() { - tree2->options = TSTreeOptionsHidden; + tree2->options.hidden = true; char *string1 = ts_tree_string(parent1, names); AssertThat(string(string1), Equals("(dog (cat))")); @@ -224,13 +224,13 @@ describe("Tree", []() { describe("when the root node is not visible", [&]() { it("still serializes it", [&]() { - parent1->options = TSTreeOptionsHidden; + parent1->options.hidden = true; char *string1 = ts_tree_string(parent1, names); - AssertThat(string(string1), Equals("(dog (cat) (cat))")); + AssertThat(string(string1), Equals("(dog (cat) (cat))")); free(string1); - tree1->options = TSTreeOptionsHidden; + tree1->options.hidden = true; char *string2 = ts_tree_string(tree1, names); AssertThat(string(string2), Equals("(cat)")); diff --git a/src/compiler/generate_code/c_code.cc b/src/compiler/generate_code/c_code.cc index bf3ca570..cc6ef182 100644 --- a/src/compiler/generate_code/c_code.cc +++ b/src/compiler/generate_code/c_code.cc @@ -88,7 +88,7 @@ class CCodeGenerator { add_state_and_symbol_counts(); add_symbol_enum(); add_symbol_names_list(); - add_hidden_symbols_list(); + add_symbol_node_types_list(); add_lex_function(); add_lex_states_list(); add_parse_table(); @@ -136,13 +136,38 @@ class CCodeGenerator { line(); } - void add_hidden_symbols_list() { - line("static const int ts_hidden_symbol_flags[SYMBOL_COUNT] = {"); + void add_symbol_node_types_list() { + line("static const TSNodeType ts_node_types[SYMBOL_COUNT] = {"); indent([&]() { - for (const auto &symbol : parse_table.symbols) - if (!symbol.is_built_in() && - (is_auxiliary(symbol) || rule_name(symbol)[0] == '_')) - line("[" + symbol_id(symbol) + "] = 1,"); + for (const auto &symbol : parse_table.symbols) { + line("[" + symbol_id(symbol) + "] = "); + + if (symbol == rules::ERROR()) { + add("TSNodeTypeNormal,"); + continue; + } else if (symbol == rules::END_OF_INPUT()) { + add("TSNodeTypeHidden,"); + continue; + } + + RuleEntry entry = entry_for_symbol(symbol); + if (entry.name[0] == '_') { + add("TSNodeTypeHidden,"); + continue; + } + + switch (entry.type) { + case RuleEntryTypeNamed: + add("TSNodeTypeNormal,"); + break; + case RuleEntryTypeAnonymous: + add("TSNodeTypeConcrete,"); + break; + case RuleEntryTypeHidden: + add("TSNodeTypeHidden,"); + break; + } + } }); line("};"); line(); @@ -328,10 +353,10 @@ class CCodeGenerator { return ""; } else { string name = sanitize_name(rule_name(symbol)); - if (is_auxiliary(symbol)) - return "aux_sym_" + name; - else + if (entry_for_symbol(symbol).type == RuleEntryTypeNamed) return "sym_" + name; + else + return "aux_sym_" + name; } } @@ -348,20 +373,15 @@ class CCodeGenerator { } } - bool is_auxiliary(const rules::Symbol &symbol) { - if (symbol.is_token) { - return lexical_grammar.rules[symbol.index].type != RuleEntryTypeNamed; - } else { - return syntax_grammar.rules[symbol.index].type != RuleEntryTypeNamed; - } + const RuleEntry &entry_for_symbol(const rules::Symbol &symbol) { + if (symbol.is_token) + return lexical_grammar.rules[symbol.index]; + else + return syntax_grammar.rules[symbol.index]; } string rule_name(const rules::Symbol &symbol) { - if (symbol.is_token) { - return lexical_grammar.rules[symbol.index].name; - } else { - return syntax_grammar.rules[symbol.index].name; - } + return entry_for_symbol(symbol).name; } bool reduce_action_is_fragile(const ParseAction &action) const { diff --git a/src/runtime/lexer.c b/src/runtime/lexer.c index e43bac15..88468cd2 100644 --- a/src/runtime/lexer.c +++ b/src/runtime/lexer.c @@ -68,8 +68,8 @@ static bool ts_lexer__advance(TSLexer *lexer, TSStateId state) { return true; } -static TSTree *ts_lexer__accept(TSLexer *lexer, TSSymbol symbol, int is_hidden, - const char *symbol_name) { +static TSTree *ts_lexer__accept(TSLexer *lexer, TSSymbol symbol, + TSNodeType node_type, const char *symbol_name) { TSLength size = ts_length_sub(lexer->current_position, lexer->token_start_position); TSLength padding = @@ -81,7 +81,7 @@ static TSTree *ts_lexer__accept(TSLexer *lexer, TSSymbol symbol, int is_hidden, return ts_tree_make_error(size, padding, lexer->lookahead); } else { DEBUG("accept_token sym:%s", symbol_name); - return ts_tree_make_leaf(symbol, size, padding, is_hidden); + return ts_tree_make_leaf(symbol, size, padding, node_type); } } @@ -91,19 +91,21 @@ static TSTree *ts_lexer__accept(TSLexer *lexer, TSSymbol symbol, int is_hidden, */ TSLexer ts_lexer_make() { - TSLexer result = (TSLexer){.start_fn = ts_lexer__start, - .start_token_fn = ts_lexer__start_token, - .advance_fn = ts_lexer__advance, - .accept_fn = ts_lexer__accept, - .chunk = NULL, - .chunk_start = 0, - .chunk_size = 0, - .current_position = ts_length_zero(), - .token_start_position = ts_length_zero(), - .token_end_position = ts_length_zero(), - .lookahead = 0, - .lookahead_size = 0, - .debugger = ts_debugger_null() }; + TSLexer result = (TSLexer){ + .start_fn = ts_lexer__start, + .start_token_fn = ts_lexer__start_token, + .advance_fn = ts_lexer__advance, + .accept_fn = ts_lexer__accept, + .chunk = NULL, + .chunk_start = 0, + .chunk_size = 0, + .current_position = ts_length_zero(), + .token_start_position = ts_length_zero(), + .token_end_position = ts_length_zero(), + .lookahead = 0, + .lookahead_size = 0, + .debugger = ts_debugger_null(), + }; return result; } diff --git a/src/runtime/parser.c b/src/runtime/parser.c index 501ae03a..faa87bb0 100644 --- a/src/runtime/parser.c +++ b/src/runtime/parser.c @@ -193,7 +193,7 @@ static void ts_parser__shift_extra(TSParser *parser, int head, TSStateId state) static TSTree *ts_parser__reduce(TSParser *parser, int head, TSSymbol symbol, size_t child_count, bool extra, bool count_extra) { - bool hidden = parser->language->hidden_symbol_flags[symbol]; + TSNodeType node_type = parser->language->node_types[symbol]; ParseStackPopResultList pop_results = ts_parse_stack_pop(parser->stack, head, child_count, count_extra); @@ -206,7 +206,7 @@ static TSTree *ts_parser__reduce(TSParser *parser, int head, TSSymbol symbol, if (pop_result.trees != last_children) { parent = ts_tree_make_node(symbol, pop_result.tree_count, - pop_result.trees, hidden); + pop_result.trees, node_type); } if (pop_result.index == last_index) { diff --git a/src/runtime/tree.c b/src/runtime/tree.c index 00b2886f..99151afd 100644 --- a/src/runtime/tree.c +++ b/src/runtime/tree.c @@ -7,7 +7,7 @@ #include "runtime/length.h" TSTree *ts_tree_make_leaf(TSSymbol sym, TSLength size, TSLength padding, - bool is_hidden) { + TSNodeType node_type) { TSTree *result = malloc(sizeof(TSTree)); *result = (TSTree){ .ref_count = 1, @@ -16,7 +16,10 @@ TSTree *ts_tree_make_leaf(TSSymbol sym, TSLength size, TSLength padding, .child_count = 0, .children = NULL, .padding = padding, - .options = is_hidden ? TSTreeOptionsHidden : 0, + .options = (TSTreeOptions){ + .hidden = (node_type == TSNodeTypeHidden), + .concrete = (node_type == TSNodeTypeConcrete), + }, }; return result; } @@ -30,7 +33,7 @@ TSTree *ts_tree_make_error(TSLength size, TSLength padding, char lookahead_char) } TSTree *ts_tree_make_node(TSSymbol symbol, size_t child_count, - TSTree **children, bool is_hidden) { + TSTree **children, TSNodeType node_type) { TSTree *result = malloc(sizeof(TSTree)); /* @@ -58,21 +61,17 @@ TSTree *ts_tree_make_node(TSSymbol symbol, size_t child_count, visible_child_count += child->visible_child_count; } - /* - * Mark the tree as hidden if it wraps a single child node. - */ - TSTreeOptions options = 0; + TSTreeOptions options = (TSTreeOptions){ + .hidden = (node_type == TSNodeTypeHidden), + .concrete = (node_type == TSNodeTypeConcrete), + }; + if (symbol == ts_builtin_sym_error) { - options |= (TSTreeOptionsFragileLeft | TSTreeOptionsFragileRight); - } else { - if (is_hidden) - options |= TSTreeOptionsHidden; - if (child_count > 0) { - if (ts_tree_is_fragile_left(children[0])) - options |= (TSTreeOptionsFragileLeft); - if (ts_tree_is_fragile_right(children[child_count - 1])) - options |= (TSTreeOptionsFragileRight); - } + options.fragile_left = true; + options.fragile_left = true; + } else if (child_count > 0) { + options.fragile_left = children[0]->options.fragile_left; + options.fragile_right = children[child_count - 1]->options.fragile_right; } *result = (TSTree){.ref_count = 1, diff --git a/src/runtime/tree.h b/src/runtime/tree.h index 70e44d12..c6f4654a 100644 --- a/src/runtime/tree.h +++ b/src/runtime/tree.h @@ -8,11 +8,12 @@ extern "C" { #include #include "tree_sitter/parser.h" -typedef enum { - TSTreeOptionsHidden = 1 << 0, - TSTreeOptionsExtra = 1 << 1, - TSTreeOptionsFragileLeft = 1 << 2, - TSTreeOptionsFragileRight = 1 << 3, +typedef struct { + bool hidden : 1; + bool concrete : 1; + bool extra : 1; + bool fragile_left : 1; + bool fragile_right : 1; } TSTreeOptions; struct TSTree { @@ -39,39 +40,35 @@ typedef struct { } TSTreeChild; static inline bool ts_tree_is_extra(const TSTree *tree) { - return !!(tree->options & TSTreeOptionsExtra); + return tree->options.extra; } static inline bool ts_tree_is_visible(const TSTree *tree) { - return !(tree->options & TSTreeOptionsHidden); -} - -static inline void ts_tree_set_options(TSTree *tree, TSTreeOptions options) { - tree->options = (TSTreeOptions)(tree->options | options); + return !(tree->options.hidden || tree->options.concrete); } static inline void ts_tree_set_extra(TSTree *tree) { - ts_tree_set_options(tree, TSTreeOptionsExtra); + tree->options.extra = true; } static inline void ts_tree_set_fragile_left(TSTree *tree) { - ts_tree_set_options(tree, TSTreeOptionsFragileLeft); + tree->options.fragile_left = true; } static inline void ts_tree_set_fragile_right(TSTree *tree) { - ts_tree_set_options(tree, TSTreeOptionsFragileRight); + tree->options.fragile_right = true; } static inline bool ts_tree_is_fragile_left(TSTree *tree) { - return tree->options & TSTreeOptionsFragileLeft; + return tree->options.fragile_left; } static inline bool ts_tree_is_fragile_right(TSTree *tree) { - return tree->options & TSTreeOptionsFragileRight; + return tree->options.fragile_right; } -TSTree *ts_tree_make_leaf(TSSymbol, TSLength, TSLength, bool); -TSTree *ts_tree_make_node(TSSymbol, size_t, TSTree **, bool); +TSTree *ts_tree_make_leaf(TSSymbol, TSLength, TSLength, TSNodeType); +TSTree *ts_tree_make_node(TSSymbol, size_t, TSTree **, TSNodeType); TSTree *ts_tree_make_error(TSLength size, TSLength padding, char lookahead_char); void ts_tree_retain(TSTree *tree); void ts_tree_release(TSTree *tree);