From 31f24395b48865d808665e78dc1504769e07737d Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Fri, 27 Sep 2024 16:28:50 -0400 Subject: [PATCH] feat: move generate logic to its own crate --- Cargo.lock | 23 ++++++++++++- Cargo.toml | 1 + cli/Cargo.toml | 4 +-- cli/generate/Cargo.toml | 33 +++++++++++++++++++ cli/generate/README.md | 4 +++ .../src}/build_tables/build_lex_table.rs | 2 +- .../src}/build_tables/build_parse_table.rs | 2 +- .../src}/build_tables/coincident_tokens.rs | 2 +- .../src}/build_tables/item.rs | 2 +- .../src}/build_tables/item_set_builder.rs | 2 +- .../src}/build_tables/minimize_parse_table.rs | 2 +- .../src}/build_tables/mod.rs | 2 +- .../src}/build_tables/token_conflicts.rs | 4 +-- cli/{src/generate => generate/src}/dedup.rs | 0 cli/{src/generate => generate/src}/dsl.js | 0 .../src}/grammar-schema.json | 0 .../src}/grammar_files.rs | 0 .../generate => generate/src}/grammars.rs | 0 .../generate/mod.rs => generate/src/lib.rs} | 4 +-- cli/{src/generate => generate/src}/nfa.rs | 0 .../generate => generate/src}/node_types.rs | 2 +- .../src}/parse_grammar.rs | 4 +-- .../src}/prepare_grammar/expand_repeats.rs | 2 +- .../src}/prepare_grammar/expand_tokens.rs | 4 +-- .../extract_default_aliases.rs | 4 +-- .../src}/prepare_grammar/extract_tokens.rs | 2 +- .../src}/prepare_grammar/flatten_grammar.rs | 4 +-- .../src}/prepare_grammar/intern_symbols.rs | 2 +- .../src}/prepare_grammar/mod.rs | 2 +- .../src}/prepare_grammar/process_inlines.rs | 4 +-- .../prepare_grammar/unicode-categories.json | 0 .../unicode-category-aliases.json | 0 .../prepare_grammar/unicode-properties.json | 0 .../unicode-property-aliases.json | 0 cli/{src/generate => generate/src}/render.rs | 0 cli/{src/generate => generate/src}/rules.rs | 0 cli/{src/generate => generate/src}/tables.rs | 0 cli/src/init.rs | 3 +- cli/src/lib.rs | 1 - cli/src/main.rs | 4 +-- cli/src/tests/corpus_test.rs | 5 ++- cli/src/tests/helpers/fixtures.rs | 3 +- cli/src/tests/node_test.rs | 6 ++-- cli/src/tests/parser_hang_test.rs | 6 ++-- cli/src/tests/parser_test.rs | 2 +- cli/src/tests/query_test.rs | 10 +++--- cli/src/wasm.rs | 3 +- 47 files changed, 103 insertions(+), 57 deletions(-) create mode 100644 cli/generate/Cargo.toml create mode 100644 cli/generate/README.md rename cli/{src/generate => generate/src}/build_tables/build_lex_table.rs (99%) rename cli/{src/generate => generate/src}/build_tables/build_parse_table.rs (99%) rename cli/{src/generate => generate/src}/build_tables/coincident_tokens.rs (99%) rename cli/{src/generate => generate/src}/build_tables/item.rs (99%) rename cli/{src/generate => generate/src}/build_tables/item_set_builder.rs (99%) rename cli/{src/generate => generate/src}/build_tables/minimize_parse_table.rs (99%) rename cli/{src/generate => generate/src}/build_tables/mod.rs (99%) rename cli/{src/generate => generate/src}/build_tables/token_conflicts.rs (99%) rename cli/{src/generate => generate/src}/dedup.rs (100%) rename cli/{src/generate => generate/src}/dsl.js (100%) rename cli/{src/generate => generate/src}/grammar-schema.json (100%) rename cli/{src/generate => generate/src}/grammar_files.rs (100%) rename cli/{src/generate => generate/src}/grammars.rs (100%) rename cli/{src/generate/mod.rs => generate/src/lib.rs} (98%) rename cli/{src/generate => generate/src}/nfa.rs (100%) rename cli/{src/generate => generate/src}/node_types.rs (99%) rename cli/{src/generate => generate/src}/parse_grammar.rs (99%) rename cli/{src/generate => generate/src}/prepare_grammar/expand_repeats.rs (99%) rename cli/{src/generate => generate/src}/prepare_grammar/expand_tokens.rs (99%) rename cli/{src/generate => generate/src}/prepare_grammar/extract_default_aliases.rs (99%) rename cli/{src/generate => generate/src}/prepare_grammar/extract_tokens.rs (99%) rename cli/{src/generate => generate/src}/prepare_grammar/flatten_grammar.rs (99%) rename cli/{src/generate => generate/src}/prepare_grammar/intern_symbols.rs (99%) rename cli/{src/generate => generate/src}/prepare_grammar/mod.rs (99%) rename cli/{src/generate => generate/src}/prepare_grammar/process_inlines.rs (99%) rename cli/{src/generate => generate/src}/prepare_grammar/unicode-categories.json (100%) rename cli/{src/generate => generate/src}/prepare_grammar/unicode-category-aliases.json (100%) rename cli/{src/generate => generate/src}/prepare_grammar/unicode-properties.json (100%) rename cli/{src/generate => generate/src}/prepare_grammar/unicode-property-aliases.json (100%) rename cli/{src/generate => generate/src}/render.rs (100%) rename cli/{src/generate => generate/src}/rules.rs (100%) rename cli/{src/generate => generate/src}/tables.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 8b131228..ac458b19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1505,12 +1505,12 @@ dependencies = [ "tiny_http", "tree-sitter", "tree-sitter-config", + "tree-sitter-generate", "tree-sitter-highlight", "tree-sitter-loader", "tree-sitter-tags", "tree-sitter-tests-proc-macro", "unindent", - "url", "walkdir", "wasmparser", "webbrowser", @@ -1526,6 +1526,27 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tree-sitter-generate" +version = "0.23.0" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap", + "indoc", + "lazy_static", + "log", + "regex", + "regex-syntax", + "rustc-hash", + "semver", + "serde", + "serde_json", + "smallbitvec", + "tree-sitter", + "url", +] + [[package]] name = "tree-sitter-highlight" version = "0.23.0" diff --git a/Cargo.toml b/Cargo.toml index 544e6149..09b0313d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,6 +89,7 @@ wasmparser = "0.215.0" webbrowser = "1.0.2" tree-sitter = { version = "0.23.0", path = "./lib" } +tree-sitter-generate = { version = "0.23.0", path = "./cli/generate" } tree-sitter-loader = { version = "0.23.0", path = "./cli/loader" } tree-sitter-config = { version = "0.23.0", path = "./cli/config" } tree-sitter-highlight = { version = "0.23.0", path = "./highlight" } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 36c2f9c8..869593fa 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -58,14 +58,12 @@ wasmparser.workspace = true webbrowser.workspace = true tree-sitter.workspace = true +tree-sitter-generate.workspace = true tree-sitter-config.workspace = true tree-sitter-highlight.workspace = true tree-sitter-loader.workspace = true tree-sitter-tags.workspace = true -[target."cfg(windows)".dependencies] -url = "2.5.2" - [dev-dependencies] tree_sitter_proc_macro = { path = "src/tests/proc_macro", package = "tree-sitter-tests-proc-macro" } diff --git a/cli/generate/Cargo.toml b/cli/generate/Cargo.toml new file mode 100644 index 00000000..a5c5f281 --- /dev/null +++ b/cli/generate/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "tree-sitter-generate" +version.workspace = true +description = "Library for generating C source code from a tree-sitter grammar" +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +readme = "README.md" +homepage.workspace = true +repository.workspace = true +license.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +anyhow.workspace = true +heck.workspace = true +indexmap.workspace = true +indoc.workspace = true +lazy_static.workspace = true +log.workspace = true +regex.workspace = true +regex-syntax.workspace = true +rustc-hash.workspace = true +semver.workspace = true +serde.workspace = true +serde_json.workspace = true +smallbitvec.workspace = true + +tree-sitter.workspace = true + +[target."cfg(windows)".dependencies] +url = "2.5.2" diff --git a/cli/generate/README.md b/cli/generate/README.md new file mode 100644 index 00000000..04a18900 --- /dev/null +++ b/cli/generate/README.md @@ -0,0 +1,4 @@ +# Tree-sitter Generate + +This helper crate implements the logic for the `tree-sitter generate` command, +and can be used by external tools to generate a parser from a grammar file. diff --git a/cli/src/generate/build_tables/build_lex_table.rs b/cli/generate/src/build_tables/build_lex_table.rs similarity index 99% rename from cli/src/generate/build_tables/build_lex_table.rs rename to cli/generate/src/build_tables/build_lex_table.rs index 215e45b1..9e079008 100644 --- a/cli/src/generate/build_tables/build_lex_table.rs +++ b/cli/generate/src/build_tables/build_lex_table.rs @@ -6,7 +6,7 @@ use std::{ use log::info; use super::{coincident_tokens::CoincidentTokenIndex, token_conflicts::TokenConflictMap}; -use crate::generate::{ +use crate::{ dedup::split_state_id_groups, grammars::{LexicalGrammar, SyntaxGrammar}, nfa::{CharacterSet, NfaCursor}, diff --git a/cli/src/generate/build_tables/build_parse_table.rs b/cli/generate/src/build_tables/build_parse_table.rs similarity index 99% rename from cli/src/generate/build_tables/build_parse_table.rs rename to cli/generate/src/build_tables/build_parse_table.rs index bb6d10bd..8d8154b8 100644 --- a/cli/src/generate/build_tables/build_parse_table.rs +++ b/cli/generate/src/build_tables/build_parse_table.rs @@ -13,7 +13,7 @@ use super::{ item::{ParseItem, ParseItemSet, ParseItemSetCore}, item_set_builder::ParseItemSetBuilder, }; -use crate::generate::{ +use crate::{ grammars::{ InlinedProductionMap, LexicalGrammar, PrecedenceEntry, SyntaxGrammar, VariableType, }, diff --git a/cli/src/generate/build_tables/coincident_tokens.rs b/cli/generate/src/build_tables/coincident_tokens.rs similarity index 99% rename from cli/src/generate/build_tables/coincident_tokens.rs rename to cli/generate/src/build_tables/coincident_tokens.rs index 9341145d..045f0375 100644 --- a/cli/src/generate/build_tables/coincident_tokens.rs +++ b/cli/generate/src/build_tables/coincident_tokens.rs @@ -1,6 +1,6 @@ use std::fmt; -use crate::generate::{ +use crate::{ grammars::LexicalGrammar, rules::Symbol, tables::{ParseStateId, ParseTable}, diff --git a/cli/src/generate/build_tables/item.rs b/cli/generate/src/build_tables/item.rs similarity index 99% rename from cli/src/generate/build_tables/item.rs rename to cli/generate/src/build_tables/item.rs index cfc72540..ebf03ed3 100644 --- a/cli/src/generate/build_tables/item.rs +++ b/cli/generate/src/build_tables/item.rs @@ -6,7 +6,7 @@ use std::{ use lazy_static::lazy_static; -use crate::generate::{ +use crate::{ grammars::{LexicalGrammar, Production, ProductionStep, SyntaxGrammar}, rules::{Associativity, Precedence, Symbol, SymbolType, TokenSet}, }; diff --git a/cli/src/generate/build_tables/item_set_builder.rs b/cli/generate/src/build_tables/item_set_builder.rs similarity index 99% rename from cli/src/generate/build_tables/item_set_builder.rs rename to cli/generate/src/build_tables/item_set_builder.rs index 16305bd3..3ef354a8 100644 --- a/cli/src/generate/build_tables/item_set_builder.rs +++ b/cli/generate/src/build_tables/item_set_builder.rs @@ -4,7 +4,7 @@ use std::{ }; use super::item::{ParseItem, ParseItemDisplay, ParseItemSet, TokenSetDisplay}; -use crate::generate::{ +use crate::{ grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar}, rules::{Symbol, SymbolType, TokenSet}, }; diff --git a/cli/src/generate/build_tables/minimize_parse_table.rs b/cli/generate/src/build_tables/minimize_parse_table.rs similarity index 99% rename from cli/src/generate/build_tables/minimize_parse_table.rs rename to cli/generate/src/build_tables/minimize_parse_table.rs index 5eb6260b..74f70869 100644 --- a/cli/src/generate/build_tables/minimize_parse_table.rs +++ b/cli/generate/src/build_tables/minimize_parse_table.rs @@ -6,7 +6,7 @@ use std::{ use log::info; use super::token_conflicts::TokenConflictMap; -use crate::generate::{ +use crate::{ dedup::split_state_id_groups, grammars::{LexicalGrammar, SyntaxGrammar, VariableType}, rules::{AliasMap, Symbol, TokenSet}, diff --git a/cli/src/generate/build_tables/mod.rs b/cli/generate/src/build_tables/mod.rs similarity index 99% rename from cli/src/generate/build_tables/mod.rs rename to cli/generate/src/build_tables/mod.rs index 34fa3fa1..1e3b65b3 100644 --- a/cli/src/generate/build_tables/mod.rs +++ b/cli/generate/src/build_tables/mod.rs @@ -19,7 +19,7 @@ use self::{ minimize_parse_table::minimize_parse_table, token_conflicts::TokenConflictMap, }; -use crate::generate::{ +use crate::{ grammars::{InlinedProductionMap, LexicalGrammar, SyntaxGrammar}, nfa::{CharacterSet, NfaCursor}, node_types::VariableInfo, diff --git a/cli/src/generate/build_tables/token_conflicts.rs b/cli/generate/src/build_tables/token_conflicts.rs similarity index 99% rename from cli/src/generate/build_tables/token_conflicts.rs rename to cli/generate/src/build_tables/token_conflicts.rs index 47a114d6..f79d362f 100644 --- a/cli/src/generate/build_tables/token_conflicts.rs +++ b/cli/generate/src/build_tables/token_conflicts.rs @@ -1,6 +1,6 @@ use std::{cmp::Ordering, collections::HashSet, fmt}; -use crate::generate::{ +use crate::{ build_tables::item::TokenSetDisplay, grammars::{LexicalGrammar, SyntaxGrammar}, nfa::{CharacterSet, NfaCursor, NfaTransition}, @@ -373,7 +373,7 @@ fn compute_conflict_status( #[cfg(test)] mod tests { use super::*; - use crate::generate::{ + use crate::{ grammars::{Variable, VariableType}, prepare_grammar::{expand_tokens, ExtractedLexicalGrammar}, rules::{Precedence, Rule, Symbol}, diff --git a/cli/src/generate/dedup.rs b/cli/generate/src/dedup.rs similarity index 100% rename from cli/src/generate/dedup.rs rename to cli/generate/src/dedup.rs diff --git a/cli/src/generate/dsl.js b/cli/generate/src/dsl.js similarity index 100% rename from cli/src/generate/dsl.js rename to cli/generate/src/dsl.js diff --git a/cli/src/generate/grammar-schema.json b/cli/generate/src/grammar-schema.json similarity index 100% rename from cli/src/generate/grammar-schema.json rename to cli/generate/src/grammar-schema.json diff --git a/cli/src/generate/grammar_files.rs b/cli/generate/src/grammar_files.rs similarity index 100% rename from cli/src/generate/grammar_files.rs rename to cli/generate/src/grammar_files.rs diff --git a/cli/src/generate/grammars.rs b/cli/generate/src/grammars.rs similarity index 100% rename from cli/src/generate/grammars.rs rename to cli/generate/src/grammars.rs diff --git a/cli/src/generate/mod.rs b/cli/generate/src/lib.rs similarity index 98% rename from cli/src/generate/mod.rs rename to cli/generate/src/lib.rs index f3b09339..e53710e5 100644 --- a/cli/src/generate/mod.rs +++ b/cli/generate/src/lib.rs @@ -39,8 +39,8 @@ struct GeneratedParser { node_types_json: String, } -pub const ALLOC_HEADER: &str = include_str!("../templates/alloc.h"); -pub const ARRAY_HEADER: &str = include_str!("../templates/array.h"); +pub const ALLOC_HEADER: &str = include_str!("../../src/templates/alloc.h"); +pub const ARRAY_HEADER: &str = include_str!("../../src/templates/array.h"); pub fn generate_parser_in_directory( repo_path: &Path, diff --git a/cli/src/generate/nfa.rs b/cli/generate/src/nfa.rs similarity index 100% rename from cli/src/generate/nfa.rs rename to cli/generate/src/nfa.rs diff --git a/cli/src/generate/node_types.rs b/cli/generate/src/node_types.rs similarity index 99% rename from cli/src/generate/node_types.rs rename to cli/generate/src/node_types.rs index 0ac159cd..ce0b99ce 100644 --- a/cli/src/generate/node_types.rs +++ b/cli/generate/src/node_types.rs @@ -730,7 +730,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::generate::{ + use crate::{ grammars::{ InputGrammar, LexicalVariable, Production, ProductionStep, SyntaxVariable, Variable, }, diff --git a/cli/src/generate/parse_grammar.rs b/cli/generate/src/parse_grammar.rs similarity index 99% rename from cli/src/generate/parse_grammar.rs rename to cli/generate/src/parse_grammar.rs index e801d531..d2f37c39 100644 --- a/cli/src/generate/parse_grammar.rs +++ b/cli/generate/src/parse_grammar.rs @@ -76,8 +76,8 @@ enum PrecedenceValueJSON { } #[derive(Deserialize)] -pub(crate) struct GrammarJSON { - pub(crate) name: String, +pub struct GrammarJSON { + pub name: String, rules: Map, #[serde(default)] precedences: Vec>, diff --git a/cli/src/generate/prepare_grammar/expand_repeats.rs b/cli/generate/src/prepare_grammar/expand_repeats.rs similarity index 99% rename from cli/src/generate/prepare_grammar/expand_repeats.rs rename to cli/generate/src/prepare_grammar/expand_repeats.rs index 4b97e53c..509ea4f2 100644 --- a/cli/src/generate/prepare_grammar/expand_repeats.rs +++ b/cli/generate/src/prepare_grammar/expand_repeats.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, mem}; use super::ExtractedSyntaxGrammar; -use crate::generate::{ +use crate::{ grammars::{Variable, VariableType}, rules::{Rule, Symbol}, }; diff --git a/cli/src/generate/prepare_grammar/expand_tokens.rs b/cli/generate/src/prepare_grammar/expand_tokens.rs similarity index 99% rename from cli/src/generate/prepare_grammar/expand_tokens.rs rename to cli/generate/src/prepare_grammar/expand_tokens.rs index a11f9ad9..97919768 100644 --- a/cli/src/generate/prepare_grammar/expand_tokens.rs +++ b/cli/generate/src/prepare_grammar/expand_tokens.rs @@ -8,7 +8,7 @@ use regex_syntax::ast::{ }; use super::ExtractedLexicalGrammar; -use crate::generate::{ +use crate::{ grammars::{LexicalGrammar, LexicalVariable}, nfa::{CharacterSet, Nfa, NfaState}, rules::{Precedence, Rule}, @@ -542,7 +542,7 @@ impl NfaBuilder { #[cfg(test)] mod tests { use super::*; - use crate::generate::{ + use crate::{ grammars::Variable, nfa::{NfaCursor, NfaTransition}, }; diff --git a/cli/src/generate/prepare_grammar/extract_default_aliases.rs b/cli/generate/src/prepare_grammar/extract_default_aliases.rs similarity index 99% rename from cli/src/generate/prepare_grammar/extract_default_aliases.rs rename to cli/generate/src/prepare_grammar/extract_default_aliases.rs index 68317913..68ea1e48 100644 --- a/cli/src/generate/prepare_grammar/extract_default_aliases.rs +++ b/cli/generate/src/prepare_grammar/extract_default_aliases.rs @@ -1,4 +1,4 @@ -use crate::generate::{ +use crate::{ grammars::{LexicalGrammar, SyntaxGrammar}, rules::{Alias, AliasMap, Symbol, SymbolType}, }; @@ -164,7 +164,7 @@ pub(super) fn extract_default_aliases( #[cfg(test)] mod tests { use super::*; - use crate::generate::{ + use crate::{ grammars::{LexicalVariable, Production, ProductionStep, SyntaxVariable, VariableType}, nfa::Nfa, }; diff --git a/cli/src/generate/prepare_grammar/extract_tokens.rs b/cli/generate/src/prepare_grammar/extract_tokens.rs similarity index 99% rename from cli/src/generate/prepare_grammar/extract_tokens.rs rename to cli/generate/src/prepare_grammar/extract_tokens.rs index 2fb99678..3fd85d6d 100644 --- a/cli/src/generate/prepare_grammar/extract_tokens.rs +++ b/cli/generate/src/prepare_grammar/extract_tokens.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, mem}; use anyhow::{anyhow, Result}; use super::{ExtractedLexicalGrammar, ExtractedSyntaxGrammar, InternedGrammar}; -use crate::generate::{ +use crate::{ grammars::{ExternalToken, Variable, VariableType}, rules::{MetadataParams, Rule, Symbol, SymbolType}, }; diff --git a/cli/src/generate/prepare_grammar/flatten_grammar.rs b/cli/generate/src/prepare_grammar/flatten_grammar.rs similarity index 99% rename from cli/src/generate/prepare_grammar/flatten_grammar.rs rename to cli/generate/src/prepare_grammar/flatten_grammar.rs index ff3f10e2..1a48706c 100644 --- a/cli/src/generate/prepare_grammar/flatten_grammar.rs +++ b/cli/generate/src/prepare_grammar/flatten_grammar.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Result}; use super::ExtractedSyntaxGrammar; -use crate::generate::{ +use crate::{ grammars::{Production, ProductionStep, SyntaxGrammar, SyntaxVariable, Variable}, rules::{Alias, Associativity, Precedence, Rule, Symbol}, }; @@ -231,7 +231,7 @@ unless they are used only as the grammar's start rule. #[cfg(test)] mod tests { use super::*; - use crate::generate::grammars::VariableType; + use crate::grammars::VariableType; #[test] fn test_flatten_grammar() { diff --git a/cli/src/generate/prepare_grammar/intern_symbols.rs b/cli/generate/src/prepare_grammar/intern_symbols.rs similarity index 99% rename from cli/src/generate/prepare_grammar/intern_symbols.rs rename to cli/generate/src/prepare_grammar/intern_symbols.rs index 567c61ac..a96d0b5b 100644 --- a/cli/src/generate/prepare_grammar/intern_symbols.rs +++ b/cli/generate/src/prepare_grammar/intern_symbols.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Result}; use super::InternedGrammar; -use crate::generate::{ +use crate::{ grammars::{InputGrammar, Variable, VariableType}, rules::{Rule, Symbol}, }; diff --git a/cli/src/generate/prepare_grammar/mod.rs b/cli/generate/src/prepare_grammar/mod.rs similarity index 99% rename from cli/src/generate/prepare_grammar/mod.rs rename to cli/generate/src/prepare_grammar/mod.rs index 16d7c85b..5e0a0471 100644 --- a/cli/src/generate/prepare_grammar/mod.rs +++ b/cli/generate/src/prepare_grammar/mod.rs @@ -164,7 +164,7 @@ fn validate_precedences(grammar: &InputGrammar) -> Result<()> { #[cfg(test)] mod tests { use super::*; - use crate::generate::grammars::VariableType; + use crate::grammars::VariableType; #[test] fn test_validate_precedences_with_undeclared_precedence() { diff --git a/cli/src/generate/prepare_grammar/process_inlines.rs b/cli/generate/src/prepare_grammar/process_inlines.rs similarity index 99% rename from cli/src/generate/prepare_grammar/process_inlines.rs rename to cli/generate/src/prepare_grammar/process_inlines.rs index ec43dd67..f2acffb6 100644 --- a/cli/src/generate/prepare_grammar/process_inlines.rs +++ b/cli/generate/src/prepare_grammar/process_inlines.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use anyhow::{anyhow, Result}; -use crate::generate::{ +use crate::{ grammars::{InlinedProductionMap, LexicalGrammar, Production, ProductionStep, SyntaxGrammar}, rules::SymbolType, }; @@ -225,7 +225,7 @@ pub(super) fn process_inlines( #[cfg(test)] mod tests { use super::*; - use crate::generate::{ + use crate::{ grammars::{LexicalVariable, SyntaxVariable, VariableType}, rules::{Associativity, Precedence, Symbol}, }; diff --git a/cli/src/generate/prepare_grammar/unicode-categories.json b/cli/generate/src/prepare_grammar/unicode-categories.json similarity index 100% rename from cli/src/generate/prepare_grammar/unicode-categories.json rename to cli/generate/src/prepare_grammar/unicode-categories.json diff --git a/cli/src/generate/prepare_grammar/unicode-category-aliases.json b/cli/generate/src/prepare_grammar/unicode-category-aliases.json similarity index 100% rename from cli/src/generate/prepare_grammar/unicode-category-aliases.json rename to cli/generate/src/prepare_grammar/unicode-category-aliases.json diff --git a/cli/src/generate/prepare_grammar/unicode-properties.json b/cli/generate/src/prepare_grammar/unicode-properties.json similarity index 100% rename from cli/src/generate/prepare_grammar/unicode-properties.json rename to cli/generate/src/prepare_grammar/unicode-properties.json diff --git a/cli/src/generate/prepare_grammar/unicode-property-aliases.json b/cli/generate/src/prepare_grammar/unicode-property-aliases.json similarity index 100% rename from cli/src/generate/prepare_grammar/unicode-property-aliases.json rename to cli/generate/src/prepare_grammar/unicode-property-aliases.json diff --git a/cli/src/generate/render.rs b/cli/generate/src/render.rs similarity index 100% rename from cli/src/generate/render.rs rename to cli/generate/src/render.rs diff --git a/cli/src/generate/rules.rs b/cli/generate/src/rules.rs similarity index 100% rename from cli/src/generate/rules.rs rename to cli/generate/src/rules.rs diff --git a/cli/src/generate/tables.rs b/cli/generate/src/tables.rs similarity index 100% rename from cli/src/generate/tables.rs rename to cli/generate/src/tables.rs diff --git a/cli/src/init.rs b/cli/src/init.rs index 4a962c96..ff87d044 100644 --- a/cli/src/init.rs +++ b/cli/src/init.rs @@ -11,8 +11,7 @@ use heck::{ToKebabCase, ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; use indoc::indoc; use serde::Deserialize; use serde_json::{json, Map, Value}; - -use crate::generate::write_file; +use tree_sitter_generate::write_file; const CLI_VERSION: &str = env!("CARGO_PKG_VERSION"); const CLI_VERSION_PLACEHOLDER: &str = "CLI_VERSION"; diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 9d5894b1..4a00747e 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1,7 +1,6 @@ #![doc = include_str!("../README.md")] pub mod fuzz; -pub mod generate; pub mod highlight; pub mod init; pub mod logger; diff --git a/cli/src/main.rs b/cli/src/main.rs index 206494ba..d60e6dcd 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -16,7 +16,7 @@ use tree_sitter_cli::{ fuzz_language_corpus, FuzzOptions, EDIT_COUNT, ITERATION_COUNT, LOG_ENABLED, LOG_GRAPH_ENABLED, START_SEED, }, - generate, highlight, + highlight, init::{generate_grammar_files, lookup_package_json_for_path}, logger, parse::{self, ParseFileOptions, ParseOutput}, @@ -461,7 +461,7 @@ impl Generate { version.parse().expect("invalid abi version flag") } }); - generate::generate_parser_in_directory( + tree_sitter_generate::generate_parser_in_directory( current_dir, self.grammar_path.as_deref(), abi_version, diff --git a/cli/src/tests/corpus_test.rs b/cli/src/tests/corpus_test.rs index f81d7543..7c9cc022 100644 --- a/cli/src/tests/corpus_test.rs +++ b/cli/src/tests/corpus_test.rs @@ -14,7 +14,6 @@ use crate::{ EDIT_COUNT, EXAMPLE_EXCLUDE, EXAMPLE_INCLUDE, ITERATION_COUNT, LANGUAGE_FILTER, LOG_GRAPH_ENABLED, START_SEED, }, - generate, parse::perform_edit, test::{parse_tests, print_diff, print_diff_key, strip_sexp_fields}, tests::{ @@ -353,8 +352,8 @@ fn test_feature_corpus_files() { grammar_path = test_path.join("grammar.json"); } let error_message_path = test_path.join("expected_error.txt"); - let grammar_json = generate::load_grammar_file(&grammar_path, None).unwrap(); - let generate_result = generate::generate_parser_for_grammar(&grammar_json); + let grammar_json = tree_sitter_generate::load_grammar_file(&grammar_path, None).unwrap(); + let generate_result = tree_sitter_generate::generate_parser_for_grammar(&grammar_json); if error_message_path.exists() { if EXAMPLE_INCLUDE.is_some() || EXAMPLE_EXCLUDE.is_some() { diff --git a/cli/src/tests/helpers/fixtures.rs b/cli/src/tests/helpers/fixtures.rs index 594c2575..766a86eb 100644 --- a/cli/src/tests/helpers/fixtures.rs +++ b/cli/src/tests/helpers/fixtures.rs @@ -6,12 +6,11 @@ use std::{ use anyhow::Context; use lazy_static::lazy_static; use tree_sitter::Language; +use tree_sitter_generate::{ALLOC_HEADER, ARRAY_HEADER}; use tree_sitter_highlight::HighlightConfiguration; use tree_sitter_loader::{CompileConfig, Loader}; use tree_sitter_tags::TagsConfiguration; -use crate::generate::{ALLOC_HEADER, ARRAY_HEADER}; - include!("./dirs.rs"); lazy_static! { diff --git a/cli/src/tests/node_test.rs b/cli/src/tests/node_test.rs index 20ae9c3b..0cbf1963 100644 --- a/cli/src/tests/node_test.rs +++ b/cli/src/tests/node_test.rs @@ -1,14 +1,12 @@ use tree_sitter::{Node, Parser, Point, Tree}; +use tree_sitter_generate::{generate_parser_for_grammar, load_grammar_file}; use super::{ get_random_edit, helpers::fixtures::{fixtures_dir, get_language, get_test_language}, Rand, }; -use crate::{ - generate::{generate_parser_for_grammar, load_grammar_file}, - parse::perform_edit, -}; +use crate::parse::perform_edit; const JSON_EXAMPLE: &str = r#" diff --git a/cli/src/tests/parser_hang_test.rs b/cli/src/tests/parser_hang_test.rs index e195a885..3f7b6394 100644 --- a/cli/src/tests/parser_hang_test.rs +++ b/cli/src/tests/parser_hang_test.rs @@ -7,11 +7,9 @@ use std::{ }; use tree_sitter::Parser; +use tree_sitter_generate::{generate_parser_for_grammar, load_grammar_file}; -use crate::{ - generate::{generate_parser_for_grammar, load_grammar_file}, - tests::helpers::fixtures::{fixtures_dir, get_test_language}, -}; +use crate::tests::helpers::fixtures::{fixtures_dir, get_test_language}; // The `sanitizing` cfg is required to don't run tests under specific sunitizer // because they don't work well with subprocesses _(it's an assumption)_. diff --git a/cli/src/tests/parser_test.rs b/cli/src/tests/parser_test.rs index 14cc9e3c..10868a2a 100644 --- a/cli/src/tests/parser_test.rs +++ b/cli/src/tests/parser_test.rs @@ -4,6 +4,7 @@ use std::{ }; use tree_sitter::{IncludedRangesError, InputEdit, LogType, Parser, Point, Range}; +use tree_sitter_generate::{generate_parser_for_grammar, load_grammar_file}; use tree_sitter_proc_macro::retry; use super::helpers::{ @@ -13,7 +14,6 @@ use super::helpers::{ }; use crate::{ fuzz::edits::Edit, - generate::{generate_parser_for_grammar, load_grammar_file}, parse::perform_edit, tests::{helpers::fixtures::fixtures_dir, invert_edit}, }; diff --git a/cli/src/tests/query_test.rs b/cli/src/tests/query_test.rs index 4f5ddd75..45dc3144 100644 --- a/cli/src/tests/query_test.rs +++ b/cli/src/tests/query_test.rs @@ -7,6 +7,7 @@ use tree_sitter::{ CaptureQuantifier, Language, Node, Parser, Point, Query, QueryCursor, QueryError, QueryErrorKind, QueryPredicate, QueryPredicateArg, QueryProperty, }; +use tree_sitter_generate::generate_parser_for_grammar; use unindent::Unindent; use super::helpers::{ @@ -14,12 +15,9 @@ use super::helpers::{ fixtures::{get_language, get_test_language}, query_helpers::{assert_query_matches, Match, Pattern}, }; -use crate::{ - generate::generate_parser_for_grammar, - tests::{ - helpers::query_helpers::{collect_captures, collect_matches}, - ITERATION_COUNT, - }, +use crate::tests::{ + helpers::query_helpers::{collect_captures, collect_matches}, + ITERATION_COUNT, }; lazy_static! { diff --git a/cli/src/wasm.rs b/cli/src/wasm.rs index 7782a330..eca6ac24 100644 --- a/cli/src/wasm.rs +++ b/cli/src/wasm.rs @@ -5,11 +5,10 @@ use std::{ use anyhow::{anyhow, Context, Result}; use tree_sitter::wasm_stdlib_symbols; +use tree_sitter_generate::parse_grammar::GrammarJSON; use tree_sitter_loader::Loader; use wasmparser::Parser; -use super::generate::parse_grammar::GrammarJSON; - pub fn load_language_wasm_file(language_dir: &Path) -> Result<(String, Vec)> { let grammar_name = get_grammar_name(language_dir) .with_context(|| "Failed to get wasm filename")