From c7b5f89392495fa144a504f28028f5e320ededa8 Mon Sep 17 00:00:00 2001 From: Will Lillis Date: Fri, 31 Oct 2025 18:23:21 -0400 Subject: [PATCH] feat(xtask): generate JSON schema for cli `TestSummary` --- Cargo.lock | 65 +++++ Cargo.toml | 1 + crates/cli/Cargo.toml | 1 + crates/cli/src/parse.rs | 3 +- crates/cli/src/test.rs | 35 ++- crates/xtask/Cargo.toml | 2 + crates/xtask/src/main.rs | 4 + crates/xtask/src/test_schema.rs | 25 ++ .../assets/schemas/test-summary.schema.json | 247 ++++++++++++++++++ 9 files changed, 376 insertions(+), 7 deletions(-) create mode 100644 crates/xtask/src/test_schema.rs create mode 100644 docs/src/assets/schemas/test-summary.schema.json diff --git a/Cargo.lock b/Cargo.lock index 1639883b..b351edac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -576,6 +576,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "either" version = "1.15.0" @@ -1438,6 +1444,26 @@ dependencies = [ "getrandom 0.2.16", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "regalloc2" version = "0.12.2" @@ -1603,6 +1629,31 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schemars" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d020396d1d138dc19f1165df7545479dcd58d93810dc5d646a16e55abefa80" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + [[package]] name = "semver" version = "1.0.27" @@ -1643,6 +1694,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_json" version = "1.0.145" @@ -1947,6 +2009,7 @@ dependencies = [ "pretty_assertions", "rand", "regex", + "schemars", "semver", "serde", "serde_json", @@ -2740,8 +2803,10 @@ dependencies = [ "notify", "notify-debouncer-full", "regex", + "schemars", "semver", "serde_json", + "tree-sitter-cli", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1d5454e5..5473d9a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -137,6 +137,7 @@ rand = "0.8.5" regex = "1.11.3" regex-syntax = "0.8.6" rustc-hash = "2.1.1" +schemars = "1.0.4" semver = { version = "1.0.27", features = ["serde"] } serde = { version = "1.0.219", features = ["derive"] } serde_json = { version = "1.0.145", features = ["preserve_order"] } diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index d8df92c8..31626eb2 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -54,6 +54,7 @@ log.workspace = true memchr.workspace = true rand.workspace = true regex.workspace = true +schemars.workspace = true semver.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/cli/src/parse.rs b/crates/cli/src/parse.rs index 84bf0e85..1023b0fc 100644 --- a/crates/cli/src/parse.rs +++ b/crates/cli/src/parse.rs @@ -11,6 +11,7 @@ use anstyle::{AnsiColor, Color, RgbColor}; use anyhow::{anyhow, Context, Result}; use clap::ValueEnum; use log::info; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use tree_sitter::{ ffi, InputEdit, Language, LogType, ParseOptions, ParseState, Parser, Point, Range, Tree, @@ -19,7 +20,7 @@ use tree_sitter::{ use crate::{fuzz::edits::Edit, logger::paint, util}; -#[derive(Debug, Default, Serialize)] +#[derive(Debug, Default, Serialize, JsonSchema)] pub struct Stats { pub successful_parses: usize, pub total_parses: usize, diff --git a/crates/cli/src/test.rs b/crates/cli/src/test.rs index 18d07d23..3753dcea 100644 --- a/crates/cli/src/test.rs +++ b/crates/cli/src/test.rs @@ -18,6 +18,7 @@ use regex::{ bytes::{Regex as ByteRegex, RegexBuilder as ByteRegexBuilder}, Regex, }; +use schemars::{JsonSchema, Schema, SchemaGenerator}; use serde::Serialize; use similar::{ChangeTag, TextDiff}; use tree_sitter::{format_sexp, Language, LogType, Parser, Query, Tree}; @@ -139,36 +140,47 @@ pub struct TestOptions<'a> { } /// A stateful object used to collect results from running a grammar's test suite -#[derive(Debug, Default, Serialize)] +#[derive(Debug, Default, Serialize, JsonSchema)] pub struct TestSummary { // Parse test results and associated data + #[schemars(schema_with = "schema_as_array")] #[serde(serialize_with = "serialize_as_array")] pub parse_results: TestResultHierarchy, pub parse_failures: Vec, pub parse_stats: Stats, + #[schemars(skip)] #[serde(skip)] pub has_parse_errors: bool, + #[schemars(skip)] #[serde(skip)] pub parse_stat_display: TestStats, // Other test results + #[schemars(schema_with = "schema_as_array")] #[serde(serialize_with = "serialize_as_array")] pub highlight_results: TestResultHierarchy, + #[schemars(schema_with = "schema_as_array")] #[serde(serialize_with = "serialize_as_array")] pub tag_results: TestResultHierarchy, + #[schemars(schema_with = "schema_as_array")] #[serde(serialize_with = "serialize_as_array")] pub query_results: TestResultHierarchy, // Data used during construction + #[schemars(skip)] #[serde(skip)] pub test_num: usize, // Options passed in from the CLI which control how the summary is displayed + #[schemars(skip)] #[serde(skip)] pub color: bool, + #[schemars(skip)] #[serde(skip)] pub overview_only: bool, + #[schemars(skip)] #[serde(skip)] pub update: bool, + #[schemars(skip)] #[serde(skip)] pub json: bool, } @@ -194,7 +206,7 @@ impl TestSummary { } } -#[derive(Debug, Default)] +#[derive(Debug, Default, JsonSchema)] pub struct TestResultHierarchy { root_group: Vec, traversal_idxs: Vec, @@ -207,6 +219,10 @@ where results.root_group.serialize(serializer) } +fn schema_as_array(gen: &mut SchemaGenerator) -> Schema { + gen.subschema_for::>() +} + /// Stores arbitrarily nested parent test groups and child cases. Supports creation /// in DFS traversal order impl TestResultHierarchy { @@ -313,14 +329,16 @@ impl<'a> Iterator for TestResultIterWithDepth<'a> { } } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, JsonSchema)] pub struct TestResult { pub name: String, + #[schemars(flatten)] #[serde(flatten)] pub info: TestInfo, } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, JsonSchema)] +#[schemars(untagged)] #[serde(untagged)] pub enum TestInfo { Group { @@ -329,6 +347,7 @@ pub enum TestInfo { ParseTest { outcome: TestOutcome, // True parse rate, adjusted parse rate + #[schemars(schema_with = "parse_rate_schema")] #[serde(serialize_with = "serialize_parse_rates")] parse_rate: Option<(f64, f64)>, test_num: usize, @@ -352,7 +371,11 @@ where } } -#[derive(Debug, Clone, Eq, PartialEq, Serialize)] +fn parse_rate_schema(gen: &mut SchemaGenerator) -> Schema { + gen.subschema_for::>() +} + +#[derive(Debug, Clone, Eq, PartialEq, Serialize, JsonSchema)] pub enum TestOutcome { // Parse outcomes Passed, @@ -726,7 +749,7 @@ impl std::fmt::Display for TestDiff<'_> { } } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, JsonSchema)] pub struct TestFailure { name: String, actual: String, diff --git a/crates/xtask/Cargo.toml b/crates/xtask/Cargo.toml index 607d64ad..90134d03 100644 --- a/crates/xtask/Cargo.toml +++ b/crates/xtask/Cargo.toml @@ -21,7 +21,9 @@ bindgen = { version = "0.72.0" } clap.workspace = true indoc.workspace = true regex.workspace = true +schemars.workspace = true semver.workspace = true serde_json.workspace = true +tree-sitter-cli = { path = "../cli/" } notify = "8.2.0" notify-debouncer-full = "0.6.0" diff --git a/crates/xtask/src/main.rs b/crates/xtask/src/main.rs index 46b2e796..3814cf66 100644 --- a/crates/xtask/src/main.rs +++ b/crates/xtask/src/main.rs @@ -7,6 +7,7 @@ mod embed_sources; mod fetch; mod generate; mod test; +mod test_schema; mod upgrade_wasmtime; use std::{path::Path, process::Command}; @@ -40,6 +41,8 @@ enum Commands { GenerateBindings, /// Generates the fixtures for testing tree-sitter. GenerateFixtures(GenerateFixtures), + /// Generates the JSON schema for the test runner summary. + GenerateTestSchema, /// Generate the list of exports from Tree-sitter Wasm files. GenerateWasmExports, /// Run the test suite @@ -236,6 +239,7 @@ fn run() -> Result<()> { Commands::GenerateFixtures(generate_fixtures_options) => { generate::run_fixtures(&generate_fixtures_options)?; } + Commands::GenerateTestSchema => test_schema::run_test_schema()?, Commands::GenerateWasmExports => generate::run_wasm_exports()?, Commands::Test(test_options) => test::run(&test_options)?, Commands::TestWasm => test::run_wasm()?, diff --git a/crates/xtask/src/test_schema.rs b/crates/xtask/src/test_schema.rs new file mode 100644 index 00000000..a2f65ed2 --- /dev/null +++ b/crates/xtask/src/test_schema.rs @@ -0,0 +1,25 @@ +use std::path::PathBuf; + +use anyhow::Result; +use serde_json::to_writer_pretty; + +use tree_sitter_cli::test::TestSummary; + +pub fn run_test_schema() -> Result<()> { + let schema = schemars::schema_for!(TestSummary); + + let xtask_path: PathBuf = env!("CARGO_MANIFEST_DIR").into(); + let schema_path = xtask_path + .parent() + .unwrap() + .parent() + .unwrap() + .join("docs") + .join("src") + .join("assets") + .join("schemas") + .join("test-summary.schema.json"); + let mut file = std::fs::File::create(schema_path)?; + + Ok(to_writer_pretty(&mut file, &schema)?) +} diff --git a/docs/src/assets/schemas/test-summary.schema.json b/docs/src/assets/schemas/test-summary.schema.json new file mode 100644 index 00000000..6211d60c --- /dev/null +++ b/docs/src/assets/schemas/test-summary.schema.json @@ -0,0 +1,247 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "TestSummary", + "description": "A stateful object used to collect results from running a grammar's test suite", + "type": "object", + "properties": { + "parse_results": { + "type": "array", + "items": { + "$ref": "#/$defs/TestResult" + } + }, + "parse_failures": { + "type": "array", + "items": { + "$ref": "#/$defs/TestFailure" + } + }, + "parse_stats": { + "$ref": "#/$defs/Stats" + }, + "highlight_results": { + "type": "array", + "items": { + "$ref": "#/$defs/TestResult" + } + }, + "tag_results": { + "type": "array", + "items": { + "$ref": "#/$defs/TestResult" + } + }, + "query_results": { + "type": "array", + "items": { + "$ref": "#/$defs/TestResult" + } + } + }, + "required": [ + "parse_results", + "parse_failures", + "parse_stats", + "highlight_results", + "tag_results", + "query_results" + ], + "$defs": { + "TestResult": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "type": "object", + "properties": { + "children": { + "type": "array", + "items": { + "$ref": "#/$defs/TestResult" + } + } + }, + "required": [ + "children" + ] + }, + { + "type": "object", + "properties": { + "outcome": { + "$ref": "#/$defs/TestOutcome" + }, + "parse_rate": { + "type": [ + "number", + "null" + ], + "format": "double" + }, + "test_num": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "outcome", + "parse_rate", + "test_num" + ] + }, + { + "type": "object", + "properties": { + "outcome": { + "$ref": "#/$defs/TestOutcome" + }, + "test_num": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "outcome", + "test_num" + ] + } + ] + }, + "TestOutcome": { + "oneOf": [ + { + "type": "string", + "enum": [ + "Passed", + "Failed", + "Updated", + "Skipped", + "Platform" + ] + }, + { + "type": "object", + "properties": { + "AssertionPassed": { + "type": "object", + "properties": { + "assertion_count": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "assertion_count" + ] + } + }, + "required": [ + "AssertionPassed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "AssertionFailed": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ] + } + }, + "required": [ + "AssertionFailed" + ], + "additionalProperties": false + } + ] + }, + "TestFailure": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "actual": { + "type": "string" + }, + "expected": { + "type": "string" + }, + "is_cst": { + "type": "boolean" + } + }, + "required": [ + "name", + "actual", + "expected", + "is_cst" + ] + }, + "Stats": { + "type": "object", + "properties": { + "successful_parses": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "total_parses": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "total_bytes": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "total_duration": { + "$ref": "#/$defs/Duration" + } + }, + "required": [ + "successful_parses", + "total_parses", + "total_bytes", + "total_duration" + ] + }, + "Duration": { + "type": "object", + "properties": { + "secs": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "nanos": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "secs", + "nanos" + ] + } + } +} \ No newline at end of file