From da61d7cac5b969ae82d475b1c7cfb92d2a06d445 Mon Sep 17 00:00:00 2001 From: Amaan Qureshi Date: Wed, 6 Aug 2025 20:10:16 -0400 Subject: [PATCH] feat: add nix flake --- .envrc | 1 + .gitignore | 2 + crates/cli/flake.nix | 118 +++++++++++++++++ docs/flake.nix | 38 ++++++ flake.lock | 61 +++++++++ flake.nix | 263 ++++++++++++++++++++++++++++++++++++++ lib/binding_web/flake.nix | 160 +++++++++++++++++++++++ lib/flake.nix | 54 ++++++++ 8 files changed, 697 insertions(+) create mode 100644 .envrc create mode 100644 crates/cli/flake.nix create mode 100644 docs/flake.nix create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 lib/binding_web/flake.nix create mode 100644 lib/flake.nix diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..3550a30f --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitignore b/.gitignore index d9a69cb6..bf1e36d6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ log*.html +.direnv .idea *.xcodeproj .vscode .cache .zig-cache +.direnv profile* fuzz-results diff --git a/crates/cli/flake.nix b/crates/cli/flake.nix new file mode 100644 index 00000000..b3cc2134 --- /dev/null +++ b/crates/cli/flake.nix @@ -0,0 +1,118 @@ +{ + perSystem = + { + self', + pkgs, + lib, + src, + version, + ... + }: + let + nativeBuildInputs = [ + pkgs.pkg-config + pkgs.nodejs_22 + ]; + + buildInputs = [ + pkgs.openssl + pkgs.installShellFiles + ]; + in + { + packages = { + cli = pkgs.rustPlatform.buildRustPackage { + inherit + src + version + nativeBuildInputs + buildInputs + ; + + pname = "tree-sitter-cli"; + + cargoLock.lockFile = ../../Cargo.lock; + + preBuild = '' + rm -rf test/fixtures + mkdir -p test/fixtures + cp -r ${self'.packages.test-grammars}/fixtures/* test/fixtures/ + chmod -R u+w test/fixtures + ''; + + preCheck = '' + export HOME=$TMPDIR + ''; + + doCheck = true; + + postInstall = '' + installShellCompletion --cmd tree-sitter \ + --bash <($out/bin/tree-sitter complete --shell bash) \ + --zsh <($out/bin/tree-sitter complete --shell zsh) \ + --fish <($out/bin/tree-sitter complete --shell fish) + ''; + + meta = { + description = "Tree-sitter CLI - A tool for developing, testing, and using Tree-sitter parsers"; + longDescription = '' + Tree-sitter is a parser generator tool and an incremental parsing library. + It can build a concrete syntax tree for a source file and efficiently update + the syntax tree as the source file is edited. This package provides the CLI + tool for developing, testing, and using Tree-sitter parsers. + ''; + homepage = "https://tree-sitter.github.io/tree-sitter"; + changelog = "https://github.com/tree-sitter/tree-sitter/releases/tag/v${version}"; + license = lib.licenses.mit; + maintainers = [ lib.maintainers.amaanq ]; + platforms = lib.platforms.all; + mainProgram = "tree-sitter"; + }; + }; + + rust-fmt = + pkgs.runCommand "rust-fmt-check" + { + nativeBuildInputs = [ + pkgs.cargo + pkgs.rustfmt + ]; + } + '' + cd ${src} + cargo fmt --all --check + touch $out + ''; + + rust-clippy = pkgs.rustPlatform.buildRustPackage { + inherit src version; + + pname = "rust-clippy-check"; + + cargoLock.lockFile = ../../Cargo.lock; + + nativeBuildInputs = [ + pkgs.pkg-config + pkgs.clippy + pkgs.cmake + pkgs.clang + pkgs.libclang + ]; + + buildInputs = [ pkgs.openssl ]; + + buildPhase = '' + export HOME=$TMPDIR + export LIBCLANG_PATH="${pkgs.libclang.lib}/lib" + cargo xtask clippy + ''; + + installPhase = '' + touch $out + ''; + + doCheck = false; + }; + }; + }; +} diff --git a/docs/flake.nix b/docs/flake.nix new file mode 100644 index 00000000..c752bf43 --- /dev/null +++ b/docs/flake.nix @@ -0,0 +1,38 @@ +{ + perSystem = + { + pkgs, + lib, + src, + version, + ... + }: + { + packages.docs = pkgs.stdenv.mkDerivation { + inherit src version; + + pname = "tree-sitter-docs"; + + nativeBuildInputs = [ + pkgs.mdbook + pkgs.mdbook-admonish + ]; + + buildPhase = '' + cd docs + mdbook build + ''; + + installPhase = '' + mkdir -p $out/share/doc + cp -r book $out/share/doc/tree-sitter + ''; + + meta = { + description = "Tree-sitter documentation"; + homepage = "https://tree-sitter.github.io/tree-sitter"; + license = lib.licenses.mit; + }; + }; + }; +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..cc94b957 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1754487366, + "narHash": "sha256-pHYj8gUBapuUzKV/kN/tR3Zvqc7o6gdFB9XKXIp1SQ8=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "af66ad14b28a127c5c0f3bbb298218fc63528a18", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1756542300, + "narHash": "sha256-tlOn88coG5fzdyqz6R93SQL5Gpq+m/DsWpekNFhqPQk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "d7600c775f877cd87b4f5a831c28aa94137377aa", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1753579242, + "narHash": "sha256-zvaMGVn14/Zz8hnp4VWT9xVnhc8vuL3TStRqwk22biA=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "0f36c44e01a6129be94e3ade315a5883f0228a6e", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..68bf8a4f --- /dev/null +++ b/flake.nix @@ -0,0 +1,263 @@ +{ + description = "Tree-sitter - A parser generator tool and an incremental parsing library"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-parts.url = "github:hercules-ci/flake-parts"; + }; + + outputs = + inputs@{ flake-parts, ... }: + flake-parts.lib.mkFlake { inherit inputs; } { + systems = [ + "x86_64-linux" + "aarch64-linux" + "x86_64-darwin" + "aarch64-darwin" + ]; + + imports = [ + ./crates/cli/flake.nix + ./lib/flake.nix + ./lib/binding_web/flake.nix + ./docs/flake.nix + ]; + + perSystem = + { + self', + pkgs, + lib, + ... + }: + let + version = "0.26.0"; + + src = pkgs.lib.cleanSourceWith { + src = ./.; + filter = + name: type: + let + baseName = baseNameOf name; + in + !( + lib.elem baseName [ + "target" + "node_modules" + ".git" + ".direnv" + "flake.lock" + ] + || lib.hasPrefix "result" baseName + ); + }; + + fixturesJson = lib.importJSON ./test/fixtures/fixtures.json; + + grammarHashes = { + bash = "sha256-vRaN/mNfpR+hdv2HVS1bzaW0o+HGjizRFsk3iinICJE="; + c = "sha256-gmzbdwvrKSo6C1fqTJFGxy8x0+T+vUTswm7F5sojzKc="; + cpp = "sha256-tP5Tu747V8QMCEBYwOEmMQUm8OjojpJdlRmjcJTbe2k="; + embedded-template = "sha256-nBQain0Lc21jOgQFfvkyq615ZmT8qdMxtqIoUcOcO3A="; + go = "sha256-y7bTET8ypPczPnMVlCaiZuswcA7vFrDOc2jlbfVk5Sk="; + html = "sha256-Pd5Me1twLGOrRB3pSMVX9M8VKenTK0896aoLznjNkGo="; + java = "sha256-OvEO1BLZLjP3jt4gar18kiXderksFKO0WFXDQqGLRIY="; + javascript = "sha256-2Jj/SUG+k8lHlGSuPZvHjJojvQFgDiZHZzH8xLu7suE="; + jsdoc = "sha256-Azzb2zBjAfwbEmAEO1YqhpaxtzbXmRjfIzRla2Hx+24="; + json = "sha256-DNZC2cTy1C8OaMOpEHM6NoRtOIbLaBf0CLXXWCKODlw="; + php = "sha256-jI7yzcoHS/tNxUqJI4aD1rdEZV3jMn1GZD0J+81Dyf0="; + python = "sha256-71Od4sUsxGEvTwmXX8hBvzqD55hnXkVJublrhp1GICg="; + ruby = "sha256-iu3MVJl0Qr/Ba+aOttmEzMiVY6EouGi5wGOx5ofROzA="; + rust = "sha256-y3sJURlSTM7LRRN5WGIAeslsdRZU522Tfcu6dnXH/XQ="; + typescript = "sha256-CU55+YoFJb6zWbJnbd38B7iEGkhukSVpBN7sli6GkGY="; + }; + + grammarSpecs = lib.listToAttrs ( + map (fixture: { + name = lib.elemAt fixture 0; + value = { + rev = lib.elemAt fixture 1; + sha256 = grammarHashes.${lib.elemAt fixture 0}; + }; + }) fixturesJson + ); + + fetchGrammar = + name: rev: sha256: + pkgs.fetchFromGitHub { + owner = "tree-sitter"; + repo = "tree-sitter-${name}"; + inherit rev sha256; + }; + + testGrammars = lib.mapAttrs (name: spec: fetchGrammar name spec.rev spec.sha256) grammarSpecs; + in + { + _module.args = { + inherit src version; + }; + + packages = { + default = self'.packages.cli; + + test-grammars = pkgs.stdenv.mkDerivation { + inherit src version; + + pname = "test-grammars"; + + buildPhase = '' + mkdir -p test/fixtures/grammars + ${lib.concatMapStrings (name: '' + cp -r ${testGrammars.${name}} test/fixtures/grammars/${name} + '') (lib.attrNames testGrammars)} + ''; + + installPhase = '' + mkdir -p $out + cp -r test/fixtures $out/fixtures + ''; + }; + }; + + apps = { + default = self'.apps.cli; + + cli = { + type = "app"; + program = "${self'.packages.cli}/bin/tree-sitter"; + meta.description = "Tree-sitter CLI for developing, testing, and using parsers"; + }; + + docs = { + type = "app"; + program = "${pkgs.writeShellScript "docs" '' + echo "📚 Serving documentation at http://localhost:3000" + cd docs && ${pkgs.mdbook}/bin/mdbook serve + ''}"; + meta.description = "Serve Tree-sitter documentation locally"; + }; + + format = { + type = "app"; + program = toString ( + pkgs.writeShellScript "format-all" '' + set -e + echo "Formatting..." + echo "" + echo "→ Rust..." + ${pkgs.cargo}/bin/cargo fmt --all + echo "→ Nix..." + ${pkgs.nixfmt-rfc-style}/bin/nixfmt *.nix crates/cli/*.nix lib/*.nix lib/binding_web/*.nix docs/*.nix + echo "→ Web (TypeScript/JavaScript)..." + cd lib/binding_web && ${pkgs.nodejs_22}/bin/npm install --silent && ${pkgs.nodejs_22}/bin/npm run lint:fix + cd ../.. + echo "" + echo "Formatting complete" + '' + ); + meta.description = "Format all Rust and Nix code"; + }; + + lint = { + type = "app"; + program = toString ( + pkgs.writeShellScript "lint-all" '' + set -e + echo "Linting code..." + echo "" + echo "→ Checking Rust formatting..." + ${pkgs.cargo}/bin/cargo fmt --all --check + echo "→ Running clippy..." + ${pkgs.cargo}/bin/cargo clippy --workspace --all-targets -- -D warnings + echo "→ Checking Nix formatting..." + ${pkgs.nixfmt-rfc-style}/bin/nixfmt --check *.nix crates/cli/*.nix lib/*.nix lib/binding_web/*.nix docs/*.nix + echo "→ Checking Web code..." + cd lib/binding_web && ${pkgs.nodejs_22}/bin/npm install --silent && ${pkgs.nodejs_22}/bin/npm run lint + cd ../.. + echo "" + echo "Linting complete" + '' + ); + meta.description = "Run all linting checks"; + }; + }; + + checks = { + inherit (self'.packages) + cli + lib + web-tree-sitter + web-lint + rust-fmt + rust-clippy + ; + + nix-fmt = + pkgs.runCommand "nix-fmt-check" + { + nativeBuildInputs = [ pkgs.nixfmt-rfc-style ]; + } + '' + cd ${src} + nixfmt --check *.nix crates/cli/*.nix lib/*.nix lib/binding_web/*.nix docs/*.nix + touch $out + ''; + }; + + formatter = pkgs.nixfmt-rfc-style; + + devShells.default = pkgs.mkShell { + buildInputs = [ + pkgs.cargo + pkgs.rustc + pkgs.clippy + pkgs.rust-analyzer + pkgs.rustfmt + + pkgs.cmake + pkgs.gnumake + pkgs.pkg-config + pkgs.clang + pkgs.libclang + + pkgs.nodejs_22 + pkgs.emscripten + pkgs.pkgsCross.wasi32.stdenv.cc + + pkgs.mdbook + pkgs.mdbook-admonish + + pkgs.git + pkgs.nixfmt-rfc-style + pkgs.openssl + pkgs.openssl.dev + ]; + + shellHook = '' + echo "Tree-sitter Dev Environment" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "" + echo "Packages:" + echo " nix build .#cli - Build CLI tool" + echo " nix build .#lib - Build C library" + echo " nix build .#web-tree-sitter - Build WASM bindings" + echo " nix build .#docs - Build documentation" + echo "" + echo "Apps:" + echo " nix run .#cli - Run tree-sitter CLI" + echo " nix run .#docs - Serve docs locally" + echo " nix run .#format - Format all code" + echo " nix run .#lint - Run all linting checks" + echo "" + echo "Tests & Checks:" + echo " nix flake check - Run all tests and checks" + echo "" + echo "Version: ${version}" + ''; + + RUST_BACKTRACE = 1; + LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; + }; + }; + }; +} diff --git a/lib/binding_web/flake.nix b/lib/binding_web/flake.nix new file mode 100644 index 00000000..ba972627 --- /dev/null +++ b/lib/binding_web/flake.nix @@ -0,0 +1,160 @@ +{ + perSystem = + { + self', + pkgs, + lib, + src, + version, + ... + }: + let + grammars = [ + "bash" + "c" + "cpp" + "embedded-template" + "html" + "javascript" + "json" + "python" + "rust" + "typescript" + ]; + + wasmTestGrammars = pkgs.stdenv.mkDerivation { + inherit src version; + + pname = "wasm-test-grammars"; + + nativeBuildInputs = [ + self'.packages.cli + pkgs.pkgsCross.wasi32.stdenv.cc + pkgs.nodejs_22 + ]; + + buildPhase = '' + export HOME=$TMPDIR + export TREE_SITTER_WASI_SDK_PATH=${pkgs.pkgsCross.wasi32.stdenv.cc} + export NIX_LDFLAGS="" + + cp -r ${self'.packages.test-grammars}/fixtures . + chmod -R u+w fixtures + + for grammar in ${lib.concatStringsSep " " grammars}; do + if [ -d "fixtures/grammars/$grammar" ]; then + echo "Building WASM for $grammar" + + if [ "$grammar" = "typescript" ]; then + tree-sitter build --wasm -o "tree-sitter-typescript.wasm" "fixtures/grammars/$grammar/typescript" + tree-sitter build --wasm -o "tree-sitter-tsx.wasm" "fixtures/grammars/$grammar/tsx" + else + tree-sitter build --wasm -o "tree-sitter-$grammar.wasm" "fixtures/grammars/$grammar" + fi + fi + done + ''; + + installPhase = '' + mkdir -p $out + for wasm in *.wasm; do + if [ -f "$wasm" ]; then + echo "Installing $wasm" + cp "$wasm" $out/ + fi + done + ''; + }; + in + { + packages = { + web-tree-sitter = pkgs.buildNpmPackage { + inherit src version; + + pname = "web-tree-sitter"; + + npmDepsHash = "sha256-y0GobcskcZTmju90TM64GjeWiBmPFCrTOg0yfccdB+Q="; + + nativeBuildInputs = [ + pkgs.rustPlatform.cargoSetupHook + pkgs.cargo + pkgs.pkg-config + pkgs.emscripten + ]; + + buildInputs = [ pkgs.openssl ]; + + cargoDeps = pkgs.rustPlatform.importCargoLock { + lockFile = ../../Cargo.lock; + }; + + doCheck = true; + + postPatch = '' + cp lib/binding_web/package{,-lock}.json . + ''; + + buildPhase = '' + cd lib/binding_web + CJS=true npm run build + CJS=true npm run build:debug + npm run build:debug + npm run build + ''; + + checkPhase = '' + cd ../../ + mkdir -p target/release + + for grammar in ${wasmTestGrammars}/*.wasm; do + if [ -f "$grammar" ]; then + cp "$grammar" target/release/ + fi + done + + cd lib/binding_web && npm test + ''; + + meta = { + description = "web-tree-sitter - WebAssembly bindings to the Tree-sitter parsing library."; + longDescription = '' + web-tree-sitter provides WebAssembly bindings to the Tree-sitter parsing library. + It can build a concrete syntax tree for a source file and efficiently update + the syntax tree as the source file is edited. This package provides the WebAssembly bindings + and a JavaScript API for using them in web browsers + ''; + homepage = "https://tree-sitter.github.io/tree-sitter"; + changelog = "https://github.com/tree-sitter/tree-sitter/releases/tag/v${version}"; + license = lib.licenses.mit; + maintainers = [ lib.maintainers.amaanq ]; + platforms = lib.platforms.all; + }; + }; + + web-lint = pkgs.buildNpmPackage { + inherit src version; + + pname = "web-tree-sitter-lint"; + + npmDepsHash = "sha256-y0GobcskcZTmju90TM64GjeWiBmPFCrTOg0yfccdB+Q="; + + postPatch = '' + cp lib/binding_web/package{,-lock}.json . + ''; + + buildPhase = '' + cd lib/binding_web + npm run lint + ''; + + installPhase = '' + touch $out + ''; + + meta = { + description = "Lint check for web-tree-sitter TypeScript/JavaScript code"; + }; + }; + }; + }; +} diff --git a/lib/flake.nix b/lib/flake.nix new file mode 100644 index 00000000..052d7eae --- /dev/null +++ b/lib/flake.nix @@ -0,0 +1,54 @@ +{ + perSystem = + { + pkgs, + lib, + src, + version, + ... + }: + { + packages.lib = pkgs.stdenv.mkDerivation { + inherit src version; + + pname = "tree-sitter"; + + nativeBuildInputs = [ + pkgs.cmake + pkgs.pkg-config + ]; + + sourceRoot = "source/lib"; + + cmakeFlags = [ + "-DBUILD_SHARED_LIBS=ON" + "-DCMAKE_INSTALL_LIBDIR=lib" + "-DCMAKE_INSTALL_INCLUDEDIR=include" + "-DTREE_SITTER_FEATURE_WASM=OFF" + ]; + + enableParallelBuilding = true; + + postInstall = '' + mkdir -p $out/{lib/pkgconfig,share/tree-sitter} + substituteInPlace $out/lib/pkgconfig/tree-sitter.pc \ + --replace-fail "\''${prefix}" "$out" 2>/dev/null + ''; + + meta = { + description = "Tree-sitter incremental parsing library"; + longDescription = '' + Tree-sitter is a parser generator tool and an incremental parsing library. + It can build a concrete syntax tree for a source file and efficiently update + the syntax tree as the source file is edited. This package provides the core + C library that can be used to parse source code using Tree-sitter grammars. + ''; + homepage = "https://tree-sitter.github.io/tree-sitter"; + changelog = "https://github.com/tree-sitter/tree-sitter/releases/tag/v${version}"; + license = lib.licenses.mit; + maintainers = [ lib.maintainers.amaanq ]; + platforms = lib.platforms.all; + }; + }; + }; +}