refactor(scripts): clean up bash scripts
This commit is contained in:
ObserverOfTime
parent
3950dddfde
commit
f50123a3ec
18 changed files with 241 additions and 261 deletions
7
.gitignore
vendored
7
.gitignore
vendored
|
|
@ -6,16 +6,19 @@ log*.html
|
|||
.cache
|
||||
.zig-cache
|
||||
|
||||
profile*
|
||||
fuzz-results
|
||||
|
||||
/tree-sitter.pc
|
||||
test/fuzz/out
|
||||
test/fixtures/grammars/*
|
||||
!test/fixtures/grammars/.gitkeep
|
||||
|
||||
package-lock.json
|
||||
node_modules
|
||||
|
||||
docs/assets/js/tree-sitter.js
|
||||
|
||||
/tree-sitter.pc
|
||||
|
||||
/target
|
||||
*.rs.bk
|
||||
*.a
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
set -e
|
||||
|
||||
function usage {
|
||||
cat <<-EOF
|
||||
cat <<EOF
|
||||
USAGE
|
||||
|
||||
$0 [-h] [-l language-name] [-e example-file-name] [-r repetition-count]
|
||||
|
|
@ -43,16 +43,20 @@ while getopts "hgl:e:r:" option; do
|
|||
r)
|
||||
export TREE_SITTER_BENCHMARK_REPETITION_COUNT=${OPTARG}
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${mode}" == "debug" ]]; then
|
||||
if [[ $mode == debug ]]; then
|
||||
test_binary=$(
|
||||
cargo bench benchmark -p tree-sitter-cli --no-run --message-format=json 2> /dev/null |\
|
||||
cargo bench benchmark -p tree-sitter-cli --no-run --message-format=json 2> /dev/null |
|
||||
jq -rs 'map(select(.target.name == "benchmark" and .executable))[0].executable'
|
||||
)
|
||||
env | grep TREE_SITTER
|
||||
echo $test_binary
|
||||
echo "$test_binary"
|
||||
else
|
||||
exec cargo bench benchmark -p tree-sitter-cli
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -1,70 +1,69 @@
|
|||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
|
||||
set -e
|
||||
|
||||
if [[ "$(uname -s)" != Linux ]]; then
|
||||
echo "Fuzzing is only supported on Linux"
|
||||
if [[ $(uname -s) != Linux ]]; then
|
||||
printf 'Fuzzing is only supported on Linux\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CC=${CC:-clang}
|
||||
CXX=${CXX:-clang++}
|
||||
|
||||
default_fuzz_flags="-fsanitize=fuzzer,address,undefined"
|
||||
default_fuzz_flags=-fsanitize=fuzzer,address,undefined
|
||||
|
||||
CFLAGS=${CFLAGS:-"$default_fuzz_flags"}
|
||||
CXXFLAGS=${CXXFLAGS:-"$default_fuzz_flags"}
|
||||
export CFLAGS="$default_fuzz_flags $CFLAGS"
|
||||
export CXXFLAGS="$default_fuzz_flags $CXXFLAGS"
|
||||
|
||||
export CFLAGS
|
||||
make CC="$CC" CXX="$CXX"
|
||||
make CC="$CC" CXX="$CXX" libtree-sitter.a
|
||||
|
||||
if [ -z "$@" ]; then
|
||||
languages=$(ls test/fixtures/grammars)
|
||||
if [[ -z $* ]]; then
|
||||
mapfile -t languages < <(ls test/fixtures/grammars)
|
||||
else
|
||||
languages="$@"
|
||||
languages=("$@")
|
||||
fi
|
||||
|
||||
mkdir -p test/fuzz/out
|
||||
|
||||
for lang in ${languages[@]}; do
|
||||
# skip typescript
|
||||
if [[ $lang == "typescript" ]]; then
|
||||
continue
|
||||
for lang in "${languages[@]}"; do
|
||||
# skip typescript & php
|
||||
if [[ $lang == typescript || $lang == php ]]; then
|
||||
continue
|
||||
fi
|
||||
echo "Building $lang fuzzer..."
|
||||
printf 'Building %s fuzzer...\n' "$lang"
|
||||
lang_dir="test/fixtures/grammars/$lang"
|
||||
lang_grammar="${lang_dir}/src/grammar.json"
|
||||
|
||||
# The following assumes each language is implemented as src/parser.c plus an
|
||||
# optional scanner in src/scanner.{c,cc}
|
||||
# optional scanner in src/scanner.c
|
||||
objects=()
|
||||
|
||||
lang_scanner="${lang_dir}/src/scanner"
|
||||
if [ -e "${lang_scanner}.cc" ]; then
|
||||
$CXX $CXXFLAGS -g -O1 "-I${lang_dir}/src" -c "${lang_scanner}.cc" -o "${lang_scanner}.o"
|
||||
objects+=("${lang_scanner}.o")
|
||||
elif [ -e "${lang_scanner}.c" ]; then
|
||||
$CC $CFLAGS -std=c11 -g -O1 "-I${lang_dir}/src" -c "${lang_scanner}.c" -o "${lang_scanner}.o"
|
||||
if [[ -f "${lang_scanner}.c" ]]; then
|
||||
$CC $CFLAGS -std=c11 -g -O1 -I "${lang_dir}/src" -c "${lang_scanner}.c" -o "${lang_scanner}.o"
|
||||
objects+=("${lang_scanner}.o")
|
||||
fi
|
||||
|
||||
|
||||
# Compiling with -O0 speeds up the build dramatically
|
||||
$CC $CFLAGS -g -O0 "-I${lang_dir}/src" "${lang_dir}/src/parser.c" -c -o "${lang_dir}/src/parser.o"
|
||||
$CC $CFLAGS -g -O0 -I "${lang_dir}/src" "${lang_dir}/src/parser.c" -c -o "${lang_dir}/src/parser.o"
|
||||
objects+=("${lang_dir}/src/parser.o")
|
||||
|
||||
highlights_filename="${lang_dir}/queries/highlights.scm"
|
||||
if [ -e "${highlights_filename}" ]; then
|
||||
if [[ -f "${highlights_filename}" ]]; then
|
||||
ts_lang_query_filename="${lang}.scm"
|
||||
cp "${highlights_filename}" "test/fuzz/out/${ts_lang_query_filename}"
|
||||
else
|
||||
ts_lang_query_filename=""
|
||||
fi
|
||||
|
||||
# FIXME: We should extract the grammar name from grammar.js. Use the name of
|
||||
# the directory instead. Also, the grammar name needs to be a valid C
|
||||
# identifier so replace any '-' characters
|
||||
ts_lang="tree_sitter_$(echo "$lang" | tr -- - _)"
|
||||
$CXX $CXXFLAGS -std=c++11 -I lib/include -D TS_LANG="$ts_lang" -D TS_LANG_QUERY_FILENAME="\"${ts_lang_query_filename}\"" \
|
||||
"test/fuzz/fuzzer.cc" "${objects[@]}" \
|
||||
ts_lang="tree_sitter_$(jq -r .name "$lang_grammar")"
|
||||
$CXX $CXXFLAGS -std=c++11 -Ilib/include \
|
||||
-D TS_LANG="$ts_lang" \
|
||||
-D TS_LANG_QUERY_FILENAME="\"${ts_lang_query_filename}\"" \
|
||||
test/fuzz/fuzzer.cc \
|
||||
"${objects[@]}" \
|
||||
libtree-sitter.a \
|
||||
-o "test/fuzz/out/${lang}_fuzzer"
|
||||
|
||||
|
|
@ -73,5 +72,5 @@ for lang in ${languages[@]}; do
|
|||
| if .type? == "STRING" or (.type? == "ALIAS" and .named? == false) then .value else empty end
|
||||
| select(test("\\S") and length == utf8bytelength)
|
||||
] | unique | .[]
|
||||
' | sort
|
||||
' "$lang_grammar" | sort > "test/fuzz/out/${lang}.dict"
|
||||
done
|
||||
|
|
|
|||
|
|
@ -29,18 +29,19 @@ EOF
|
|||
|
||||
set -e
|
||||
|
||||
web_dir=lib/binding_web
|
||||
src_dir=lib/src
|
||||
emscripten_flags="-O3"
|
||||
WEB_DIR=lib/binding_web
|
||||
SRC_DIR=lib/src
|
||||
EMSCRIPTEN_VERSION=$(< cli/loader/emscripten-version)
|
||||
|
||||
minify_js=1
|
||||
force_docker=0
|
||||
emscripen_version=$(cat "$(dirname "$0")"/../cli/loader/emscripten-version)
|
||||
emscripten_flags=(-O3)
|
||||
|
||||
while [[ $# > 0 ]]; do
|
||||
while (($# > 0)); do
|
||||
case "$1" in
|
||||
--debug)
|
||||
minify_js=0
|
||||
emscripten_flags="-s ASSERTIONS=1 -s SAFE_HEAP=1 -O0"
|
||||
emscripten_flags=(-s ASSERTIONS=1 -s SAFE_HEAP=1 -O0)
|
||||
;;
|
||||
|
||||
--help)
|
||||
|
|
@ -53,59 +54,60 @@ while [[ $# > 0 ]]; do
|
|||
;;
|
||||
|
||||
-v|--verbose)
|
||||
emscripten_flags="-s VERBOSE=1 -v $emscripten_flags"
|
||||
emscripten_flags+=(-s VERBOSE=1 -v)
|
||||
;;
|
||||
|
||||
*)
|
||||
usage
|
||||
echo "Unrecognized argument '$1'"
|
||||
printf "Unrecognized argument '%s'\n" "$1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
emcc=""
|
||||
docker=""
|
||||
if which emcc > /dev/null && [[ "$force_docker" == "0" ]]; then
|
||||
emcc=
|
||||
docker=
|
||||
if [[ $force_docker == 0 ]] && command -v emcc > /dev/null; then
|
||||
emcc=emcc
|
||||
elif which docker > /dev/null; then
|
||||
elif command -v docker > /dev/null; then
|
||||
# detect which one to use
|
||||
docker=docker
|
||||
elif which podman > /dev/null; then
|
||||
elif command -v podman > /dev/null; then
|
||||
docker=podman
|
||||
fi
|
||||
|
||||
if [ -z "$emcc" ] && [ -n "$docker" ]; then
|
||||
export PODMAN_USERNS=keep-id
|
||||
emcc="$docker run \
|
||||
--rm \
|
||||
-v $(pwd):/src:Z \
|
||||
-u $(id -u) \
|
||||
emscripten/emsdk:$emscripen_version \
|
||||
if [[ -z $emcc ]] && [[ -n $docker ]]; then
|
||||
if [[ $docker == podman ]]; then
|
||||
export PODMAN_USERNS=keep-id
|
||||
fi
|
||||
emcc="$docker run \
|
||||
--rm \
|
||||
-v $PWD:/src:Z \
|
||||
-u $UID \
|
||||
emscripten/emsdk:$EMSCRIPTEN_VERSION \
|
||||
emcc"
|
||||
fi
|
||||
|
||||
if [ -z "$emcc" ]; then
|
||||
if [[ "$force_docker" == "1" ]]; then
|
||||
echo 'You must have `docker` or `podman` on your PATH to run this script with --docker'
|
||||
if [[ -z $emcc ]]; then
|
||||
if [[ $force_docker == 1 ]]; then
|
||||
# shellcheck disable=SC2016
|
||||
printf 'You must have `docker` or `podman` in your PATH to run this script with --docker\n' >&2
|
||||
else
|
||||
echo 'You must have either `docker`, `podman`, or `emcc` on your PATH to run this script'
|
||||
# shellcheck disable=SC2016
|
||||
printf 'You must have either `docker`, `podman`, or `emcc` in your PATH to run this script\n' >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p target/scratch
|
||||
|
||||
runtime_methods='stringToUTF16','AsciiToString'
|
||||
runtime_methods=stringToUTF16,AsciiToString
|
||||
|
||||
# Remove quotes, add leading underscores, remove newlines, remove trailing comma.
|
||||
EXPORTED_FUNCTIONS=$( \
|
||||
cat ${src_dir}/wasm/stdlib-symbols.txt ${web_dir}/exports.txt | \
|
||||
sed -e 's/"//g' | \
|
||||
sed -e 's/^/_/g' | \
|
||||
tr -d '\n"' | \
|
||||
sed -e 's/,$//' \
|
||||
exported_functions=$(
|
||||
cat ${SRC_DIR}/wasm/stdlib-symbols.txt ${WEB_DIR}/exports.txt |
|
||||
sed -e 's/"//g;s/^/_/g' | tr -d '\n' | sed -e 's/,$//'
|
||||
)
|
||||
|
||||
# Use emscripten to generate `tree-sitter.js` and `tree-sitter.wasm`
|
||||
|
|
@ -118,40 +120,37 @@ $emcc \
|
|||
-s FILESYSTEM=0 \
|
||||
-s NODEJS_CATCH_EXIT=0 \
|
||||
-s NODEJS_CATCH_REJECTION=0 \
|
||||
-s EXPORTED_FUNCTIONS=${EXPORTED_FUNCTIONS} \
|
||||
-s EXPORTED_FUNCTIONS="${exported_functions}" \
|
||||
-s EXPORTED_RUNTIME_METHODS=$runtime_methods \
|
||||
$emscripten_flags \
|
||||
"${emscripten_flags[@]}" \
|
||||
-fno-exceptions \
|
||||
-std=c11 \
|
||||
-D 'fprintf(...)=' \
|
||||
-D NDEBUG= \
|
||||
-I ${src_dir} \
|
||||
-I ${SRC_DIR} \
|
||||
-I lib/include \
|
||||
--js-library ${web_dir}/imports.js \
|
||||
--pre-js ${web_dir}/prefix.js \
|
||||
--post-js ${web_dir}/binding.js \
|
||||
--post-js ${web_dir}/suffix.js \
|
||||
--js-library ${WEB_DIR}/imports.js \
|
||||
--pre-js ${WEB_DIR}/prefix.js \
|
||||
--post-js ${WEB_DIR}/binding.js \
|
||||
--post-js ${WEB_DIR}/suffix.js \
|
||||
lib/src/lib.c \
|
||||
${web_dir}/binding.c \
|
||||
${WEB_DIR}/binding.c \
|
||||
-o target/scratch/tree-sitter.js
|
||||
|
||||
# Use terser to write a minified version of `tree-sitter.js` into
|
||||
# the `lib/binding_web` directory.
|
||||
if [[ "$minify_js" == "1" ]]; then
|
||||
if [ ! -d ${web_dir}/node_modules/terser ]; then
|
||||
(
|
||||
cd ${web_dir}
|
||||
npm install
|
||||
)
|
||||
if [[ $minify_js == 1 ]]; then
|
||||
if [[ ! -d ${WEB_DIR}/node_modules/terser ]]; then
|
||||
(cd ${WEB_DIR} && npm install)
|
||||
fi
|
||||
${web_dir}/node_modules/.bin/terser \
|
||||
--compress \
|
||||
--mangle \
|
||||
--keep-classnames \
|
||||
-- target/scratch/tree-sitter.js \
|
||||
> ${web_dir}/tree-sitter.js
|
||||
${WEB_DIR}/node_modules/.bin/terser \
|
||||
--compress \
|
||||
--mangle \
|
||||
--keep-classnames \
|
||||
-- target/scratch/tree-sitter.js \
|
||||
> ${WEB_DIR}/tree-sitter.js
|
||||
else
|
||||
cp target/scratch/tree-sitter.js ${web_dir}/tree-sitter.js
|
||||
cp target/scratch/tree-sitter.js ${WEB_DIR}/tree-sitter.js
|
||||
fi
|
||||
|
||||
mv target/scratch/tree-sitter.wasm ${web_dir}/tree-sitter.wasm
|
||||
mv target/scratch/tree-sitter.wasm ${WEB_DIR}/tree-sitter.wasm
|
||||
|
|
|
|||
|
|
@ -1,17 +1,11 @@
|
|||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
# Remove quotes and commas
|
||||
EXPORTED_FUNCTIONS=$( \
|
||||
cat lib/src/wasm/stdlib-symbols.txt | \
|
||||
tr -d ',"' \
|
||||
)
|
||||
|
||||
EXPORT_FLAGS=""
|
||||
for function in ${EXPORTED_FUNCTIONS}; do
|
||||
EXPORT_FLAGS+=" -Wl,--export=${function}"
|
||||
done
|
||||
declare -a EXPORT_FLAGS
|
||||
while read -r -d, function; do
|
||||
EXPORT_FLAGS+=("-Wl,--export=${function:1:-1}")
|
||||
done < lib/src/wasm/stdlib-symbols.txt
|
||||
|
||||
target/wasi-sdk-21.0/bin/clang-17 \
|
||||
-o stdlib.wasm \
|
||||
|
|
@ -27,7 +21,7 @@ target/wasi-sdk-21.0/bin/clang-17 \
|
|||
-Wl,--export=reset_heap \
|
||||
-Wl,--export=__wasm_call_ctors \
|
||||
-Wl,--export=__stack_pointer \
|
||||
${EXPORT_FLAGS} \
|
||||
"${EXPORT_FLAGS[@]}" \
|
||||
lib/src/wasm/stdlib.c
|
||||
|
||||
xxd -C -i stdlib.wasm > lib/src/wasm/wasm-stdlib.h
|
||||
|
|
|
|||
|
|
@ -1,20 +1,12 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
src_dir="lib/src"
|
||||
src_dir=lib/src
|
||||
allocation_functions=(malloc calloc realloc free)
|
||||
|
||||
allocation_functions=(
|
||||
malloc
|
||||
calloc
|
||||
realloc
|
||||
free
|
||||
)
|
||||
|
||||
for function in ${allocation_functions[@]}; do
|
||||
usages=$(grep --line-number -E "\b${function}\(" -r "${src_dir}" --exclude alloc.h )
|
||||
|
||||
if [[ ! -z $usages ]]; then
|
||||
echo "The ${function} function should not be called directly, but is called here:"
|
||||
echo "$usages"
|
||||
for function in "${allocation_functions[@]}"; do
|
||||
usages=$(grep -n -E "\b${function}\(" -r $src_dir --exclude alloc.c --exclude stdlib.c)
|
||||
if [[ -n $usages ]]; then
|
||||
printf 'The %s function should not be called directly, but is called here:\n%s\n' "$function" "$usages" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
|
|
|||
|
|
@ -1,32 +1,26 @@
|
|||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
EMSCRIPTEN_VERSION=$(cat "$(dirname "$0")/../cli/loader/emscripten-version")
|
||||
EMSDK_DIR=target/emsdk
|
||||
EMSCRIPTEN_VERSION=$(< cli/loader/emscripten-version)
|
||||
|
||||
mkdir -p target
|
||||
EMSDK_DIR="./target/emsdk"
|
||||
|
||||
(
|
||||
if [ ! -f "$EMSDK_DIR/emsdk" ]; then
|
||||
echo 'Downloading emscripten SDK...'
|
||||
{
|
||||
if [[ ! -f $EMSDK_DIR/emsdk ]]; then
|
||||
printf 'Downloading emscripten SDK...\n'
|
||||
git clone https://github.com/emscripten-core/emsdk.git $EMSDK_DIR
|
||||
fi
|
||||
|
||||
cd $EMSDK_DIR
|
||||
|
||||
echo 'Updating emscripten SDK...'
|
||||
printf 'Updating emscripten SDK...\n'
|
||||
git reset --hard
|
||||
git pull
|
||||
./emsdk list
|
||||
|
||||
echo 'Installing emscripten...'
|
||||
./emsdk install $EMSCRIPTEN_VERSION
|
||||
printf 'Installing emscripten...\n'
|
||||
./emsdk install "$EMSCRIPTEN_VERSION"
|
||||
|
||||
echo 'Activating emscripten...'
|
||||
./emsdk activate $EMSCRIPTEN_VERSION
|
||||
) >&2
|
||||
|
||||
(
|
||||
echo "source \"$EMSDK_DIR/emsdk_env.sh\""
|
||||
)
|
||||
printf 'Activating emscripten...\n'
|
||||
./emsdk activate "$EMSCRIPTEN_VERSION"
|
||||
} >&2
|
||||
|
|
|
|||
|
|
@ -1,24 +1,23 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
GRAMMARS_DIR=$(dirname "$0")/../test/fixtures/grammars
|
||||
set -e
|
||||
|
||||
GRAMMARS_DIR="$PWD/test/fixtures/grammars"
|
||||
|
||||
fetch_grammar() {
|
||||
local grammar=$1
|
||||
local ref=$2
|
||||
local grammar_dir=${GRAMMARS_DIR}/${grammar}
|
||||
local grammar_dir="${GRAMMARS_DIR}/${grammar}"
|
||||
local grammar_url=https://github.com/tree-sitter/tree-sitter-${grammar}
|
||||
|
||||
echo "Updating ${grammar} grammar..."
|
||||
printf 'Updating %s grammar...\n' "$grammar"
|
||||
|
||||
if [ ! -d "$grammar_dir" ]; then
|
||||
if [[ ! -d "$grammar_dir" ]]; then
|
||||
git clone "$grammar_url" "$grammar_dir" --depth=1
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$grammar_dir" || exit
|
||||
git fetch origin "$ref" --depth=1
|
||||
git reset --hard FETCH_HEAD
|
||||
)
|
||||
git -C "$grammar_dir" fetch origin "$ref" --depth=1
|
||||
git -C "$grammar_dir" reset --hard FETCH_HEAD
|
||||
}
|
||||
|
||||
fetch_grammar bash master
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
output_path=lib/binding_rust/bindings.rs
|
||||
header_path='lib/include/tree_sitter/api.h'
|
||||
header_path=lib/include/tree_sitter/api.h
|
||||
no_derive_copy=(
|
||||
TSInput
|
||||
TSLanguage
|
||||
|
|
@ -19,13 +19,13 @@ no_copy=$(IFS='|'; echo "${no_derive_copy[*]}")
|
|||
|
||||
file_version=$(head -n1 "$output_path" | cut -d' ' -f6)
|
||||
tool_version=$(bindgen --version | cut -d' ' -f2)
|
||||
higher_version=$(echo -e "${file_version}\n${tool_version}" | sort -V | tail -n1)
|
||||
higher_version=$(printf '%s\n' "$file_version" "$tool_version" | sort -V | tail -n1)
|
||||
|
||||
if [ "$higher_version" != "$tool_version" ]; then
|
||||
echo "Latest used bindgen version was $file_version" >&2
|
||||
echo "Currently installed bindgen CLI version is $tool_version" >&2
|
||||
echo >&2
|
||||
echo "It's needed to upgrade bindgen CLI first with \`cargo install bindgen-cli\`" >&2
|
||||
if [[ "$higher_version" != "$tool_version" ]]; then
|
||||
printf 'Latest used bindgen version was %s\n' "$file_version" >&2
|
||||
printf 'Currently installed bindgen CLI version is %s\n\n' "$tool_version" >&2
|
||||
# shellcheck disable=SC2016
|
||||
printf 'You must upgrade bindgen CLI first with `cargo install bindgen-cli`\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
@ -33,11 +33,11 @@ bindgen \
|
|||
--no-layout-tests \
|
||||
--allowlist-type '^TS.*' \
|
||||
--allowlist-function '^ts_.*' \
|
||||
--allowlist-var "^TREE_SITTER.*" \
|
||||
--allowlist-var '^TREE_SITTER.*' \
|
||||
--blocklist-type '^__.*' \
|
||||
--no-prepend-enum-name \
|
||||
--no-copy "$no_copy" \
|
||||
$header_path \
|
||||
"$header_path" \
|
||||
-- \
|
||||
-D TREE_SITTER_FEATURE_WASM \
|
||||
> $output_path
|
||||
> "$output_path"
|
||||
|
|
|
|||
|
|
@ -2,32 +2,26 @@
|
|||
|
||||
set -e
|
||||
|
||||
root_dir=$PWD
|
||||
ROOT_DIR="$PWD"
|
||||
GRAMMARS_DIR="$ROOT_DIR/test/fixtures/grammars"
|
||||
|
||||
if [ "$CI" == true ]; then
|
||||
if [[ $CI == true ]]; then
|
||||
set -x
|
||||
tree_sitter="$TREE_SITTER"
|
||||
else
|
||||
cargo build --release
|
||||
tree_sitter=${root_dir}/target/release/tree-sitter
|
||||
TREE_SITTER="$ROOT_DIR/target/release/tree-sitter"
|
||||
fi
|
||||
|
||||
filter_grammar_name=$1
|
||||
|
||||
grammars_dir=${root_dir}/test/fixtures/grammars
|
||||
grammar_files=$(find "$grammars_dir" -name grammar.js | grep -v node_modules)
|
||||
filter_grammar_name="$1"
|
||||
|
||||
while read -r grammar_file; do
|
||||
grammar_dir=$(dirname "$grammar_file")
|
||||
grammar_name=$(basename "$grammar_dir")
|
||||
grammar_dir="${grammar_file%/*}"
|
||||
grammar_name="${grammar_dir##*/}"
|
||||
|
||||
if [[ -n $filter_grammar_name && "$filter_grammar_name" != "$grammar_name" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Regenerating ${grammar_name} parser"
|
||||
(
|
||||
cd "$grammar_dir"
|
||||
"$tree_sitter" generate src/grammar.json --no-bindings --abi=latest
|
||||
)
|
||||
done <<<"$grammar_files"
|
||||
printf 'Regenerating %s parser\n' "$grammar_name"
|
||||
(cd "$grammar_dir" && "$TREE_SITTER" generate src/grammar.json --no-bindings --abi=latest)
|
||||
done < <(find "$GRAMMARS_DIR" -name grammar.js -not -path '*/node_modules/*')
|
||||
|
|
|
|||
|
|
@ -2,35 +2,32 @@
|
|||
|
||||
set -e
|
||||
|
||||
root_dir=$PWD
|
||||
ROOT_DIR="$PWD"
|
||||
GRAMMARS_DIR="$ROOT_DIR/test/fixtures/grammars"
|
||||
|
||||
if [ "$CI" == true ]; then
|
||||
if [[ $CI == true ]]; then
|
||||
set -x
|
||||
tree_sitter="$TREE_SITTER"
|
||||
else
|
||||
cargo build --release
|
||||
tree_sitter=${root_dir}/target/release/tree-sitter
|
||||
TREE_SITTER="$ROOT_DIR/target/release/tree-sitter"
|
||||
fi
|
||||
|
||||
build_wasm_args=
|
||||
if [[ $1 == "--docker" ]]; then
|
||||
build_wasm_args="--docker"
|
||||
if [[ $1 == --docker ]]; then
|
||||
build_wasm_args=--docker
|
||||
shift
|
||||
fi
|
||||
|
||||
filter_grammar_name=$1
|
||||
|
||||
grammars_dir=${root_dir}/test/fixtures/grammars
|
||||
grammar_files=$(find "$grammars_dir" -name grammar.js | grep -v node_modules)
|
||||
filter_grammar_name="$1"
|
||||
|
||||
while read -r grammar_file; do
|
||||
grammar_dir=$(dirname "$grammar_file")
|
||||
grammar_name=$(basename "$grammar_dir")
|
||||
grammar_dir="${grammar_file%/*}"
|
||||
grammar_name="${grammar_dir##*/}"
|
||||
|
||||
if [[ -n $filter_grammar_name && "$filter_grammar_name" != "$grammar_name" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Compiling ${grammar_name} parser to wasm"
|
||||
"$tree_sitter" build --wasm $build_wasm_args -o target/release/tree-sitter-"${grammar_name}".wasm "$grammar_dir"
|
||||
done <<<"$grammar_files"
|
||||
printf 'Compiling %s parser to wasm\n' "$grammar_name"
|
||||
"$TREE_SITTER" build --wasm $build_wasm_args -o "target/release/tree-sitter-${grammar_name}.wasm" "$grammar_dir"
|
||||
done < <(find "$GRAMMARS_DIR" -name grammar.js -not -path '*/node_modules/*')
|
||||
|
|
|
|||
|
|
@ -2,17 +2,7 @@
|
|||
|
||||
set -e
|
||||
|
||||
symbol_file=$(mktemp)
|
||||
wasm_files=$(find target -maxdepth 2 -name 'tree-sitter-*.wasm')
|
||||
|
||||
while read -r wasm_file; do
|
||||
wasm-objdump \
|
||||
--details "$wasm_file" \
|
||||
--section Import \
|
||||
| egrep -o '<\w+>' \
|
||||
| tr -d '<>' \
|
||||
>> $symbol_file
|
||||
done <<< "$wasm_files"
|
||||
|
||||
sort -u -o $symbol_file $symbol_file
|
||||
cat $symbol_file
|
||||
wasm-objdump --details "$wasm_file" --section Import | \
|
||||
sed -n 's/.*<env\.\([A-Za-z0-9_]*\)>.*/\1/p'
|
||||
done < <(find target -maxdepth 2 -name 'tree-sitter-*.wasm') | sort -u
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
|
||||
# Usage:
|
||||
# script/heap-profile
|
||||
#
|
||||
|
|
@ -11,24 +11,26 @@
|
|||
|
||||
set -e
|
||||
|
||||
GRAMMARS_DIR=$PWD/test/fixtures/grammars
|
||||
GRAMMARS_DIR="$PWD/test/fixtures/grammars"
|
||||
|
||||
# Build the library
|
||||
make
|
||||
make libtree-sitter.a
|
||||
|
||||
# Build the heap-profiling harness
|
||||
clang++ \
|
||||
-I lib/include \
|
||||
-I $GRAMMARS_DIR \
|
||||
-D GRAMMARS_DIR=\"${GRAMMARS_DIR}/\" \
|
||||
-l tcmalloc \
|
||||
./libtree-sitter.a \
|
||||
test/profile/heap.cc \
|
||||
clang++ \
|
||||
-Wno-reorder-init-list \
|
||||
-Wno-c99-designator \
|
||||
-I lib/include \
|
||||
-I "$GRAMMARS_DIR" \
|
||||
-D GRAMMARS_DIR="\"${GRAMMARS_DIR}/\"" \
|
||||
test/profile/heap.cc \
|
||||
-l tcmalloc \
|
||||
libtree-sitter.a \
|
||||
-o target/heap-profile
|
||||
|
||||
# Run the harness with heap profiling enabled.
|
||||
export HEAPPROFILE=$PWD/profile
|
||||
target/heap-profile $@
|
||||
export HEAPPROFILE="$PWD/profile"
|
||||
target/heap-profile "$@"
|
||||
|
||||
# Extract statistics using pprof.
|
||||
pprof -top -cum profile.0001.heap
|
||||
|
|
|
|||
|
|
@ -1,22 +1,26 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
root=$(dirname "$0")/..
|
||||
export ASAN_OPTIONS="quarantine_size_mb=10:detect_leaks=1:symbolize=1"
|
||||
export UBSAN="print_stacktrace=1:halt_on_error=1:symbolize=1"
|
||||
|
||||
# check if CI env var exists
|
||||
|
||||
if [ -z "${CI:-}" ]; then
|
||||
declare -A mode_config=( ["halt"]="-timeout=1 -rss_limit_mb=2048" ["recover"]="-timeout=10 -rss_limit_mb=2048" )
|
||||
else
|
||||
declare -A mode_config=( ["halt"]="-max_total_time=120 -timeout=1 -rss_limit_mb=2048" ["recover"]="-time=120 -timeout=10 -rss_limit_mb=2048" )
|
||||
if (($# < 3)); then
|
||||
echo "usage: $0 <language> <halt|recover> <testcase> [libFuzzer args...]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$#" -lt 3 ]; then
|
||||
echo "usage: $0 <language> (halt|recover) <testcase> <libFuzzer args...>"
|
||||
exit 1
|
||||
set -eu
|
||||
|
||||
export ASAN_OPTIONS=quarantine_size_mb=10:detect_leaks=1:symbolize=1
|
||||
export UBSAN=print_stacktrace=1:halt_on_error=1:symbolize=1
|
||||
|
||||
# check if CI env var exists
|
||||
if [[ -z ${CI:-} ]]; then
|
||||
declare -A mode_config=(
|
||||
[halt]='-timeout=1 -rss_limit_mb=2048'
|
||||
[recover]='-timeout=10 -rss_limit_mb=2048'
|
||||
)
|
||||
else
|
||||
declare -A mode_config=(
|
||||
[halt]='-max_total_time=120 -timeout=1 -rss_limit_mb=2048'
|
||||
[recover]='-time=120 -timeout=10 -rss_limit_mb=2048'
|
||||
)
|
||||
fi
|
||||
|
||||
lang="$1"
|
||||
|
|
@ -27,4 +31,5 @@ testcase="$1"
|
|||
shift
|
||||
# Treat remainder of arguments as libFuzzer arguments
|
||||
|
||||
"${root}/test/fuzz/out/${lang}_fuzzer" "${mode_config[$mode]}" -runs=1 "${testcase}" "$@"
|
||||
# shellcheck disable=SC2086
|
||||
test/fuzz/out/${lang}_fuzzer ${mode_config[$mode]} -runs=1 "$testcase" "$@"
|
||||
|
|
|
|||
|
|
@ -1,22 +1,26 @@
|
|||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -eux
|
||||
|
||||
root=$(dirname "$0")/..
|
||||
export ASAN_OPTIONS="quarantine_size_mb=10:detect_leaks=1:symbolize=1"
|
||||
export UBSAN="print_stacktrace=1:halt_on_error=1:symbolize=1"
|
||||
|
||||
# check if CI env var exists
|
||||
|
||||
if [ -z "${CI:-}" ]; then
|
||||
declare -A mode_config=( ["halt"]="-timeout=1 -rss_limit_mb=2048" ["recover"]="-timeout=10 -rss_limit_mb=2048" )
|
||||
else
|
||||
declare -A mode_config=( ["halt"]="-max_total_time=120 -timeout=1 -rss_limit_mb=2048" ["recover"]="-time=120 -timeout=10 -rss_limit_mb=2048" )
|
||||
if (($# < 2)); then
|
||||
echo "usage: $0 <language> <halt|recover> [libFuzzer args...]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$#" -lt 2 ]; then
|
||||
echo "usage: $0 <language> <halt|recover> <libFuzzer args...>"
|
||||
exit 1
|
||||
set -eu
|
||||
|
||||
export ASAN_OPTIONS=quarantine_size_mb=10:detect_leaks=1:symbolize=1
|
||||
export UBSAN=print_stacktrace=1:halt_on_error=1:symbolize=1
|
||||
|
||||
# check if CI env var exists
|
||||
if [[ -z ${CI:-} ]]; then
|
||||
declare -A mode_config=(
|
||||
[halt]='-timeout=1 -rss_limit_mb=2048'
|
||||
[recover]='-timeout=10 -rss_limit_mb=2048'
|
||||
)
|
||||
else
|
||||
declare -A mode_config=(
|
||||
[halt]='-max_total_time=120 -timeout=1 -rss_limit_mb=2048'
|
||||
[recover]='-time=120 -timeout=10 -rss_limit_mb=2048'
|
||||
)
|
||||
fi
|
||||
|
||||
lang="$1"
|
||||
|
|
@ -26,7 +30,7 @@ shift
|
|||
# Treat remainder of arguments as libFuzzer arguments
|
||||
|
||||
# Fuzzing logs and testcases are always written to `pwd`, so `cd` there first
|
||||
results="${root}/test/fuzz/out/fuzz-results/${lang}"
|
||||
results="$PWD/test/fuzz/out/fuzz-results/${lang}"
|
||||
mkdir -p "${results}"
|
||||
cd "${results}"
|
||||
|
||||
|
|
@ -34,5 +38,5 @@ cd "${results}"
|
|||
# then be loaded on subsequent fuzzing runs
|
||||
mkdir -p corpus
|
||||
|
||||
pwd
|
||||
"../../${lang}_fuzzer" "-dict=../../${lang}.dict" "-artifact_prefix=${lang}_" -max_len=2048 "${mode_config[$mode]}" "./corpus" "$@"
|
||||
# shellcheck disable=SC2086
|
||||
../../${lang}_fuzzer -dict="../../${lang}.dict" -artifact_prefix=${lang}_ -max_len=2048 ${mode_config[$mode]} corpus "$@"
|
||||
|
|
|
|||
24
script/test
24
script/test
|
|
@ -3,7 +3,7 @@
|
|||
set -e
|
||||
|
||||
function usage {
|
||||
cat <<-EOF
|
||||
cat <<EOF
|
||||
USAGE
|
||||
|
||||
$0 [-adDg] [-s SEED] [-l LANGUAGE] [-e EXAMPLE]
|
||||
|
|
@ -32,7 +32,7 @@ EOF
|
|||
export RUST_BACKTRACE=full
|
||||
|
||||
mode=normal
|
||||
test_flags=""
|
||||
test_flags=()
|
||||
|
||||
while getopts "adDghl:e:s:i:" option; do
|
||||
case ${option} in
|
||||
|
|
@ -41,7 +41,7 @@ while getopts "adDghl:e:s:i:" option; do
|
|||
exit
|
||||
;;
|
||||
a)
|
||||
export CFLAGS="-fsanitize=undefined,address"
|
||||
export CFLAGS=-fsanitize=undefined,address
|
||||
|
||||
# When the Tree-sitter C library is compiled with the address sanitizer, the address sanitizer
|
||||
# runtime library needs to be linked into the final test executable. When using Xcode clang,
|
||||
|
|
@ -58,10 +58,10 @@ while getopts "adDghl:e:s:i:" option; do
|
|||
release=${BASH_REMATCH[1]}
|
||||
current_target=${BASH_REMATCH[2]}
|
||||
else
|
||||
echo "Failed to parse toolchain '${toolchain}'"
|
||||
printf "Failed to parse toolchain '%s'\n" "$toolchain" >&2
|
||||
fi
|
||||
|
||||
test_flags+=" --target ${current_target}"
|
||||
test_flags+=("--target=$current_target")
|
||||
;;
|
||||
e)
|
||||
export TREE_SITTER_EXAMPLE=${OPTARG}
|
||||
|
|
@ -81,17 +81,21 @@ while getopts "adDghl:e:s:i:" option; do
|
|||
g)
|
||||
mode=debug
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
shift $(expr $OPTIND - 1)
|
||||
shift $((OPTIND - 1))
|
||||
|
||||
if [[ "${mode}" == "debug" ]]; then
|
||||
if [[ ${mode} == debug ]]; then
|
||||
test_binary=$(
|
||||
cargo test $test_flags --no-run --message-format=json 2> /dev/null |\
|
||||
cargo test "${test_flags[@]}" --no-run --message-format=json 2> /dev/null |
|
||||
jq -rs 'map(select(.target.name == "tree-sitter-cli" and .executable))[0].executable'
|
||||
)
|
||||
lldb "${test_binary}" -- $1
|
||||
lldb "${test_binary}" -- "$1"
|
||||
else
|
||||
cargo test $test_flags $1 -- --nocapture
|
||||
cargo test "${test_flags[@]}" "$1" -- --nocapture
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ set -e
|
|||
|
||||
cd lib/binding_web
|
||||
|
||||
if [ ! -d "node_modules/chai" ] || [ ! -d "node_modules/mocha" ]; then
|
||||
echo "Installing test dependencies..."
|
||||
if [[ ! -d node_modules/chai ]] || [[ ! -d node_modules/mocha ]]; then
|
||||
printf 'Installing test dependencies...\n'
|
||||
npm install
|
||||
fi
|
||||
|
||||
./node_modules/.bin/mocha
|
||||
node_modules/.bin/mocha
|
||||
|
|
|
|||
|
|
@ -6,17 +6,17 @@ function scan_build {
|
|||
# which doesn't support the '-std=c++14' (it is available via '-std=c++1y').
|
||||
# Use the system-wide installed clang instead which is 3.5 and does support
|
||||
# '-std=c++14'.
|
||||
extra_args+=("--use-analyzer=$(which clang)")
|
||||
extra_args+=("--use-analyzer=$(command -v clang)")
|
||||
|
||||
# scan-build will try to guess which CXX should be used to compile the actual
|
||||
# code, which is usually g++ but we need g++5 in the CI. Explicitly pass
|
||||
# $CC/$CXX to scan-build if they are set in the environment.
|
||||
|
||||
if [[ ! -z "$CC" ]]; then
|
||||
if [[ -n $CC ]]; then
|
||||
extra_args+=("--use-cc=$CC")
|
||||
fi
|
||||
|
||||
if [[ ! -z "$CXX" ]]; then
|
||||
if [[ -n $CXX ]]; then
|
||||
extra_args+=("--use-c++=$CXX")
|
||||
fi
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue