Merge pull request #142 from tree-sitter/fuzz-halt-recover

Add 'halt' and 'recover' modes to fuzzer

script/build-fuzzers

@@ -54,10 +54,13 @@ for lang in ${languages[@]}; do
   $CC $CFLAGS -g -O0 "-I${lang_dir}/src" "${lang_dir}/src/parser.c" -c -o "${lang_dir}/src/parser.o"
   objects+=("${lang_dir}/src/parser.o")
 
-  $CXX $CXXFLAGS -std=c++11 -Iinclude -D TSLANG="tree_sitter_$lang" \
-    "test/fuzz/fuzzer.cc" "${objects[@]}" \
-    out/Fuzz/obj.target/libruntime.a "$LIB_FUZZER_PATH" \
-    -o "out/${lang}_fuzzer"
+  modes=(true halt false recover)
+  for i in 0 2; do
+    $CXX $CXXFLAGS -std=c++11 -Iinclude -D TS_HALT_ON_ERROR="${modes[i]}" -D TS_LANG="tree_sitter_$lang" \
+      "test/fuzz/fuzzer.cc" "${objects[@]}" \
+      out/Fuzz/obj.target/libruntime.a "$LIB_FUZZER_PATH" \
+      -o "out/${lang}_fuzzer_${modes[i+1]}"
+  done
 
   python test/fuzz/gen-dict.py "${lang_dir}/src/grammar.json" > "out/$lang.dict"
 done

script/reproduce

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-if [ "$#" -lt 2 ]; then
-  echo "usage: $0 <language> <testcase>"
-  exit 1
-fi
-
-lang="$1"
-testcase="$2"
-
-out="out"
-ASAN_OPTIONS="quarantine_size_mb=10:detect_leaks=1" UBSAN="print_stacktrace=1:halt_on_error=1" \
-  "${out}/${lang}_fuzzer" "$testcase" -timeout=1 -runs=100

script/reproduce

@@ -0,0 +1 @@
+run-fuzzer

script/run-fuzzer

@@ -1,23 +1,57 @@
 #!/bin/bash
 
-set -ex
+set -eux
 
-if [ "$#" -lt 1 ]; then
-  echo "usage: $0 <language> <libFuzzer args...>"
-  exit 1
+root=$(dirname "$0")/..
+export ASAN_OPTIONS="quarantine_size_mb=10:detect_leaks=1:symbolize=1"
+export UBSAN="print_stacktrace=1:halt_on_error=1:symbolize=1"
+
+declare -A mode_config=( ["halt"]="-timeout=1 -rss_limit_mb=256" ["recover"]="-timeout=10 -rss_limit_mb=256" )
+
+run_fuzzer() {
+  if [ "$#" -lt 2 ]; then
+    echo "usage: $0 <language> <halt|recover> <libFuzzer args...>"
+    exit 1
+  fi
+
+  lang="$1"
+  shift
+  mode="$1"
+  shift
+  # Treat remainder of arguments as libFuzzer arguments
+
+  # Fuzzing logs and testcases are always written to `pwd`, so `cd` there first
+  results="${root}/out/fuzz-results/${lang}_${mode}"
+  mkdir -p "${results}"
+  cd "${results}"
+
+  # Create a corpus directory, so new discoveries are stored on disk. These will
+  # then be loaded on subsequent fuzzing runs
+  mkdir -p corpus
+
+  "../../${lang}_fuzzer_${mode}" "-dict=../../${lang}.dict" "-artifact_prefix=${lang}_${mode}_" -max_len=2048 ${mode_config[$mode]} "./corpus" "$@"
+}
+
+reproduce() {
+  if [ "$#" -lt 3 ]; then
+    echo "usage: $0 <language> (halt|recover) <testcase> <libFuzzer args...>"
+    exit 1
+  fi
+
+  lang="$1"
+  shift
+  mode="$1"
+  shift
+  testcase="$1"
+  shift
+  # Treat remainder of arguments as libFuzzer arguments
+
+  "${root}/out/${lang}_fuzzer_${mode}" ${mode_config[$mode]} -runs=1 "${testcase}" "$@"
+}
+
+script=$(basename "$0")
+if [ "$script" == "run-fuzzer" ]; then
+  run_fuzzer "$@"
+elif [ "$script" == "reproduce" ]; then
+  reproduce "$@"
 fi
-
-lang="$1"
-shift # Treat remainder of arguments as libFuzzer arguments
-
-# Fuzzing logs and testcases are always written to `pwd`, so `cd` there first
-mkdir -p "fuzz-results/${lang}"
-cd "fuzz-results/${lang}"
-
-# Create a corpus directory, so new discoveries are stored on disk. These will
-# then be loaded on subsequent fuzzing runs
-mkdir -p corpus
-
-out="../../out"
-ASAN_OPTIONS="quarantine_size_mb=10:detect_leaks=1" UBSAN="print_stacktrace=1:halt_on_error=1" \
-  "${out}/${lang}_fuzzer" "-dict=${out}/${lang}.dict" "-artifact_prefix=${lang}_" -max_len=128 -timeout=1 "./corpus" "$@"

test/fuzz/README.md

@@ -4,12 +4,12 @@ The tree-sitter fuzzing support requires 1) the `libFuzzer` runtime library and
 
 ## libFuzzer
 
-The main fuzzing logic is implemented by `libFuzzer` which is part of the LLVM project but is not shipped by distros. It will need to be built from source but does not require building the _whole_ LLVM project. LLVM can be downloaded from llvm.org using SVN or [llvm-mirror](https://github.com/llvm-mirror/llvm) using git. `libFuzzer` can be built as, e.g.:
+The main fuzzing logic is implemented by `libFuzzer` which is part of the compiler-rt project but is not shipped by distros. `libFuzzer` will need to be built from source, e.g.:
 
 ```
 cd ~/src
-git clone https://github.com/llvm-mirror/llvm
-cd llvm/lib/Fuzzer
+git clone https://github.com/llvm-mirror/compiler-rt
+cd compiler-rt/lib/fuzzer
 ./build.sh
 ```
 
@@ -21,7 +21,7 @@ The fuzzers can then be built with:
 ```
 export CLANG_DIR=$HOME/src/third_party/llvm-build/Release+Asserts/bin
 CC="$CLANG_DIR/clang" CXX="$CLANG_DIR/clang++" LINK="$CLANG_DIR/clang++" \
-  LIB_FUZZER_PATH=$HOME/src/llvm/lib/Fuzzer/libFuzzer.a \
+  LIB_FUZZER_PATH=$HOME/src/compiler-rt/lib/fuzzer/libFuzzer.a \
   ./script/build_fuzzers
 ```
 
@@ -29,15 +29,15 @@ This will generate a separate fuzzer for each grammar defined in `test/fixtures/
 
 The `run-fuzzer` script handles running an individual fuzzer with a sensible default set of arguments:
 ```
-./script/run-fuzzer <grammar-name> <extra libFuzzer arguments...>
+./script/run-fuzzer <grammar-name> (halt|recover) <extra libFuzzer arguments...>
 ```
 
 which will log information to stdout. Failing testcases and a fuzz corpus will be saved to `fuzz-results/<grammar-name>`. The most important extra `libFuzzer` options are `-jobs` and `-workers` which allow parallel fuzzing. This is can done with, e.g.:
 ```
-./script/run-fuzzer <grammer-name> -jobs=32 -workers=32
+./script/run-fuzzer <grammer-name> halt -jobs=32 -workers=32
 ```
 
 The testcase can be used to reproduce the crash by running:
 ```
-./script/reproduce <grammar-name> <path-to-testcase>
+./script/reproduce <grammar-name> (halt|recover) <path-to-testcase>
 ```

test/fuzz/fuzzer.cc

@@ -7,17 +7,17 @@ TSLogger logger = {
   .log = test_log,
 };
 
-extern "C" const TSLanguage *TSLANG();
+extern "C" const TSLanguage *TS_LANG();
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   const char *str = reinterpret_cast<const char *>(data);
 
   TSDocument *document = ts_document_new();
-  ts_document_set_language(document, TSLANG());
+  ts_document_set_language(document, TS_LANG());
   ts_document_set_input_string_with_length(document, str, size);
 
   TSParseOptions options = {};
-  options.halt_on_error = false;
+  options.halt_on_error = TS_HALT_ON_ERROR;
   ts_document_parse_with_options(document, options);
 
   TSNode root_node = ts_document_root_node(document);