Merge branch 'MozillaSecurity:master' into master

Author: moz-mdauer

.gitignore

@@ -12,3 +12,6 @@
 # Python egg metadata, regenerated from source files by setuptools.
 *.egg-info
 .eggs/
+
+# Intellij IDEA
+.idea/

recipes/linux/common.sh

@@ -81,7 +81,7 @@ function resolve-url () {
 
 # wrap curl with sane defaults
 function retry-curl () {
-  curl --connect-timeout 25 --fail --location --retry 5 --show-error --silent --write-out "%{stderr}[downloaded %{url_effective}]\n" "$@"
+  curl --connect-timeout 25 --fail --location --retry 5 --retry-all-errors --show-error --silent --write-out "%{stderr}[downloaded %{url_effective}]\n" "$@"
 }
 
 function get-deadline () {

services/afl/Dockerfile

@@ -29,6 +29,7 @@ COPY \
     services/afl/pyproject.toml \
     services/nyx/nyx_utils.py \
     /srv/repos/nyx_utils/
+COPY services/afl/patches/ /home/worker/patches/
 RUN /srv/repos/setup/setup.sh
 COPY \
     services/afl/launch-root.sh \

services/afl/launch-worker.sh

@@ -75,7 +75,6 @@ fi
 ASAN_OPTIONS=\
 abort_on_error=1:\
 hard_rss_limit_mb=4096:\
-log_path=/tmp/data.log:\
 max_allocation_size_mb=3073:\
 strip_path_prefix=/builds/worker/workspace/build/src/:\
 symbolize=0:\
@@ -94,9 +93,20 @@ if [[ "$COVERAGE" = 1 ]]; then
   export SOURCE_URL
   REVISION="$(retry-curl --compressed "$ARTIFACT_ROOT/coverage-revision.txt")"
   export REVISION
+
+  export AFL_FAST_CAL=1
 fi
 
 TARGET_BIN="$(./setup-target.sh)"
+JS="${JS:-0}"
+if [[ "$JS" = 1 ]] || [[ -n "$JSRT" ]]
+then
+  export GCOV_PREFIX="$HOME/js"
+else
+  export GCOV_PREFIX="$HOME/firefox"
+fi
+GCOV_PREFIX_STRIP="$(grep pathprefix "$HOME/${TARGET_BIN}.fuzzmanagerconf" | grep -E -o "/.+$" | tr -cd '/' | wc -c)"
+export GCOV_PREFIX_STRIP
 
 mkdir -p corpus.out
 
@@ -120,14 +130,19 @@ DAEMON_ARGS=(
   --afl-binary-dir /opt/afl-instrumentation/bin
   --afl-timeout "${AFL_TIMEOUT-30000}"
   --afl
+  --instances "${AFL_INSTANCES:-$(ncpu)}"
   --stats ./stats
   --memory-limit "${MEMORY_LIMIT:-0}"
   "$TARGET_BIN"
 )
 
+unset AFL_INSTANCES
+
 S3_PROJECT="${S3_PROJECT:-afl-$FUZZER}"
 S3_PROJECT_ARGS=(--provider GCS --bucket guided-fuzzing-data --project "$S3_PROJECT")
 
+export AFL_MAP_SIZE=8388608
+
 if [[ -n "$S3_CORPUS_REFRESH" ]]
 then
   update-status "starting corpus refresh"
@@ -157,17 +172,13 @@ else
     echo "Hello world" > ./corpus/input0
   fi
 
-  instance_count="${AFL_INSTANCES:-$(ncpu)}"
-  unset AFL_INSTANCES
-  export AFL_MAP_SIZE=8388608
   # run and watch for results
   update-status "launching guided-fuzzing-daemon"
   time xvfb-run guided-fuzzing-daemon "${S3_PROJECT_ARGS[@]}" \
     --afl-log-pattern /logs/afl%d.log \
     --fuzzmanager \
     --max-runtime "$(get-target-time)" \
     --afl-async-corpus \
-    --instances "$instance_count" \
     --queue-upload \
     --tool "$TOOLNAME" \
     --corpus-in ./corpus \
@@ -182,7 +193,20 @@ fi
 
 if [[ $COVERAGE -eq 1 ]]
 then
-  # TODO: coverage.json
+  retry-curl --compressed -O "$SOURCE_URL"
+  unzip source.zip
+
+  # Collect coverage count data.
+  RUST_BACKTRACE=1 grcov "$GCOV_PREFIX" \
+    -t coveralls+ \
+    --commit-sha "$REVISION" \
+    --token NONE \
+    --guess-directory-when-missing \
+    --ignore-not-existing \
+    -p "$(rg -Nor '$1' "pathprefix = (.*)" "$HOME/${TARGET_BIN}.fuzzmanagerconf")" \
+    -s "./${REPO-mozilla-central}-$REVISION" \
+    > ./coverage.json
+
   # Submit coverage data.
   cov-reporter \
     --repository mozilla-central \

services/afl/patches/afl-cmin.patch

@@ -0,0 +1,122 @@
+diff --git a/afl-cmin b/afl-cmin
+index a88460a8..e43877ac 100755
+--- a/afl-cmin
++++ b/afl-cmin
+@@ -258,7 +258,7 @@ BEGIN {
+   # sanity checks
+   if (!prog_args[0] || !in_dir || !out_dir) usage()
+ 
+-  target_bin = prog_args[0] 
++  target_bin = prog_args[0]
+ 
+   # Do a sanity check to discourage the use of /tmp, since we can't really
+   # handle this safely from an awk script.
+@@ -330,14 +330,18 @@ BEGIN {
+     target_bin = tnew
+   }
+ 
+-  if (0 == system ( "grep -aq AFL_DUMP_MAP_SIZE " target_bin )) {
+-    echo "[!] Trying to obtain the map size of the target ..."
+-    get_map_size = "AFL_DUMP_MAP_SIZE=1 " target_bin
+-    get_map_size | getline mapsize
+-    close(get_map_size)
+-    if (mapsize && mapsize > 65535 && mapsize < 100000000) {
+-      AFL_MAP_SIZE = "AFL_MAP_SIZE="mapsize" "
+-      print "[+] Setting "AFL_MAP_SIZE
++  if (!nyx_mode) {
++    if (!ENVIRON["AFL_MAP_SIZE"] && 0 == system ( "grep -aq AFL_DUMP_MAP_SIZE " target_bin )) {
++      echo "[!] Trying to obtain the map size of the target ..."
++      get_map_size = "AFL_DUMP_MAP_SIZE=1 " target_bin
++      get_map_size | getline mapsize
++      close(get_map_size)
++      if (mapsize && mapsize > 65535 && mapsize < 100000000) {
++        AFL_MAP_SIZE = "AFL_MAP_SIZE="mapsize" "
++        print "[+] Setting "AFL_MAP_SIZE
++      }
++    } else {
++      AFL_MAP_SIZE = "AFL_MAP_SIZE=" ENVIRON["AFL_MAP_SIZE"] " "
+     }
+   }
+ 
+@@ -348,6 +352,8 @@ BEGIN {
+     }
+   }
+ 
++  AFL_PRELOAD = ("AFL_PRELOAD" in ENVIRON) ? "AFL_PRELOAD=" ENVIRON["AFL_PRELOAD"] " " : ""
++
+   if (0 != system( "test -d "in_dir )) {
+     print "[-] Error: directory '"in_dir"' not found." > "/dev/stderr"
+     exit 1
+@@ -470,10 +476,10 @@ BEGIN {
+     print "[*] Testing the target binary..."
+ 
+     if (!stdin_file) {
+-      system(AFL_MAP_SIZE "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"")
++      system(AFL_MAP_SIZE AFL_PRELOAD "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"")
+     } else {
+       system("cp \""in_dir"/"first_file"\" "stdin_file)
+-      system(AFL_MAP_SIZE "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
++      system(AFL_MAP_SIZE AFL_PRELOAD "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
+     }
+ 
+     first_count = 0
+@@ -537,12 +543,12 @@ BEGIN {
+     for (i = 1; i <= threads; i++) {
+ 
+       if (!stdin_file) {
+-#        print " { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -- \""target_bin"\" "prog_args_string"; > "tmpfile"."i".done ; } &"
+-        retval = system(" { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -- \""target_bin"\" "prog_args_string"; > "tmpfile"."i".done ; } &")
++#        print " { "AFL_MAP_SIZE AFL_PRELOAD AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -- \""target_bin"\" "prog_args_string"; > "tmpfile"."i".done ; } &"
++        retval = system(" { "AFL_MAP_SIZE AFL_PRELOAD AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -- \""target_bin"\" "prog_args_string"; > "tmpfile"."i".done ; } &")
+       } else {
+         stdin_file=tmpfile"."i".stdin"
+-#        print " { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null; > "tmpfile"."i".done ; } &"
+-        retval = system(" { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null; > "tmpfile"."i".done ; } &")
++#        print " { "AFL_MAP_SIZE AFL_PRELOAD AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null; > "tmpfile"."i".done ; } &"
++        retval = system(" { "AFL_MAP_SIZE AFL_PRELOAD AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null; > "tmpfile"."i".done ; } &")
+       }
+     }
+     print "[*] Waiting for parallel tasks to complete ..."
+@@ -562,11 +568,11 @@ BEGIN {
+     if (!stdin_file) {
+       print "    Processing "in_count" files (forkserver mode)..."
+ #      print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string
+-      retval = system(AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string)
++      retval = system(AFL_MAP_SIZE AFL_PRELOAD AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string)
+     } else {
+       print "    Processing "in_count" files (forkserver mode)..."
+ #    print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null"
+-      retval = system(AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
++      retval = system(AFL_MAP_SIZE AFL_PRELOAD AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
+     }
+ 
+     if (retval && (!AFL_CMIN_CRASHES_ONLY && !AFL_CMIN_ALLOW_ANY)) {
+diff --git a/afl-cmin.bash b/afl-cmin.bash
+index 99ae80d9..2909ebfa 100755
+--- a/afl-cmin.bash
++++ b/afl-cmin.bash
+@@ -245,14 +245,16 @@ if [ "$NYX_MODE" = "" ]; then
+ 
+ fi
+ 
+-grep -aq AFL_DUMP_MAP_SIZE "$TARGET_BIN" && {
+-  echo "[!] Trying to obtain the map size of the target ..."
+-  MAPSIZE=`AFL_DUMP_MAP_SIZE=1 "./$TARGET_BIN" 2>/dev/null`
+-  test -n "$MAPSIZE" && {
+-    export AFL_MAP_SIZE=$MAPSIZE
+-    echo "[+] Setting AFL_MAP_SIZE=$MAPSIZE"
+-  }
+-}
++if [ -z "$NYX_MODE" ]; then
++  if [ -z "$AFL_MAP_SIZE" ] && grep -aq AFL_DUMP_MAP_SIZE "$TARGET_BIN"; then
++    echo "[!] Trying to obtain the map size of the target ..."
++    MAPSIZE=`AFL_DUMP_MAP_SIZE=1 "./$TARGET_BIN" 2>/dev/null`
++    test -n "$MAPSIZE" && {
++      export AFL_MAP_SIZE=$MAPSIZE
++      echo "[+] Setting AFL_MAP_SIZE=$MAPSIZE"
++    }
++  fi
++fi
+ 
+ if [ "$AFL_SKIP_BIN_CHECK" = "" -a "$QEMU_MODE" = "" -a "$FRIDA_MODE" = "" -a "$UNICORN_MODE" = "" -a "$NYX_MODE" = "" ]; then
+ 

services/afl/setup-target.sh

@@ -18,7 +18,8 @@ FETCH_ARGS=(-o "$HOME" --afl --fuzzing)
 if [[ -n "$JSRT" ]] && [[ -z "$COVERAGE" ]]
 then
   FETCH_ARGS+=(--debug)
-else
+elif [[ -z "$COVERAGE" ]]
+then
   FETCH_ARGS+=(--asan)
 fi
 

services/afl/setup.sh

@@ -51,6 +51,7 @@ pkgs=(
   libosmesa6
   libpci3
   openssh-client
+  patch
   pipx
   psmisc
   screen
@@ -81,6 +82,9 @@ afl_ver="$(resolve-tc-alias afl-instrumentation)"
 retry-curl "$(resolve-tc "$afl_ver")" | zstdcat | tar -x -C /opt
 # shellcheck disable=SC2016
 echo 'PATH=$PATH:/opt/afl-instrumentation/bin' >> /etc/bash.bashrc
+pushd /opt/afl-instrumentation/bin
+patch -p1 < /home/worker/patches/afl-cmin.patch
+popd >/dev/null
 
 cd ..
 su worker << EOF