Preliminary support for libfuzzer

Author: smoelius

.github/workflows/ci.yml

@@ -101,13 +101,52 @@ jobs:
           cargo clean && cargo +nightly udeps --features=test-fuzz/auto_concretize --all-targets
 
   test:
-    runs-on: ubuntu-latest
-
     strategy:
       fail-fast: false
       matrix:
-        serde_format: [bincode, cbor, cbor4ii]
-        toolchain: [stable, nightly]
+        include:
+          - fuzzer: aflplusplus
+            environment: ubuntu-latest
+            serde_format: bincode
+            toolchain: stable
+          - fuzzer: aflplusplus
+            environment: macos-latest
+            serde_format: bincode
+            toolchain: nightly
+          - fuzzer: aflplusplus-persistent
+            environment: ubuntu-latest
+            serde_format: cbor
+            toolchain: stable
+          - fuzzer: libfuzzer
+            environment: ubuntu-latest
+            serde_format: cbor
+            toolchain: nightly
+          - fuzzer: libfuzzer
+            environment: macos-latest
+            serde_format: cbor4ii
+            toolchain: stable
+          - fuzzer: aflplusplus
+            environment: ubuntu-latest
+            serde_format: cbor4ii
+            toolchain: nightly
+          - fuzzer: aflplusplus
+            environment: macos-latest
+            serde_format: bincode
+            toolchain: stable
+          - fuzzer: aflplusplus-persistent
+            environment: ubuntu-latest
+            serde_format: bincode
+            toolchain: nightly
+          - fuzzer: libfuzzer
+            environment: ubuntu-latest
+            serde_format: cbor
+            toolchain: stable
+          - fuzzer: libfuzzer
+            environment: macos-latest
+            serde_format: cbor
+            toolchain: nightly
+
+    runs-on: ${{ matrix.environment }}
 
     steps:
       - uses: actions/checkout@v3
@@ -118,11 +157,17 @@ jobs:
         run: rustup default ${{ matrix.toolchain }}
 
       - name: Install llvm
+        if: ${{ matrix.environment == 'ubuntu-latest' }}
         run: sudo apt-get install llvm
 
       # smoelius: The Substrate tests require `protoc`.
       - name: Install protoc
-        run: sudo apt-get install protobuf-compiler
+        run: |
+          if [[ ${{ matrix.environment }} = 'ubuntu-latest' ]]; then
+            sudo apt-get install protobuf-compiler
+          else
+            brew install protobuf
+          fi
 
       - name: Install afl
         run: cargo install afl
@@ -132,6 +177,11 @@ jobs:
           test "$(ls ~/.local/share/afl.rs | wc -l)" -eq 1
           test "$(ls ~/.local/share/afl.rs/rustc-* | wc -l)" -eq 1
           sudo "$HOME"/.local/share/afl.rs/rustc-*/afl.rs-*/afl/bin/afl-system-config
+          if [[ ${{ matrix.environment }} = 'macos-latest' ]]; then
+            SL=/System/Library; PL=com.apple.ReportCrash
+            launchctl unload -w ${SL}/LaunchAgents/${PL}.plist
+            sudo launchctl unload -w ${SL}/LaunchDaemons/${PL}.Root.plist
+          fi
 
       # smoelius: The wasm32 target is needed for some Substrate tests, regardless of the toolchain
       # used to build test-fuzz.
@@ -144,7 +194,7 @@ jobs:
         run: |
           AUTO_CONCRETIZE=
           SHUFFLE=
-          if [[ ${{ matrix.toolchain }} = nightly ]]; then
+          if [[ ${{ matrix.toolchain }} = 'nightly' ]]; then
             AUTO_CONCRETIZE='--features=test-fuzz/auto_concretize'
             SHUFFLE='-Z unstable-options --shuffle --test-threads=1'
           fi
@@ -158,6 +208,7 @@ jobs:
         env:
           RUST_BACKTRACE: 1
           RUST_LOG: warn
+          TEST_FUZZ_FUZZER: ${{ matrix.fuzzer }}
 
   test-uninstalled-cargo-afl:
     runs-on: ubuntu-latest
@@ -172,13 +223,14 @@ jobs:
         run: |
           OUTPUT="$(cargo run -p cargo-test-fuzz -- test-fuzz -p test-fuzz-examples --no-run 2>&1 1>/dev/null || true)"
           echo "$OUTPUT"
-          echo "$OUTPUT" | grep '^Error: Could not determine `cargo-afl` version. Is it installed? Try `cargo install afl`.$'
+          echo "$OUTPUT" | grep 'Could not determine `cargo-afl` version. Is it installed? Try `cargo install afl`.'
 
   test-incompatible-cargo-afl:
     runs-on: ubuntu-latest
 
     env:
       RUSTUP_TOOLCHAIN: nightly
+      TEST_FUZZ_FUZZER: aflplusplus-persistent
 
     steps:
       - uses: actions/checkout@v3
@@ -200,6 +252,7 @@ jobs:
 
     env:
       RUSTUP_TOOLCHAIN: nightly
+      TEST_FUZZ_FUZZER: aflplusplus-persistent
 
     steps:
       - uses: actions/checkout@v3

README.md

@@ -248,50 +248,52 @@ The `cargo test-fuzz` command is used to interact with fuzz targets, and to mani
 #### Options
 
 ```
-      --backtrace             Display backtraces
-      --consolidate           Move one target's crashes, hangs, and work queue to its corpus; to
-                              consolidate all targets, use --consolidate-all
-      --display <OBJECT>      Display concretizations, corpus, crashes, `impl` concretizations,
-                              hangs, or work queue. By default, corpus uses an uninstrumented fuzz
-                              target; the others use an instrumented fuzz target. To display the
-                              corpus with instrumentation, use --display corpus-instrumented.
-                              [possible values: concretizations, corpus, corpus-instrumented,
-                              crashes, hangs, impl-concretizations, queue]
-      --exact                 Target name is an exact name rather than a substring
-      --exit-code             Exit with 0 if the time limit was reached, 1 for other programmatic
-                              aborts, and 2 if an error occurred; implies --no-ui, does not imply
-                              --run-until-crash or -- -V <SECONDS>
-      --features <FEATURES>   Space or comma separated list of features to activate
-      --list                  List fuzz targets
-      --manifest-path <PATH>  Path to Cargo.toml
-      --no-default-features   Do not activate the `default` feature
-      --no-instrumentation    Compile without instrumentation (for testing build process)
-      --no-run                Compile, but don't fuzz
-      --no-ui                 Disable user interface
-  -p, --package <PACKAGE>     Package containing fuzz target
-      --persistent            Enable persistent mode fuzzing
-      --pretty-print          Pretty-print debug output when displaying/replaying
-      --replay <OBJECT>       Replay corpus, crashes, hangs, or work queue. By default, corpus uses
-                              an uninstrumented fuzz target; the others use an instrumented fuzz
-                              target. To replay the corpus with instrumentation, use --replay
-                              corpus-instrumented. [possible values: concretizations, corpus,
-                              corpus-instrumented, crashes, hangs, impl-concretizations, queue]
-      --reset                 Clear fuzzing data for one target, but leave corpus intact; to reset
-                              all targets, use --reset-all
-      --resume                Resume target's last fuzzing session
-      --run-until-crash       Stop fuzzing once a crash is found
-      --test <NAME>           Integration test containing fuzz target
-      --timeout <TIMEOUT>     Number of seconds to consider a hang when fuzzing or replaying
-                              (equivalent to -- -t <TIMEOUT * 1000> when fuzzing)
-      --verbose               Show build output when displaying/replaying
-  -h, --help                  Print help
-  -V, --version               Print version
-
-To fuzz at most <SECONDS> of time, use:
-
-    cargo test-fuzz ... -- -V <SECONDS>
-
-Try `cargo afl fuzz --help` to see additional fuzzer options.
+      --backtrace                 Display backtraces
+      --consolidate               Move one target's crashes, hangs, and work queue to its corpus; to
+                                  consolidate all targets, use --consolidate-all
+      --display <OBJECT>          Display concretizations, corpus, crashes, `impl` concretizations,
+                                  hangs, or work queue. By default, corpus uses an uninstrumented
+                                  fuzz target; the others use an instrumented fuzz target. To
+                                  display the corpus with instrumentation, use --display
+                                  corpus-instrumented. [possible values: concretizations, corpus,
+                                  corpus-instrumented, crashes, hangs, impl-concretizations, queue]
+      --exact                     Target name is an exact name rather than a substring
+      --exit-code                 Exit with 0 if the time limit was reached, 1 for other
+                                  programmatic aborts, and 2 if an error occurred; implies --no-ui,
+                                  does not imply --run-until-crash or --max-total-time <SECONDS>
+      --features <FEATURES>       Space or comma separated list of features to activate
+      --fuzzer <FUZZER>           Fuzz using <FUZZER> [possible values: aflplusplus,
+                                  aflplusplus-persistent, libfuzzer]
+      --list                      List fuzz targets
+      --manifest-path <PATH>      Path to Cargo.toml
+      --max-total-time <SECONDS>  Fuzz at most <SECONDS> of time (equivalent to -- -V <SECONDS> for
+                                  aflplusplus, and -- --max_total_time <SECONDS> for libfuzzer)
+      --no-default-features       Do not activate the `default` feature
+      --no-instrumentation        Compile without instrumentation (for testing build process)
+      --no-run                    Compile, but don't fuzz
+      --no-ui                     Disable user interface
+  -p, --package <PACKAGE>         Package containing fuzz target
+      --pretty-print              Pretty-print debug output when displaying/replaying
+      --replay <OBJECT>           Replay corpus, crashes, hangs, or work queue. By default, corpus
+                                  uses an uninstrumented fuzz target; the others use an instrumented
+                                  fuzz target. To replay the corpus with instrumentation, use
+                                  --replay corpus-instrumented. [possible values: concretizations,
+                                  corpus, corpus-instrumented, crashes, hangs, impl-concretizations,
+                                  queue]
+      --reset                     Clear fuzzing data for one target, but leave corpus intact; to
+                                  reset all targets, use --reset-all
+      --resume                    Resume target's last fuzzing session
+      --run-until-crash           Stop fuzzing once a crash is found
+      --test <NAME>               Integration test containing fuzz target
+      --timeout <TIMEOUT>         Number of seconds to consider a hang when fuzzing or replaying
+                                  (equivalent to -- -t <TIMEOUT * 1000> when fuzzing with
+                                  aflplusplus, and -- -timeout <TIMEOUT> when fuzzing with
+                                  libfuzzer)
+      --verbose                   Show build output when displaying/replaying
+  -h, --help                      Print help
+  -V, --version                   Print version
+
+Try `cargo afl fuzz --help` to see additional AFLplusplus options.
 ```
 
 ### Convenience functions and macros

cargo-test-fuzz/Cargo.toml

@@ -19,16 +19,20 @@ doctest = false
 [dependencies]
 anyhow = { version = "1.0", features = ["backtrace"] }
 bitflags = "2.3"
+cargo-fuzz = { git = "https://github.com/trail-of-forks/cargo-fuzz", features = ["no-manifest-check"] }
 cargo_metadata = "0.15"
 clap = { version = "4.3", features = ["cargo", "derive", "wrap_help"] }
 env_logger = "0.10"
+fs_extra = "1.3"
 heck = "0.4"
 lazy_static = "1.4"
 log = "0.4"
+once_cell = "1.16"
 paste = "1.0"
 remain = "0.2"
 semver = "1.0"
 serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 strum_macros = "0.24"
 subprocess = "0.2"