Implement code coverage transformation

.github/workflows/c_apps.sh

@@ -31,7 +31,9 @@ case "$(uname)" in
   sudo rm -f /swapfile
   sudo apt clean
   # shellcheck disable=SC2046
-  docker rmi $(docker image ls -aq)
+  if [ -n "$(docker image ls -aq)" ]; then
+    docker rmi $(docker image ls -aq)
+  fi
   df -h
   ;;
 
@@ -79,6 +81,12 @@ popd
 # Check that dredd works on some projects
 DREDD_EXECUTABLE="${DREDD_ROOT}/third_party/clang+llvm/bin/dredd"
 cp "${DREDD_ROOT}/build/src/dredd/dredd" "${DREDD_EXECUTABLE}"
+if [ "$SEMANTICS_PRESERVING" == "true" ]; then
+  DREDD_SEMANTICS_PRESERVING="--semantics-preserving-coverage-instrumentation"
+else
+  DREDD_SEMANTICS_PRESERVING=""
+fi
+
 
 echo "Curl"
 date
@@ -96,7 +104,7 @@ pushd curl
       FILES+=("${f}")
   done
 
-  "${DREDD_EXECUTABLE}" --mutation-info-file temp.json -p "build/compile_commands.json" "${FILES[@]}"
+  "${DREDD_EXECUTABLE}" $DREDD_SEMANTICS_PRESERVING --mutation-info-file temp.json -p "build/compile_commands.json" "${FILES[@]}"
   pushd build
     ninja
     # TODO: run some tests
@@ -125,7 +133,7 @@ pushd zstd
   do
     FILES+=("${f}")
   done
-  "${DREDD_EXECUTABLE}" --mutation-info-file temp.json -p "temp/compile_commands.json" "${FILES[@]}"
+  "${DREDD_EXECUTABLE}" $DREDD_SEMANTICS_PRESERVING --mutation-info-file temp.json -p "temp/compile_commands.json" "${FILES[@]}"
   # Build mutated zstd
   make clean
   CFLAGS=-O0 make zstd-release

.github/workflows/c_apps.yml

@@ -27,11 +27,13 @@ jobs:
           - ubuntu-22.04
         config:
           - Release
+        semantics-preserving: [true, false]
 
     runs-on: ${{ matrix.os }}
     env:
       OS: ${{ matrix.os }}
       CONFIG: ${{ matrix.config }}
+      SEMANTICS_PRESERVING: ${{ matrix.semantics-preserving }}
     steps:
 
       - name: cancel_previous

.github/workflows/cxx_apps.sh

@@ -75,11 +75,16 @@ cmake --build build --config Debug
 # Check that dredd works on some projects
 DREDD_EXECUTABLE="${DREDD_ROOT}/third_party/clang+llvm/bin/dredd"
 cp "${DREDD_ROOT}/build/src/dredd/dredd" "${DREDD_EXECUTABLE}"
+if [ "$SEMANTICS_PRESERVING" == "true" ]; then
+  DREDD_SEMANTICS_PRESERVING="--semantics-preserving-coverage-instrumentation"
+else
+  DREDD_SEMANTICS_PRESERVING=""
+fi
 
 echo "examples/simple/pi.cc: check that we can build the simple example"
 date
 
-${DREDD_EXECUTABLE} --mutation-info-file temp.json examples/simple/pi.cc
+${DREDD_EXECUTABLE} $DREDD_SEMANTICS_PRESERVING --mutation-info-file temp.json examples/simple/pi.cc
 clang++ examples/simple/pi.cc -o examples/simple/pi
 diff <(./examples/simple/pi) <(echo "3.14159")
 
@@ -91,7 +96,7 @@ pushd examples/math
   cp -r math math-original
   cmake -S . -B build -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
   cmake --build build
-  ${DREDD_EXECUTABLE} -p build/compile_commands.json --mutation-info-file mutation-info.json math/src/*.cc
+  ${DREDD_EXECUTABLE} $DREDD_SEMANTICS_PRESERVING -p build/compile_commands.json --mutation-info-file mutation-info.json math/src/*.cc
   ./build/mathtest/mathtest
   NUM_MUTANTS=`python3 ${DREDD_ROOT}/scripts/query_mutant_info.py mutation-info.json --largest-mutant-id`
   EXPECTED_NUM_MUTANTS=1132
@@ -120,7 +125,7 @@ pushd examples/threaded
     [[ -e "$f" ]] || break
     FILES+=("${DREDD_ROOT}/examples/threaded/${f}")
   done
-  ${DREDD_EXECUTABLE} --mutation-info-file temp.json -p "${DREDD_ROOT}/examples/threaded/build/compile_commands.json" "${FILES[@]}"
+  ${DREDD_EXECUTABLE} $DREDD_SEMANTICS_PRESERVING --mutation-info-file temp.json -p "${DREDD_ROOT}/examples/threaded/build/compile_commands.json" "${FILES[@]}"
   cmake --build build
   # Check that the application runs correctly and that there are no data races.
   TSAN_OPTIONS=halt_on_error=1 ./build/threaded > threaded_output.txt
@@ -150,7 +155,7 @@ pushd SPIRV-Tools
     [[ -e "$f" ]] || break
     FILES+=("${DREDD_ROOT}/SPIRV-Tools/${f}")
   done
-  ${DREDD_EXECUTABLE} --mutation-info-file mutation-info.json -p "${DREDD_ROOT}/SPIRV-Tools/build/compile_commands.json" "${FILES[@]}"
+  ${DREDD_EXECUTABLE} $DREDD_SEMANTICS_PRESERVING --mutation-info-file mutation-info.json -p "${DREDD_ROOT}/SPIRV-Tools/build/compile_commands.json" "${FILES[@]}"
   cmake --build build --target test_val_abcde test_val_capability test_val_fghijklmnop test_val_limits test_val_rstuvw
   ./build/test/val/test_val_abcde
   ./build/test/val/test_val_capability
@@ -180,7 +185,7 @@ pushd llvm-project
     [[ -e "$f" ]] || break
     FILES+=("${DREDD_ROOT}/llvm-project/${f}")
   done
-  ${DREDD_EXECUTABLE} --mutation-info-file mutation-info.json -p "${DREDD_ROOT}/llvm-project/build/compile_commands.json" "${FILES[@]}"
+  ${DREDD_EXECUTABLE} $DREDD_SEMANTICS_PRESERVING --mutation-info-file mutation-info.json -p "${DREDD_ROOT}/llvm-project/build/compile_commands.json" "${FILES[@]}"
   cmake --build build --target LLVMInstCombine
   NUM_MUTANTS=`python3 ${DREDD_ROOT}/scripts/query_mutant_info.py mutation-info.json --largest-mutant-id`
   EXPECTED_NUM_MUTANTS=97675

.github/workflows/cxx_apps.yml

@@ -27,11 +27,13 @@ jobs:
           - ubuntu-22.04
         config:
           - Release
+        semantics-preserving: [true, false]
 
     runs-on: ${{ matrix.os }}
     env:
       OS: ${{ matrix.os }}
       CONFIG: ${{ matrix.config }}
+      SEMANTICS_PRESERVING: ${{ matrix.semantics-preserving }}
     steps:
 
       - name: cancel_previous

README.md

@@ -72,11 +72,7 @@ We first show how to apply Dredd to a simple stand-alone program. We will then s
 
 ```
 # This will modify pi.cc in-place.
-# The --mutation-info-file argument is used to specify a JSON file to
-# which Dredd will output machine-readable information about the
-# mutations it applied. For the purposes of this example, this file
-# can be ignored.
-${DREDD_EXECUTABLE} examples/simple/pi.cc --mutation-info-file temp.json
+${DREDD_EXECUTABLE} examples/simple/pi.cc
 # Now compile the mutated version of the example
 ${DREDD_CLANG_BIN_DIR}/clang++ examples/simple/pi.cc -o examples/simple/pi
 ```

scripts/check_one_single_file_test.sh

@@ -23,80 +23,84 @@ DREDD_INSTALLED_EXECUTABLE="${DREDD_REPO_ROOT}/third_party/clang+llvm/bin/dredd"
 # Move to the temporary directory
 pushd "${DREDD_REPO_ROOT}/temp"
 
-  if [ -z "${DREDD_SKIP_COPY_EXECUTABLE+x}" ]
+if [ -z "${DREDD_SKIP_COPY_EXECUTABLE+x}" ]
+then
+  # Ensure that Dredd is in its installed location. This depends on a
+  # debug build being available
+  cp build-Debug/src/dredd/dredd "${DREDD_INSTALLED_EXECUTABLE}"
+fi
+
+f="${DREDD_REPO_ROOT}/${1}"
+
+for opt in "semantics_preserving" "opt" "noopt"
+do
+  # Determine whether optimisations should be disabled or not, and whether the
+  # name of the expectation file should reflect this.
+  DREDD_EXTRA_ARGS=""
+  DREDD_EXPECTED_FILE=""
+  if [ ${opt} == "noopt" ]
   then
-    # Ensure that Dredd is in its installed location. This depends on a
-    # debug build being available
-    cp build-Debug/src/dredd/dredd "${DREDD_INSTALLED_EXECUTABLE}"
+    DREDD_EXTRA_ARGS="--no-mutation-opts"
+    DREDD_EXPECTED_FILE="$f.noopt.expected"
+  elif [ ${opt} == "opt" ]
+  then
+    DREDD_EXPECTED_FILE="$f.expected"
+  else
+    DREDD_EXTRA_ARGS="--semantics-preserving-coverage-instrumentation"
+    DREDD_EXPECTED_FILE="$f.semantics_preserving.expected"
   fi
 
-  f="${DREDD_REPO_ROOT}/${1}"
-
-  for opt in "opt" "noopt"
-  do
-    # Determine whether optimisations should be disabled or not, and whether the
-    # name of the expectation file should reflect this.
-    DREDD_EXTRA_ARGS=""
-    DREDD_EXPECTED_FILE=""
-    if [ ${opt} == "noopt" ]
-    then
-      DREDD_EXTRA_ARGS="--no-mutation-opts"
-      DREDD_EXPECTED_FILE="$f.noopt.expected"
-    else
-      DREDD_EXPECTED_FILE="$f.expected"
-    fi
+  # Copy the single-file test case to the temporary directory so
+  # that it can be mutated without affecting the original
+  cp "$f" .
+  copy_of_f=$(basename "$f")
 
-    # Copy the single-file test case to the temporary directory so
-    # that it can be mutated without affecting the original
-    cp "$f" .
-    copy_of_f=$(basename "$f")
+  # Mutate the test case using Dredd
+  ${DREDD_INSTALLED_EXECUTABLE} ${DREDD_EXTRA_ARGS} --mutation-info-file temp.json "${copy_of_f}" --
 
-    # Mutate the test case using Dredd
-    ${DREDD_INSTALLED_EXECUTABLE} ${DREDD_EXTRA_ARGS} --mutation-info-file temp.json "${copy_of_f}" --
+  # Check that the JSON generated by Dredd is valid
+  check_json.py temp.json
 
-    # Check that the JSON generated by Dredd is valid
-    check_json.py temp.json
+  if [ -z "${DREDD_REGENERATE_TEST_CASE+x}" ]
+  then
+    # Check that the mutated test case is as expected
+    diff --strip-trailing-cr "${copy_of_f}" "${DREDD_EXPECTED_FILE}"
+  fi
 
-    if [ -z "${DREDD_REGENERATE_TEST_CASE+x}" ]
+  # Check that the mutated file compiles
+  if [[ $f == *.cc ]]
+  then
+    # Extra C++ arguments can be passed;
+    # this is needed on OSX to set an appropriate C++ standard.
+    if [ -z "${DREDD_EXTRA_CXX_ARGS+x}" ]
     then
-      # Check that the mutated test case is as expected
-      diff --strip-trailing-cr "${copy_of_f}" "${DREDD_EXPECTED_FILE}"
+      ${CXX} -c "${copy_of_f}"
+    else
+      ${CXX} "${DREDD_EXTRA_CXX_ARGS}" -c "${copy_of_f}"
     fi
-
-    # Check that the mutated file compiles
-    if [[ $f == *.cc ]]
+  else
+    # Extra C arguments can be passed;
+    # this is needed on Windows to set an appropriate C standard.
+    if [ -z "${DREDD_EXTRA_C_ARGS+x}" ]
     then
-      # Extra C++ arguments can be passed;
-      # this is needed on OSX to set an appropriate C++ standard.
-      if [ -z "${DREDD_EXTRA_CXX_ARGS+x}" ]
-      then
-        ${CXX} -c "${copy_of_f}"
-      else
-        ${CXX} "${DREDD_EXTRA_CXX_ARGS}" -c "${copy_of_f}"
-      fi
+      ${CC} -c "${copy_of_f}"
     else
-      # Extra C arguments can be passed;
-      # this is needed on Windows to set an appropriate C standard.
-      if [ -z "${DREDD_EXTRA_C_ARGS+x}" ]
-      then
-        ${CC} -c "${copy_of_f}"
-      else
-        ${CC} "${DREDD_EXTRA_C_ARGS}" -c "${copy_of_f}"
-      fi
+      ${CC} "${DREDD_EXTRA_C_ARGS}" -c "${copy_of_f}"
     fi
+  fi
 
-    if [ "${DREDD_REGENERATE_TEST_CASE+x}" ]
-    then
-      # Copy the mutated file so that it becomes the new test expectation
-      cp "${copy_of_f}" "${DREDD_EXPECTED_FILE}"
-    fi
+  if [ "${DREDD_REGENERATE_TEST_CASE+x}" ]
+  then
+    # Copy the mutated file so that it becomes the new test expectation
+    cp "${copy_of_f}" "${DREDD_EXPECTED_FILE}"
+  fi
 
-    # Clean up
-    rm "${copy_of_f}"
-    # Account for the fact that the script may be running under various OSes
-    rm -f "${copy_of_f%.*}".o
-    rm -f "${copy_of_f%.*}".obj
+  # Clean up
+  rm "${copy_of_f}"
+  # Account for the fact that the script may be running under various OSes
+  rm -f "${copy_of_f%.*}".o
+  rm -f "${copy_of_f%.*}".obj
 
-  done
+done
 
 popd

src/.clang-tidy

@@ -38,4 +38,6 @@ WarningsAsErrors: '*'
 HeaderFilterRegex: '.*dredd.*'
 AnalyzeTemporaryDtors: false
 FormatStyle:     file
+CheckOptions:
+  readability-function-cognitive-complexity.Threshold: 70
 ...

src/dredd/src/main.cc

@@ -14,6 +14,7 @@
 
 #include <fstream>
 #include <memory>
+#include <optional>
 #include <string>
 
 #include "clang/Tooling/CommonOptionsParser.h"
@@ -55,13 +56,20 @@ static llvm::cl::opt<bool> only_track_mutant_coverage(
                    "an input, rather than actually applying any mutants."),
     llvm::cl::cat(mutate_category));
 // NOLINTNEXTLINE
+static llvm::cl::opt<bool> semantics_preserving_coverage_instrumentation(
+    "semantics-preserving-coverage-instrumentation",
+    llvm::cl::desc(
+        "Apply a semantics preserving transformation to the source code "
+        "which adds extra code coverage points."),
+    llvm::cl::cat(mutate_category));
+// NOLINTNEXTLINE
 static llvm::cl::opt<bool> dump_asts(
     "dump-asts",
     llvm::cl::desc("Dump each AST that is processed; useful for debugging"),
     llvm::cl::cat(mutate_category));
 // NOLINTNEXTLINE
 static llvm::cl::opt<std::string> mutation_info_file(
-    "mutation-info-file", llvm::cl::Required,
+    "mutation-info-file",
     llvm::cl::desc(
         ".json file into which mutation information should be written"),
     llvm::cl::cat(mutate_category));
@@ -94,24 +102,30 @@ int main(int argc, const char** argv) {
 
   // Keeps track of the mutations that are applied to each source file,
   // including their hierarchical structure.
-  dredd::protobufs::MutationInfo mutation_info;
+  std::optional<dredd::protobufs::MutationInfo> mutation_info;
+
+  if (mutation_info_file.empty()) {
+    mutation_info = std::nullopt;
+  } else {
+    mutation_info = dredd::protobufs::MutationInfo();
+  }
 
   const std::unique_ptr<clang::tooling::FrontendActionFactory> factory =
-      dredd::NewMutateFrontendActionFactory(!no_mutation_opts, dump_asts,
-                                            only_track_mutant_coverage,
-                                            mutation_id, mutation_info);
+      dredd::NewMutateFrontendActionFactory(
+          !no_mutation_opts, semantics_preserving_coverage_instrumentation,
+          dump_asts, only_track_mutant_coverage, mutation_id, mutation_info);
 
   const int return_code = Tool.run(factory.get());
 
-  if (return_code == 0) {
+  if (mutation_info.has_value() && return_code == 0) {
     // Application of mutations was successful, so write out the mutation info
     // in JSON format.
     std::string json_string;
     auto json_options = google::protobuf::util::JsonOptions();
     json_options.add_whitespace = true;
     json_options.always_print_primitive_fields = true;
     auto json_generation_status = google::protobuf::util::MessageToJsonString(
-        mutation_info, &json_string, json_options);
+        mutation_info.value(), &json_string, json_options);
     if (json_generation_status.ok()) {
       std::ofstream transformations_json_file(mutation_info_file);
       transformations_json_file << json_string;

src/libdredd/include/libdredd/mutation.h

@@ -50,10 +50,11 @@ class Mutation {
   // avoiding redundant repeat declarations.
   virtual protobufs::MutationGroup Apply(
       clang::ASTContext& ast_context, const clang::Preprocessor& preprocessor,
-      bool optimise_mutations, bool only_track_mutant_coverage,
-      int first_mutation_id_in_file, int& mutation_id,
-      clang::Rewriter& rewriter,
-      std::unordered_set<std::string>& dredd_declarations) const = 0;
+      bool optimise_mutations, bool semantics_preserving_mutation,
+      bool only_track_mutant_coverage, int first_mutation_id_in_file,
+      int& mutation_id, clang::Rewriter& rewriter,
+      std::unordered_set<std::string>& dredd_declarations,
+      std::unordered_set<std::string>& dredd_macros) const = 0;
 };
 
 }  // namespace dredd

src/libdredd/include/libdredd/mutation_remove_stmt.h

@@ -37,10 +37,11 @@ class MutationRemoveStmt : public Mutation {
 
   protobufs::MutationGroup Apply(
       clang::ASTContext& ast_context, const clang::Preprocessor& preprocessor,
-      bool optimise_mutations, bool only_track_mutant_coverage,
-      int first_mutation_id_in_file, int& mutation_id,
-      clang::Rewriter& rewriter,
-      std::unordered_set<std::string>& dredd_declarations) const override;
+      bool optimise_mutations, bool semantics_preserving_mutation,
+      bool only_track_mutant_coverage, int first_mutation_id_in_file,
+      int& mutation_id, clang::Rewriter& rewriter,
+      std::unordered_set<std::string>& dredd_declarations,
+      std::unordered_set<std::string>& dredd_macros) const override;
 
  private:
   // Helper method to determine whether the token immediately following the