db_crashtest: re-enable track_and_verify_wals with disabled fault injection (#14860)

Summary:

See title. This is to harvest the coverage of the feature before we fix the tricky compatibility issue (see #13871)

**Test plan:**
Will only land after intensive rehersal stress test passes

Differential Revision: D108797734

Author: Hui Xiao

tools/db_crashtest.py

@@ -497,8 +497,7 @@ def apply_random_seed_per_iteration():
     "memtable_verify_per_key_checksum_on_seek": lambda: random.choice([0] * 7 + [1]),
     "memtable_batch_lookup_optimization": lambda: random.randint(0, 1),
     "allow_unprepared_value": lambda: random.choice([0, 1]),
-    # TODO(hx235): enable `track_and_verify_wals` after stabalizing the stress test
-    "track_and_verify_wals": lambda: random.choice([0]),
+    "track_and_verify_wals": lambda: random.choice([0, 1]),
     "remote_compaction_worker_threads": lambda: random.choice([0, 8]),
     "allow_resumption_one_in": lambda: random.choice([0, 1, 2, 20]),
     # TODO(jaykorean): Change to lambda: random.choice([0, 1]) after addressing all remote compaction failures
@@ -1571,6 +1570,22 @@ def finalize_and_sanitize(src_params):
         dest_params["clear_column_family_one_in"] = 0
         dest_params["test_multi_ops_txns"] = 0
 
+    if dest_params.get("track_and_verify_wals", 0) == 1:
+        # Disable fault injection while track_and_verify_wals is on. These fault
+        # injections trigger auto recovery, which flushes the memtable and
+        # rotates the WAL; that is currently incompatible with WAL tracking and
+        # verification and surfaces as spurious WAL verification failures.
+        # TODO: re-enable once the auto-recovery vs. track_and_verify_wals
+        # compatibility issue is fixed.
+        dest_params["write_fault_one_in"] = 0
+        dest_params["metadata_write_fault_one_in"] = 0
+        dest_params["read_fault_one_in"] = 0
+        dest_params["metadata_read_fault_one_in"] = 0
+        dest_params["open_metadata_write_fault_one_in"] = 0
+        dest_params["open_metadata_read_fault_one_in"] = 0
+        dest_params["open_write_fault_one_in"] = 0
+        dest_params["open_read_fault_one_in"] = 0
+
     return dest_params