diff --git a/test/jepsen/README.md b/test/jepsen/README.md index 199c9e7..5f43195 100644 --- a/test/jepsen/README.md +++ b/test/jepsen/README.md @@ -109,9 +109,18 @@ count, concurrency, keys, owners, and recovery time. Run mutation qualification plus live positive- and negative-checker tests: ```bash -test/jepsen/qualify.sh +elixir test/jepsen/qualify.exs ``` +`qualify.sh` remains a thin compatibility launcher for the same Elixir script. +The script owns artifact creation, command sequencing, and result validation. +Each live run still uses GNU `timeout` with a three-minute deadline and a +30-second TERM/KILL grace period, and streams its output to a separate log. +Qualification requires both the expected exit status and a fresh checker record +confirming that the intended corruption was actually detected; a failed command +alone never counts. Executable ExUnit regressions exercise this runner with +stubbed external commands as part of normal `mix test`, without Docker. + This runs every mutation defined by `test/mutation/run.exs`, then verifies that a healthy live history is accepted and deliberately injected owner-death, internal-index, stranded snapshot-cursor, registry claim/projection, and diff --git a/test/jepsen/invariant_qualification_test.exs b/test/jepsen/invariant_qualification_test.exs new file mode 100644 index 0000000..64eb1f8 --- /dev/null +++ b/test/jepsen/invariant_qualification_test.exs @@ -0,0 +1,90 @@ +# MIX_ENV=test mix run --no-start test/jepsen/invariant_qualification_test.exs +# Compile only the actual oracle modules, not the node entrypoint. Redirect +# the injection marker into the workspace so this probe needs no Docker or +# machine-global files. +ExUnit.start() + +defmodule Group.Jepsen.InvariantQualificationTest do + use ExUnit.Case, async: false + + alias Group.Jepsen.Invariant + alias Group.Replica.Data + + @compile {:no_warn_undefined, [Group.Jepsen.Invariant, Group.Jepsen.EDN]} + + setup_all do + work = + Path.expand( + "../../tmp/invariant-qualification-#{System.unique_integer([:positive])}", + __DIR__ + ) + + File.mkdir_p!(work) + marker = Path.join(work, "cursor-marker") + on_exit(fn -> File.rm_rf!(work) end) + {:ok, _} = Application.ensure_all_started(:group) + + {:__block__, _, expressions} = + __DIR__ + |> Path.join("node.exs") + |> File.read!() + |> Code.string_to_quoted!() + + modules = + Enum.filter(expressions, fn + {:defmodule, _, [{:__aliases__, _, [:Group, :Jepsen, name]}, _]} -> + name in [:InvariantViolation, :Invariant, :ConflictResolver, :EDN] + + _ -> + false + end) + + ast = + Macro.prewalk({:__block__, [], modules}, fn + "/tmp/group-jepsen-cursor-marker-corruption" -> marker + node -> node + end) + + Code.compile_quoted(ast) + {:ok, marker: marker} + end + + setup %{marker: marker} do + File.rm(marker) + start_supervised!({Group, name: :jepsen_group, shards: 1, log: false}) + :ok + end + + test "arming with no remote cursor reports no injection or invariant evidence", %{ + marker: marker + } do + File.write!(marker, "enabled\n") + snapshot = Invariant.snapshot([]) + refute snapshot.healthy + assert snapshot.snapshot_staging_count == -1 + assert snapshot.injected_corruptions == [] + assert snapshot.failed_invariants == [] + end + + test "a real marker insertion reports its exact invariant", %{marker: marker} do + File.write!(marker, "enabled\n") + :ets.insert(Data.replica_cursor_table(:jepsen_group, 0), {:probe_stream, 1}) + snapshot = Invariant.snapshot([]) + refute snapshot.healthy + assert snapshot.injected_corruptions == [:"cursor-marker"] + assert snapshot.failed_invariants == [:cursor_snapshot_marker] + assert Group.Jepsen.EDN.encode(snapshot) =~ ":cursor-snapshot-marker" + end + + test "an unrelated index failure cannot masquerade as a cursor marker" do + :ets.insert( + Data.reg_by_pid_table(:jepsen_group, 0), + {{self(), nil, "qualification-probe"}, %{}, 0, node()} + ) + + snapshot = Invariant.snapshot([]) + refute snapshot.healthy + assert snapshot.injected_corruptions == [] + assert snapshot.failed_invariants == [:registry_dual_indexes] + end +end diff --git a/test/jepsen/node.exs b/test/jepsen/node.exs index 65bd92a..4d8253d 100644 --- a/test/jepsen/node.exs +++ b/test/jepsen/node.exs @@ -860,6 +860,10 @@ defmodule Group.Jepsen.Cluster do defp result({:error, reason}), do: %{status: :fail, error: inspect(reason)} end +defmodule Group.Jepsen.InvariantViolation do + defexception [:message, :invariant] +end + defmodule Group.Jepsen.Invariant do @moduledoc false @@ -868,9 +872,9 @@ defmodule Group.Jepsen.Invariant do def snapshot(retired_nodes) do config = Group.get_config(:jepsen_group) shards = 0..(config.num_shards - 1) - maybe_inject_cursor_marker_corruption(shards) + injected_corruptions = maybe_inject_cursor_marker_corruption(shards) - errors = + failures = check("dual indexes", &assert_dual_indexes/0) ++ check("registry claims", &assert_registry_claims/0) ++ check("oplog", &assert_oplogs/0) ++ @@ -889,8 +893,10 @@ defmodule Group.Jepsen.Invariant do end) %{ - healthy: errors == [] and staging_count == 0, - errors: errors, + healthy: failures == [] and staging_count == 0, + errors: Enum.map(failures, & &1.message), + failed_invariants: Enum.flat_map(failures, &List.wrap(&1.invariant)), + injected_corruptions: injected_corruptions, snapshot_staging_count: staging_count, oplog_entries: oplog_entries, oplog_max_entries_per_shard: config.replicated_oplog_max_entries, @@ -905,6 +911,8 @@ defmodule Group.Jepsen.Invariant do %{ healthy: false, errors: ["invariant snapshot failed: #{Exception.message(exception)}"], + failed_invariants: [], + injected_corruptions: [], snapshot_staging_count: -1 } end @@ -928,9 +936,13 @@ defmodule Group.Jepsen.Invariant do {stream, {:snapshot_installing, 1}} ) + [:"cursor-marker"] + nil -> raise "no remote replica cursor available for corruption" end + else + [] end end @@ -938,9 +950,13 @@ defmodule Group.Jepsen.Invariant do fun.() [] rescue - exception -> ["#{label}: #{Exception.message(exception)}"] + exception in Group.Jepsen.InvariantViolation -> + [%{invariant: exception.invariant, message: "#{label}: #{Exception.message(exception)}"}] + + exception -> + [%{invariant: nil, message: "#{label}: #{Exception.message(exception)}"}] catch - kind, reason -> ["#{label}: #{inspect({kind, reason})}"] + kind, reason -> [%{invariant: nil, message: "#{label}: #{inspect({kind, reason})}"}] end defp assert_dual_indexes do @@ -973,7 +989,13 @@ defmodule Group.Jepsen.Invariant do {cluster, key, pid, meta, time, origin} end) - assert_equal!(reg_key, reg_pid, "registry dual indexes shard #{shard}") + assert_equal!( + reg_key, + reg_pid, + "registry dual indexes shard #{shard}", + :registry_dual_indexes + ) + assert_equal!(pg_key, pg_pid, "PG dual indexes shard #{shard}") expected_counts = @@ -1074,7 +1096,7 @@ defmodule Group.Jepsen.Invariant do {cluster, key, pid, meta, time, origin} end) - assert_equal!(expected, visible, "registry projection shard #{shard}") + assert_equal!(expected, visible, "registry projection shard #{shard}", :registry_projection) end) end @@ -1140,6 +1162,12 @@ defmodule Group.Jepsen.Invariant do Data.replica_cursor_table(:jepsen_group, shard) |> :ets.tab2list() |> Enum.each(fn {stream, seq} -> + if match?({:snapshot_installing, _}, seq) do + raise Group.Jepsen.InvariantViolation, + invariant: :cursor_snapshot_marker, + message: "cursor contains uncommitted snapshot marker #{inspect({stream, seq})}" + end + origin = WireProtocol.stream_origin(stream) cluster = WireProtocol.stream_cluster(stream) @@ -1202,10 +1230,13 @@ defmodule Group.Jepsen.Invariant do Enum.each(0..(num_shards - 1), fun) end - defp assert_equal!(left, right, label) do + defp assert_equal!(left, right, label, invariant \\ nil) do if left != right do - raise "#{label}: left-only=#{inspect(MapSet.difference(left, right))} " <> - "right-only=#{inspect(MapSet.difference(right, left))}" + raise Group.Jepsen.InvariantViolation, + invariant: invariant, + message: + "#{label}: left-only=#{inspect(MapSet.difference(left, right))} " <> + "right-only=#{inspect(MapSet.difference(right, left))}" end end @@ -1439,7 +1470,7 @@ defmodule Group.Jepsen.Wire do defp corrupt("internal-index") do table = Group.Replica.Data.reg_by_pid_table(:jepsen_group, 0) :ets.insert(table, {{self(), nil, "jepsen/registry/corrupt"}, %{}, 0, node()}) - %{status: :ok} + %{status: :ok, injected: :"internal-index"} end defp corrupt("cursor-marker") do @@ -1483,7 +1514,7 @@ defmodule Group.Jepsen.Wire do end) if corrupted do - %{status: :ok} + %{status: :ok, injected: :"registry-projection"} else %{status: :fail, error: "no visible registry claim available for corruption"} end diff --git a/test/jepsen/qualify.exs b/test/jepsen/qualify.exs new file mode 100644 index 0000000..14da196 --- /dev/null +++ b/test/jepsen/qualify.exs @@ -0,0 +1,99 @@ +defmodule Group.Jepsen.Qualification do + @moduledoc false + + # An intentionally corrupted history must be invalid, but still demonstrate + # that its intended corruption was injected and detected by the checker. + @checks [ + {"none", true}, + {"unexpected-death", false}, + {"internal-index", false}, + {"cursor-marker", false}, + {"registry-projection", false}, + {"terminal-unavailable", false} + ] + + def run do + repo = Path.expand("../..", __DIR__) + cache = Path.join(__DIR__, ".cache") + File.mkdir_p!(cache) + suffix = Base.url_encode64(:crypto.strong_rand_bytes(12), padding: false) + artifacts = Path.join(cache, "qualification.#{suffix}") + File.mkdir!(artifacts) + File.chmod!(artifacts, 0o700) + IO.puts("qualification artifacts: #{artifacts}") + + {_output, status} = + System.cmd("mix", ["run", "test/mutation/run.exs"], + cd: repo, + into: IO.stream(), + stderr_to_stdout: true + ) + + if status != 0, do: System.halt(status) + + Enum.each(@checks, fn {corruption, expected_valid?} -> + qualify!(repo, artifacts, corruption, expected_valid?) + end) + + IO.puts("mutation and live checker qualification passed") + end + + defp qualify!(repo, artifacts, corruption, expected_valid?) do + log = Path.join(artifacts, "#{corruption}.log") + result = Path.join(artifacts, "#{corruption}.result") + + # Keep the existing process-tree deadline and TERM/KILL grace period. + # Streaming output to disk avoids retaining a live history on this VM's heap. + {_output, status} = + System.cmd( + "timeout", + [ + "--signal=TERM", + "--kill-after=30", + "180", + Path.join(__DIR__, "run.sh"), + "test", + "--no-ssh", + "--nodes", + "n1,n2,n3", + "--concurrency", + "2n", + "--time-limit", + "6", + "--fault-interval", + "1", + "--recovery-time", + "5", + "--transport", + "distribution", + "--scenario", + "mixed", + "--corruption", + corruption + ], + cd: repo, + env: [ + {"GROUP_JEPSEN_SKIP_CHECKER", "1"}, + {"GROUP_JEPSEN_QUALIFICATION_RESULT", result} + ], + into: File.stream!(log, [:write, :binary]), + stderr_to_stdout: true + ) + + expected_status = if expected_valid?, do: 0, else: 1 + + unless status == expected_status do + raise "#{corruption}: expected exit #{expected_status}, got #{status}; see #{log}" + end + + expected_record = "group-qualification-v1\t#{corruption}\t#{expected_valid?}\ttrue\n" + + unless File.read!(result) == expected_record do + raise "#{corruption}: missing or mismatched checker evidence; see #{result} and #{log}" + end + + IO.puts("qualified #{corruption} (#{log})") + end +end + +Group.Jepsen.Qualification.run() diff --git a/test/jepsen/qualify.sh b/test/jepsen/qualify.sh index bbd3c3d..ebc3893 100755 --- a/test/jepsen/qualify.sh +++ b/test/jepsen/qualify.sh @@ -2,54 +2,4 @@ set -euo pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -repo_dir="$(cd "${script_dir}/../.." && pwd)" -mkdir -p "${script_dir}/.cache" -artifact_dir="$(mktemp -d "${script_dir}/.cache/qualification.XXXXXX")" - -cd "${repo_dir}" - -mix run test/mutation/run.exs - -export GROUP_JEPSEN_SKIP_CHECKER=1 - -run_jepsen() { - local expectation="$1" - local corruption="$2" - local log="${artifact_dir}/${expectation}-${corruption}.log" - local status=0 - - set +e - timeout --signal=TERM --kill-after=30 180 "${script_dir}/run.sh" test \ - --no-ssh \ - --nodes n1,n2,n3 \ - --concurrency 2n \ - --time-limit 6 \ - --fault-interval 1 \ - --recovery-time 5 \ - --transport distribution \ - --scenario mixed \ - --corruption "${corruption}" >"${log}" 2>&1 - status=$? - set -e - - if [[ "${expectation}" == "pass" ]] && [[ "${status}" -ne 0 ]]; then - echo "healthy Jepsen baseline failed; see ${log}" >&2 - return 1 - fi - - if [[ "${expectation}" == "fail" ]] && [[ "${status}" -eq 0 ]]; then - echo "Jepsen checker accepted corruption ${corruption}; see ${log}" >&2 - return 1 - fi - - echo "${expectation}: ${corruption} (${log})" -} - -run_jepsen pass none -run_jepsen fail unexpected-death -run_jepsen fail internal-index -run_jepsen fail cursor-marker -run_jepsen fail registry-projection -run_jepsen fail terminal-unavailable - -echo "mutation and live checker qualification passed" +exec elixir "${script_dir}/qualify.exs" "$@" diff --git a/test/jepsen/src/group/jepsen/core.clj b/test/jepsen/src/group/jepsen/core.clj index e01ca2d..b6a3f48 100644 --- a/test/jepsen/src/group/jepsen/core.clj +++ b/test/jepsen/src/group/jepsen/core.clj @@ -4,6 +4,7 @@ [group.jepsen.db :as group-db] [group.jepsen.model :as model] [group.jepsen.nemesis :as group-nemesis] + [group.jepsen.qualification :as qualification] [jepsen.cli :as cli] [jepsen.generator :as gen] [jepsen.os :as os] @@ -168,7 +169,7 @@ :nemesis (group-nemesis/nemesis db) :pure-generators true :generator (workload opts) - :checker (model/checker)}))) + :checker (qualification/checker (model/checker))}))) (def cli-options [[nil "--key-count NUMBER" "Number of keys in each cluster and data type" diff --git a/test/jepsen/src/group/jepsen/qualification.clj b/test/jepsen/src/group/jepsen/qualification.clj new file mode 100644 index 0000000..fbe2cdb --- /dev/null +++ b/test/jepsen/src/group/jepsen/qualification.clj @@ -0,0 +1,59 @@ +(ns group.jepsen.qualification + (:require [jepsen.checker :as checker])) + +(def internal-invariants + {:internal-index :registry-dual-indexes + :cursor-marker :cursor-snapshot-marker + :registry-projection :registry-projection}) + +(defn internal-qualified? [mode history result] + ;; Match completion and the precise assertion on the same injected node. + ;; Arming the cursor marker file is not injection: only a successful snapshot + ;; insertion can attest that a remote cursor existed and was corrupted. + (some (fn [op] + (let [internal (get-in result [:internal-invariant-errors + (get-in op [:value :node])]) + completed? (if (= :cursor-marker mode) + (some #{mode} (:injected-corruptions internal)) + (= mode (get-in op [:value :response :injected])))] + (and (= :corrupt (:f op)) + (= :ok (:type op)) + (= mode (get-in op [:value :request :mode])) + completed? + (some #{(internal-invariants mode)} (:failed-invariants internal))))) + history)) + +(defn qualified? [test history result] + (let [mode (keyword (:corruption test)) + injected? (some #(and (= :corrupt (:f %)) + (= :ok (:type %)) + (= mode (get-in % [:value :request :mode]))) + history)] + (boolean + (case mode + :none (true? (:valid? result)) + :unexpected-death (and injected? (seq (:unexpected-owner-deaths result))) + :internal-index (internal-qualified? mode history result) + :cursor-marker (internal-qualified? mode history result) + :registry-projection (internal-qualified? mode history result) + :terminal-unavailable + (let [target (first (:terminal-nodes test))] + (and (some #(and (= :retire-node (:f %)) + (= :info (:type %)) + (= target (get-in % [:value :retired]))) + history) + (contains? (:missing-nodes result) (name target)))) + false)))) + +(defn checker [delegate] + (reify checker/Checker + (check [_ test history opts] + (let [result (checker/check delegate test history opts)] + ;; A dedicated per-run artifact, never human log text. Exceptions and + ;; indeterminate results cannot certify a completed checker decision. + (when-let [path (System/getenv "GROUP_JEPSEN_QUALIFICATION_RESULT")] + (when (boolean? (:valid? result)) + (spit path (str "group-qualification-v1\t" (:corruption test) "\t" + (:valid? result) "\t" + (qualified? test history result) "\n")))) + result)))) diff --git a/test/jepsen/test/group/jepsen/qualification_test.clj b/test/jepsen/test/group/jepsen/qualification_test.clj new file mode 100644 index 0000000..f1dfa80 --- /dev/null +++ b/test/jepsen/test/group/jepsen/qualification_test.clj @@ -0,0 +1,62 @@ +(ns group.jepsen.qualification-test + (:require [clojure.test :refer :all] + [group.jepsen.qualification :as qualification])) + +(deftest corruption-must-be-injected-and-reach-its-check + (doseq [[mode field] [["unexpected-death" :unexpected-owner-deaths]]] + (let [test {:corruption mode} + history [{:f :corrupt :type :ok + :value {:request {:mode (keyword mode)}}}] + result {:valid? false field {:n1 :evidence}}] + (is (qualification/qualified? test history result)) + (is (not (qualification/qualified? test [] result))) + (is (not (qualification/qualified? test history {:valid? false}))) + (is (not (qualification/qualified? test + [(assoc (first history) :type :fail)] + result)))))) + +(deftest internal-corruption-requires-specific-completion-and-invariant + (doseq [[mode invariant] qualification/internal-invariants] + (let [test {:corruption (name mode)} + op {:f :corrupt :type :ok + :value {:node "n1" :request {:mode mode} + :response {:injected mode}}} + internal {:failed-invariants [invariant] :injected-corruptions [mode]} + result {:valid? false :internal-invariant-errors {"n1" internal}} + qualifies? #(qualification/qualified? test [%1] %2)] + (is (qualifies? op result)) + (is (not (qualifies? (assoc op :type :fail) result))) + (is (not (qualifies? (assoc-in op [:value :node] "n2") result))) + (is (not (qualifies? op (assoc-in result + [:internal-invariant-errors "n1" :failed-invariants] + [:unrelated-invariant])))) + (is (not (qualifies? (update-in op [:value] dissoc :response) + (assoc-in result + [:internal-invariant-errors "n1" :injected-corruptions] + []))))))) + +(deftest arming-cursor-injection-with-no-remote-cursor-is-not-qualification + (let [test {:corruption "cursor-marker"} + history [{:f :corrupt :type :ok + :value {:node "n1" :request {:mode :cursor-marker} + :response {:status :ok}}}] + result {:valid? false + :internal-invariant-errors + {"n1" {:healthy false + :errors ["invariant snapshot failed: no remote replica cursor available for corruption"] + :failed-invariants [] + :injected-corruptions [] + :snapshot-staging-count -1}}}] + (is (not (qualification/qualified? test history result))) + ;; Even a matching assertion elsewhere cannot replace injection completion. + (is (not (qualification/qualified? + test history + (assoc-in result [:internal-invariant-errors "n1" :failed-invariants] + [:cursor-snapshot-marker])))))) + +(deftest terminal-qualification-requires-retirement-and-missing-target + (let [test {:corruption "terminal-unavailable" :terminal-nodes [:n1 :n2 :n3]} + history [{:f :retire-node :type :info :value {:retired :n1}}]] + (is (qualification/qualified? test history {:missing-nodes #{"n1"}})) + (is (not (qualification/qualified? test [] {:missing-nodes #{"n1"}}))) + (is (not (qualification/qualified? test history {:missing-nodes #{"n2"}}))))) diff --git a/test/jepsen_qualification_cache_test.exs b/test/jepsen_qualification_cache_test.exs index 62de269..d5ce0d7 100644 --- a/test/jepsen_qualification_cache_test.exs +++ b/test/jepsen_qualification_cache_test.exs @@ -15,6 +15,7 @@ defmodule Group.JepsenQualificationCacheTest do File.mkdir_p!(script_dir) File.mkdir_p!(bin) File.cp!("test/jepsen/qualify.sh", Path.join(script_dir, "qualify.sh")) + File.cp!("test/jepsen/qualify.exs", Path.join(script_dir, "qualify.exs")) if existing_cache? do File.mkdir_p!(cache) diff --git a/test/jepsen_qualification_runner_test.exs b/test/jepsen_qualification_runner_test.exs new file mode 100644 index 0000000..e165529 --- /dev/null +++ b/test/jepsen_qualification_runner_test.exs @@ -0,0 +1,170 @@ +defmodule Group.JepsenQualificationRunnerTest do + use ExUnit.Case, async: true + + @moduletag :local + @moduletag tmp_dir: System.pid() + + @corruptions ~w(none unexpected-death internal-index cursor-marker registry-projection terminal-unavailable) + + setup %{tmp_dir: directory} do + repo = Path.expand("checkout with spaces", directory) + scripts = Path.join(repo, "test/jepsen") + bin = Path.expand("bin", directory) + probes = Path.expand("probes", directory) + + for path <- [scripts, bin, probes], do: File.mkdir_p!(path) + script = Path.join(scripts, "qualify.exs") + File.cp!("test/jepsen/qualify.exs", script) + + # Exercise the actual executable runner, replacing only the external + # mutation/Jepsen commands. Neither Docker nor the real mutation campaign runs. + for command <- ["mix", "timeout"] do + path = Path.join(bin, command) + File.write!(path, command_stub()) + File.chmod!(path, 0o755) + end + + {:ok, repo: repo, scripts: scripts, script: script, bin: bin, probes: probes} + end + + test "qualifies the healthy baseline and all five corruptions with bounded commands", context do + {output, status} = run_qualification(context) + assert status == 0, output + assert calls(context) == ["mutations" | @corruptions] + assert [artifacts] = artifact_directories(context) + + for mode <- @corruptions do + [cwd, skip_checker, result | args] = invocation(context, mode) + assert cwd == context.repo + assert skip_checker == "1" + assert result == Path.join(artifacts, "#{mode}.result") + + assert args == [ + "--signal=TERM", + "--kill-after=30", + "180", + Path.join(context.scripts, "run.sh"), + "test", + "--no-ssh", + "--nodes", + "n1,n2,n3", + "--concurrency", + "2n", + "--time-limit", + "6", + "--fault-interval", + "1", + "--recovery-time", + "5", + "--transport", + "distribution", + "--scenario", + "mixed", + "--corruption", + mode + ] + + assert File.read!(Path.join(artifacts, "#{mode}.log")) == "probe log #{mode}\n" + valid? = mode == "none" + assert File.read!(result) == "group-qualification-v1\t#{mode}\t#{valid?}\ttrue\n" + end + end + + test "a failed mutation baseline propagates without running live qualification", context do + assert {_, 42} = run_qualification(context, baseline_status: 42) + assert calls(context) == ["mutations"] + end + + test "a failing healthy history stops before any corruption", context do + assert {_, 1} = run_qualification(context, mode: "none", status: 1) + assert calls(context) == ["mutations", "none"] + end + + for {label, status, record} <- [ + {"unexpected acceptance", 0, "wrong-validity"}, + {"unqualified rejection", 1, "unqualified"}, + {"wrong corruption", 1, "wrong-mode"}, + {"wrong schema", 1, "wrong-version"}, + {"malformed record", 1, "malformed"}, + {"missing result", 1, "missing"}, + {"timeout with completed evidence", 124, "valid"}, + {"timeout invocation failure", 125, "valid"}, + {"killed process", 137, "missing"}, + {"missing executable", 127, "missing"} + ] do + @tag status: status, record: record + test "rejects #{label} and does not proceed to the next corruption", context do + assert {_, 1} = run_qualification(context, status: context.status, record: context.record) + assert calls(context) == ["mutations", "none", "unexpected-death"] + end + end + + test "a prior successful run cannot supply a missing result for the next run", context do + assert {_, 0} = run_qualification(context) + [first] = artifact_directories(context) + assert {_, 1} = run_qualification(context, record: "missing") + assert length(artifact_directories(context)) == 2 + [_, _, result | _] = invocation(context, "unexpected-death") + refute Path.dirname(result) == first + refute File.exists?(result) + assert File.exists?(Path.join(first, "unexpected-death.result")) + end + + defp run_qualification(context, opts \\ []) do + System.cmd("elixir", [context.script], + env: [ + {"PATH", context.bin <> ":" <> System.fetch_env!("PATH")}, + {"ERL_FLAGS", "+S 1:1"}, + {"QUALIFICATION_PROBE_DIR", context.probes}, + {"QUALIFICATION_PROBE_MODE", Keyword.get(opts, :mode, "unexpected-death")}, + {"QUALIFICATION_PROBE_RECORD", Keyword.get(opts, :record, "valid")}, + {"QUALIFICATION_PROBE_STATUS", opts[:status] && to_string(opts[:status])}, + {"QUALIFICATION_BASELINE_STATUS", to_string(Keyword.get(opts, :baseline_status, 0))} + ], + stderr_to_stdout: true + ) + end + + defp calls(context) do + context.probes |> Path.join("calls") |> File.read!() |> String.split("\n", trim: true) + end + + defp invocation(context, mode) do + context.probes |> Path.join("#{mode}.txt") |> File.read!() |> String.split("\n") + end + + defp artifact_directories(context) do + Path.wildcard(Path.join(context.scripts, ".cache/qualification.*")) + end + + defp command_stub do + ~S""" + #!/usr/bin/env elixir + args = System.argv() + probes = System.fetch_env!("QUALIFICATION_PROBE_DIR") + mode = if args == ["run", "test/mutation/run.exs"], do: "mutations", else: List.last(args) + File.write!(Path.join(probes, "calls"), mode <> "\n", [:append]) + result = System.get_env("GROUP_JEPSEN_QUALIFICATION_RESULT", "absent") + skip = System.get_env("GROUP_JEPSEN_SKIP_CHECKER", "unset") + File.write!(Path.join(probes, "#{mode}.txt"), Enum.join([File.cwd!(), skip, result | args], "\n")) + IO.puts("probe log #{mode}") + + if mode == "mutations" do + System.halt(String.to_integer(System.fetch_env!("QUALIFICATION_BASELINE_STATUS"))) + end + + targeted? = mode == System.fetch_env!("QUALIFICATION_PROBE_MODE") + status = if mode == "none", do: "0", else: "1" + status = if targeted?, do: System.get_env("QUALIFICATION_PROBE_STATUS") || status, else: status + kind = if targeted?, do: System.fetch_env!("QUALIFICATION_PROBE_RECORD"), else: "valid" + valid? = mode == "none" + valid? = if kind == "wrong-validity", do: not valid?, else: valid? + record_mode = if kind == "wrong-mode", do: "different-corruption", else: mode + version = if kind == "wrong-version", do: "group-qualification-v2", else: "group-qualification-v1" + record = "#{version}\t#{record_mode}\t#{valid?}\t#{kind != "unqualified"}\n" + record = if kind == "malformed", do: "not a qualification record\n", else: record + unless kind == "missing", do: File.write!(result, record) + System.halt(String.to_integer(status)) + """ + end +end