Add fallback region for rpt s3

Author: louiscb

lib/remote_persistent_term/fetcher/s3.ex

@@ -9,9 +9,10 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
   @type t :: %__MODULE__{
           bucket: String.t(),
           key: String.t(),
-          region: String.t()
+          region: String.t(),
+          failover_region: String.t() | nil
         }
-  defstruct [:bucket, :key, :region]
+  defstruct [:bucket, :key, :region, :failover_region]
 
   @opts_schema [
     bucket: [
@@ -28,6 +29,11 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
       type: :string,
       required: true,
       doc: "The AWS region of the s3 bucket."
+    ],
+    failover_region: [
+      type: :string,
+      required: false,
+      doc: "The AWS region to use if calls to the default region fail."
     ]
   ]
 
@@ -50,7 +56,8 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
        %__MODULE__{
          bucket: valid_opts[:bucket],
          key: valid_opts[:key],
-         region: valid_opts[:region]
+         region: valid_opts[:region],
+         failover_region: valid_opts[:failover_region]
        }}
     end
   end
@@ -94,7 +101,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
     res =
       state.bucket
       |> ExAws.S3.get_bucket_object_versions(prefix: state.key)
-      |> aws_client_request(state.region)
+      |> aws_client_request(state)
 
     with {:ok, %{body: %{versions: versions}}} <- res do
       {:ok, versions}
@@ -104,7 +111,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
   defp get_object(state) do
     state.bucket
     |> ExAws.S3.get_object(state.key)
-    |> aws_client_request(state.region)
+    |> aws_client_request(state)
   end
 
   defp find_latest([_ | _] = contents) do
@@ -123,8 +130,17 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
 
   defp find_latest(_), do: {:error, :not_found}
 
-  defp aws_client_request(op, region) do
-    client().request(op, region: region)
+  defp aws_client_request(op, %{region: region, failover_region: nil}),
+    do: client().request(op, region: region)
+
+  defp aws_client_request(op, %{region: region, failover_region: failover_region}) do
+    with {:error, reason} <- client().request(op, region: region) do
+      Logger.error(
+        "Failed to fetch from primary region #{region}: #{inspect(reason)}, will try failover region #{failover_region}"
+      )
+
+      client().request(op, region: failover_region)
+    end
   end
 
   defp client, do: Application.get_env(:remote_persistent_term, :aws_client, ExAws)

test/remote_persistent_term/fetcher/s3_test.exs

@@ -5,6 +5,12 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do
   setup :verify_on_exit!
   import ExUnit.CaptureLog
 
+  @bucket "test-bucket"
+  @key "test-key"
+  @region "test-region"
+  @failover_region "failover-region"
+  @version "F76V.weh4uOlU15f7a2OLHPgCLXkDpm4"
+
   test "Unknown error returns an error for current_version/1" do
     expect(AwsClientMock, :request, fn _op, _opts ->
       {:error, :unknown_error}
@@ -26,4 +32,101 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do
                S3.init(bucket: bucket, key: key, region: region)
     end
   end
+
+  describe "failover_region" do
+    test "current_identifiers/1 tries failover region when primary region fails" do
+      # Setup state with failover region
+      state = %S3{
+        bucket: @bucket,
+        key: @key,
+        region: @region,
+        failover_region: @failover_region
+      }
+
+      # Mock the AWS client to fail for primary region but succeed for failover region
+      expect(AwsClientMock, :request, 2, fn _op, opts ->
+        case opts do
+          [region: @region] ->
+            {:error, "Primary region connection error"}
+
+          [region: @failover_region] ->
+            {:ok,
+             %{
+               body: %{
+                 versions: [
+                   %{version_id: @version, etag: "current-etag", is_latest: "true"}
+                 ]
+               }
+             }}
+        end
+      end)
+
+      log =
+        capture_log(fn ->
+          result = S3.current_version(state)
+          assert {:ok, "current-etag"} = result
+        end)
+
+      assert log =~ "Failed to fetch from primary region #{@region}"
+      assert log =~ "will try failover region #{@failover_region}"
+    end
+
+    test "download/1 tries failover region when primary region fails" do
+      state = %S3{
+        bucket: @bucket,
+        key: @key,
+        region: @region,
+        failover_region: @failover_region
+      }
+
+      # Mock the AWS client to fail for primary region but succeed for failover region
+      expect(AwsClientMock, :request, 2, fn _op, opts ->
+        case opts do
+          [region: @region] ->
+            {:error, "Primary region connection error"}
+
+          [region: @failover_region] ->
+            {:ok, %{body: "content from failover region"}}
+        end
+      end)
+
+      log =
+        capture_log(fn ->
+          result = S3.download(state)
+          assert {:ok, "content from failover region"} = result
+        end)
+
+      assert log =~ "Failed to fetch from primary region #{@region}"
+      assert log =~ "will try failover region #{@failover_region}"
+    end
+
+    test "returns error when both primary and failover regions fail" do
+      state = %S3{
+        bucket: @bucket,
+        key: @key,
+        region: @region,
+        failover_region: @failover_region
+      }
+
+      # Mock the AWS client to fail for both regions
+      expect(AwsClientMock, :request, 2, fn _op, opts ->
+        case opts do
+          [region: @region] ->
+            {:error, "Primary region connection error"}
+
+          [region: @failover_region] ->
+            {:error, "Failover region connection error"}
+        end
+      end)
+
+      log =
+        capture_log(fn ->
+          result = S3.download(state)
+          assert {:error, _} = result
+        end)
+
+      assert log =~ "Failed to fetch from primary region #{@region}"
+      assert log =~ "will try failover region #{@failover_region}"
+    end
+  end
 end