Hardened the handled marking on non-transactional idempotency. Closes GH-2036

Author: jeremydmiller

src/Wolverine/Persistence/Durability/PersistHandled.cs

@@ -0,0 +1,14 @@
+using Wolverine.Runtime;
+using Wolverine.Runtime.Agents;
+
+namespace Wolverine.Persistence.Durability;
+
+public class PersistHandled(Envelope Handled) : IAgentCommand
+{
+    public async Task<AgentCommands> ExecuteAsync(IWolverineRuntime runtime, CancellationToken cancellationToken)
+    {
+        await runtime.Storage.Inbox.StoreIncomingAsync(Handled);
+        
+        return AgentCommands.Empty;
+    }
+}

src/Wolverine/Runtime/MessageContext.cs

@@ -102,9 +102,18 @@ public async Task AssertEagerIdempotencyAsync(CancellationToken cancellation)
 
     public async Task PersistHandledAsync()
     {
-        // TODO -- add some retries here!!!!
         var handled = Envelope.ForPersistedHandled(Envelope, DateTimeOffset.UtcNow, Runtime.Options.Durability);
-        await Runtime.Storage.Inbox.StoreIncomingAsync(handled);
+        try
+        {
+            await Runtime.Storage.Inbox.StoreIncomingAsync(handled);
+        }
+        catch (Exception e)
+        {
+            Runtime.Logger.LogError(e, "Error trying to mark message {Id} as handled. Retrying later.", handled.Id);
+
+            // Retry this off to the side...
+            await new MessageBus(Runtime).PublishAsync(new PersistHandled(handled));
+        }
     }
 
     public async Task FlushOutgoingMessagesAsync()