feat: Add orchestration termination functionality and enhance cancel operation with instance ID handling

Author: frasermolyneux

src/XtremeIdiots.Portal.Web/Controllers/MapRotationsController.cs

@@ -726,7 +726,7 @@ public async Task<IActionResult> GetSyncProgress(string instanceId, Cancellation
 
     [HttpPost]
     [ValidateAntiForgeryToken]
-    public async Task<IActionResult> CancelOperation(Guid operationId, Guid assignmentId, CancellationToken cancellationToken = default)
+    public async Task<IActionResult> CancelOperation(Guid operationId, Guid assignmentId, string? instanceId, CancellationToken cancellationToken = default)
     {
         return await ExecuteWithErrorHandlingAsync(async () =>
         {
@@ -759,6 +759,24 @@ public async Task<IActionResult> CancelOperation(Guid operationId, Guid assignme
 
             if (updateResult.IsSuccess)
             {
+                // Terminate the durable function orchestration so a new one can be started
+                if (!string.IsNullOrEmpty(instanceId))
+                {
+                    // Validate the instance ID belongs to this assignment
+                    var allowedPrefixes = new[]
+                    {
+                        $"maprot-sync-{assignmentId}",
+                        $"maprot-activate-{assignmentId}",
+                        $"maprot-deactivate-{assignmentId}",
+                        $"maprot-remove-{assignmentId}"
+                    };
+
+                    if (allowedPrefixes.Any(p => string.Equals(instanceId, p, StringComparison.OrdinalIgnoreCase)))
+                    {
+                        await syncApiClient.TerminateOrchestration(instanceId, cancellationToken).ConfigureAwait(false);
+                    }
+                }
+
                 // Also reset the assignment state so the user can retry
                 var deploymentReset = assignment.DeploymentState is DeploymentState.Syncing or DeploymentState.Removing
                     ? DeploymentState.Failed
@@ -791,4 +809,57 @@ public async Task<IActionResult> CancelOperation(Guid operationId, Guid assignme
             return RedirectToAction(nameof(AssignmentStatus), new { id = assignmentId });
         }, nameof(CancelOperation)).ConfigureAwait(false);
     }
+
+    [HttpPost]
+    [ValidateAntiForgeryToken]
+    public async Task<IActionResult> TerminateOrchestration(string instanceId, Guid assignmentId, CancellationToken cancellationToken = default)
+    {
+        return await ExecuteWithErrorHandlingAsync(async () =>
+        {
+            // Validate the instance ID belongs to this assignment
+            var allowedPrefixes = new[]
+            {
+                $"maprot-sync-{assignmentId}",
+                $"maprot-activate-{assignmentId}",
+                $"maprot-deactivate-{assignmentId}",
+                $"maprot-remove-{assignmentId}"
+            };
+
+            if (!allowedPrefixes.Any(p => string.Equals(instanceId, p, StringComparison.OrdinalIgnoreCase)))
+                return BadRequest("The instance ID does not belong to the specified assignment.");
+
+            var assignmentResponse = await repositoryApiClient.MapRotations.V1.GetServerAssignment(assignmentId, cancellationToken).ConfigureAwait(false);
+
+            if (assignmentResponse.IsNotFound || assignmentResponse.Result?.Data is null)
+                return NotFound();
+
+            var rotationResponse = await repositoryApiClient.MapRotations.V1.GetMapRotation(assignmentResponse.Result.Data.MapRotationId, cancellationToken).ConfigureAwait(false);
+
+            if (rotationResponse.IsNotFound || rotationResponse.Result?.Data is null)
+                return NotFound();
+
+            var authResult = await CheckAuthorizationAsync(
+                authorizationService,
+                rotationResponse.Result.Data.GameType,
+                AuthPolicies.ManageMapRotations,
+                nameof(TerminateOrchestration),
+                "MapRotation").ConfigureAwait(false);
+
+            if (authResult != null)
+                return authResult;
+
+            var terminated = await syncApiClient.TerminateOrchestration(instanceId, cancellationToken).ConfigureAwait(false);
+
+            if (terminated)
+            {
+                this.AddAlertSuccess($"Orchestration '{instanceId}' terminated. You can now re-trigger the operation.");
+            }
+            else
+            {
+                this.AddAlertDanger($"Failed to terminate orchestration '{instanceId}'. It may have already completed or the sync service is unavailable.");
+            }
+
+            return RedirectToAction(nameof(AssignmentStatus), new { id = assignmentId });
+        }, nameof(TerminateOrchestration)).ConfigureAwait(false);
+    }
 }

src/XtremeIdiots.Portal.Web/Services/ISyncApiClient.cs

@@ -7,6 +7,7 @@ public interface ISyncApiClient
     Task<SyncTriggerResult> TriggerDeactivate(Guid assignmentId, CancellationToken cancellationToken = default);
     Task<SyncTriggerResult> TriggerRemove(Guid assignmentId, CancellationToken cancellationToken = default);
     Task<OrchestrationStatusQueryResult> GetOrchestrationStatus(string instanceId, CancellationToken cancellationToken = default);
+    Task<bool> TerminateOrchestration(string instanceId, CancellationToken cancellationToken = default);
 }
 
 public record SyncTriggerResult(bool Success, string? InstanceId = null, string? Error = null);

src/XtremeIdiots.Portal.Web/Services/NoOpSyncApiClient.cs

@@ -26,4 +26,9 @@ public Task<OrchestrationStatusQueryResult> GetOrchestrationStatus(string instan
     {
         return Task.FromResult(new OrchestrationStatusQueryResult(OrchestrationStatusQueryOutcome.Error));
     }
+
+    public Task<bool> TerminateOrchestration(string instanceId, CancellationToken cancellationToken = default)
+    {
+        return Task.FromResult(false);
+    }
 }

src/XtremeIdiots.Portal.Web/Services/SyncApiClient.cs

@@ -63,6 +63,34 @@ public async Task<OrchestrationStatusQueryResult> GetOrchestrationStatus(string
         }
     }
 
+    public async Task<bool> TerminateOrchestration(string instanceId, CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            var tokenResult = await sharedCredential.GetTokenAsync(
+                new TokenRequestContext([applicationAudience + "/.default"]), cancellationToken).ConfigureAwait(false);
+
+            using var request = new HttpRequestMessage(HttpMethod.Post, baseUrl.TrimEnd('/') + $"/api/map-rotations/terminate/{instanceId}");
+            request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", tokenResult.Token);
+
+            var response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false);
+
+            if (response.IsSuccessStatusCode)
+            {
+                logger.LogInformation("Successfully terminated orchestration {InstanceId}", instanceId);
+                return true;
+            }
+
+            logger.LogWarning("Failed to terminate orchestration {InstanceId}, status {StatusCode}", instanceId, (int)response.StatusCode);
+            return false;
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to terminate orchestration {InstanceId}", instanceId);
+            return false;
+        }
+    }
+
     private async Task<SyncTriggerResult> TriggerOrchestration(string path, CancellationToken cancellationToken)
     {
         try

src/XtremeIdiots.Portal.Web/Views/MapRotations/AssignmentStatus.cshtml

@@ -263,11 +263,23 @@
                                                             @Html.AntiForgeryToken()
                                                             <input type="hidden" name="operationId" value="@op.MapRotationAssignmentOperationId" />
                                                             <input type="hidden" name="assignmentId" value="@Model.Assignment.MapRotationServerAssignmentId" />
+                                                            <input type="hidden" name="instanceId" value="@op.DurableFunctionInstanceId" />
                                                             <button type="submit" class="btn btn-outline-danger btn-sm" onclick="return confirm('Cancel this operation? The assignment state will be reset so you can retry.')">
                                                                 <i class="fa-solid fa-ban"></i> Cancel
                                                             </button>
                                                         </form>
                                                     }
+                                                    @if (op.Status is AssignmentOperationStatus.Cancelled or AssignmentOperationStatus.Failed && !string.IsNullOrEmpty(op.DurableFunctionInstanceId))
+                                                    {
+                                                        <form policy="@AuthPolicies.ManageMapRotations" asp-action="TerminateOrchestration" method="post" class="d-inline">
+                                                            @Html.AntiForgeryToken()
+                                                            <input type="hidden" name="instanceId" value="@op.DurableFunctionInstanceId" />
+                                                            <input type="hidden" name="assignmentId" value="@Model.Assignment.MapRotationServerAssignmentId" />
+                                                            <button type="submit" class="btn btn-outline-warning btn-sm" onclick="return confirm('Terminate the underlying orchestration? This clears the stuck durable function so you can re-trigger.')">
+                                                                <i class="fa-solid fa-skull-crossbones"></i> Terminate
+                                                            </button>
+                                                        </form>
+                                                    }
                                                 </td>
                                             </tr>
                                             @if (isInProgress)