feat: implement TableCleanupService for periodic cleanup of excess discrete probabilities and utilities (#345)

* feat: implement TableCleanupService for periodic cleanup of excess discrete probabilities and utilities

* refactor: optimize cleanup loop in TableCleanupService using PeriodicTimer

Author: EinarSalbouvet

PrismaDotnetApi/PrismaApi.Api/Program.cs

@@ -5,6 +5,7 @@
 using PrismaApi.Api.Configuration.Extensions;
 using PrismaApi.Api.Configuration.JsonResponseOptions;
 using PrismaApi.Api.SecurityPolicy;
+using PrismaApi.Application.BackgroundServices;
 using PrismaApi.Application.Interfaces.Repositories;
 using PrismaApi.Application.Interfaces.Services;
 using PrismaApi.Application.Repositories;
@@ -137,6 +138,7 @@ public static void Main(string[] args)
         builder.Services.AddScoped<IObjectiveService, ObjectiveService>();
         builder.Services.AddScoped<IProjectRoleService, ProjectRoleService>();
         builder.Services.AddScoped<IBoardNodeService, BoardNodeService>();
+        builder.Services.AddHostedService<TableCleanupService>();
 
         if (isPublicInstance)
         {

PrismaDotnetApi/PrismaApi.Application/BackgroundServices/TableCleanupService.cs

@@ -0,0 +1,62 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using PrismaApi.Application.Interfaces.Services;
+
+namespace PrismaApi.Application.BackgroundServices;
+
+public class TableCleanupService : BackgroundService
+{
+    private readonly IServiceScopeFactory _scopeFactory;
+    private readonly ILogger<TableCleanupService> _logger;
+    private readonly TimeSpan _delay = TimeSpan.FromMinutes(10);
+
+    public TableCleanupService(IServiceScopeFactory serviceScopeFactory, ILogger<TableCleanupService> logger)
+    {
+        _scopeFactory = serviceScopeFactory;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken ct = default)
+    {
+        try
+        {
+            await CleanupLoopAsync(ct);
+        }
+        catch (TaskCanceledException)
+        {
+            _logger.LogInformation("TableCleanupService is stopping due to cancellation.");
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "An error occurred in TableCleanupService.");
+        }
+    }
+
+    /// <summary>
+    /// This is a temporary solution to clean up excess discrete probabilities and utilities until we implement a more robust solution in the future. It continuously runs a loop that cleans up excess discrete probabilities and utilities every 10 or so minutes. This is necessary because currently we do not have a way to automatically clean up excess discrete probabilities and utilities.
+    /// </summary>
+    private async Task CleanupLoopAsync(CancellationToken ct = default)
+    {
+        using PeriodicTimer timer = new(_delay);
+        while (await timer.WaitForNextTickAsync(ct))
+        {
+            try
+            {
+                var startTime = DateTime.UtcNow;
+                using var scope = _scopeFactory.CreateScope();
+                var tableRebuildingService = scope.ServiceProvider.GetRequiredService<ITableRebuildingService>();
+                await tableRebuildingService.RemoveExcessDiscreteProbabilities(ct);
+                await tableRebuildingService.RemoveExcessDiscreteUtilities(ct);
+                var endTime = DateTime.UtcNow;
+                // logging the duration of the cleanup process, 
+                // if the duration in the future becomes consistantly short it implyies that our fixes for excess discrete probabilities and utilities are working and we can remove this background service
+                _logger.LogInformation("Table cleanup completed successfully in {Duration} ms.", (endTime - startTime).TotalMilliseconds);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "An error occurred in CleanupLoopAsync.");
+            }
+        }
+    }
+}

PrismaDotnetApi/PrismaApi.Application/Interfaces/Services/ITableRebuildingService.cs

@@ -5,5 +5,7 @@ public interface ITableRebuildingService
     Task RebuildTablesAsync(CancellationToken ct = default);
     Task RebuildIssuesFromIssueIds(ICollection<Guid> issueIds, CancellationToken ct = default);
     Task RemoveExcessDiscreteProbabilities(Guid uncertaintyId, CancellationToken ct = default);
+    Task RemoveExcessDiscreteProbabilities(CancellationToken ct = default);
     Task RemoveExcessDiscreteUtilities(Guid utilityId, CancellationToken ct = default);
+    Task RemoveExcessDiscreteUtilities(CancellationToken ct = default);
 }

PrismaDotnetApi/PrismaApi.Application/Services/TableRebuildingService.cs

@@ -282,12 +282,28 @@ public async Task RemoveExcessDiscreteProbabilities(Guid uncertaintyId, Cancella
     {
         var discreteProbabilities = await FindExcessDiscreteProbabilities(uncertaintyId, ct);
         RemoveDiscreteProbabilities(discreteProbabilities);
+        await DbContext.SaveChangesAsync(ct);
+    }
+
+    public async Task RemoveExcessDiscreteProbabilities(CancellationToken ct = default)
+    {
+        var discreteProbabilities = await FindExcessDiscreteProbabilities(ct);
+        RemoveDiscreteProbabilities(discreteProbabilities);
+        await DbContext.SaveChangesAsync(ct);
     }
 
     public async Task RemoveExcessDiscreteUtilities(Guid utilityId, CancellationToken ct = default)
     {
         var discreteUtilities = await FindExcessDiscreteUtility(utilityId, ct);
         RemoveDiscreteUtilities(discreteUtilities);
+        await DbContext.SaveChangesAsync(ct);
+    }
+
+    public async Task RemoveExcessDiscreteUtilities(CancellationToken ct = default)
+    {
+        var discreteUtilities = await FindExcessDiscreteUtility(ct);
+        RemoveDiscreteUtilities(discreteUtilities);
+        await DbContext.SaveChangesAsync(ct);
     }
 
     private async Task<IEnumerable<DiscreteProbability>> FindExcessDiscreteProbabilities(Guid uncertaintyId, CancellationToken ct = default)
@@ -307,6 +323,22 @@ private async Task<IEnumerable<DiscreteProbability>> FindExcessDiscreteProbabili
             .SelectMany(group => group.Skip(1));
     }
 
+    private async Task<IEnumerable<DiscreteProbability>> FindExcessDiscreteProbabilities(CancellationToken ct = default)
+    {
+        var discreteProbabilities = await DbContext.DiscreteProbabilities
+            .Include(p => p.ParentOutcomes)
+            .Include(p => p.ParentOptions)
+            .ToListAsync(ct);
+
+        return discreteProbabilities
+            .GroupBy(p => (
+                p.OutcomeId,
+                ParentOutcomes: string.Join(",", p.ParentOutcomes.Select(o => o.ParentOutcomeId).OrderBy(id => id)),
+                ParentOptions: string.Join(",", p.ParentOptions.Select(o => o.ParentOptionId).OrderBy(id => id))
+            ))
+            .SelectMany(group => group.Skip(1));
+    }
+
     private async Task<IEnumerable<DiscreteUtility>> FindExcessDiscreteUtility(Guid utilityId, CancellationToken ct = default)
     {
         var discreteUtilities = await DbContext.DiscreteUtilities
@@ -324,6 +356,22 @@ private async Task<IEnumerable<DiscreteUtility>> FindExcessDiscreteUtility(Guid
             .SelectMany(group => group.Skip(1));
     }
 
+    private async Task<IEnumerable<DiscreteUtility>> FindExcessDiscreteUtility(CancellationToken ct = default)
+    {
+        var discreteUtilities = await DbContext.DiscreteUtilities
+            .Include(p => p.ParentOutcomes)
+            .Include(p => p.ParentOptions)
+            .ToListAsync(ct);
+
+        return discreteUtilities
+            .GroupBy(p => (
+                p.ValueMetricId,
+                ParentOutcomes: string.Join(",", p.ParentOutcomes.Select(o => o.ParentOutcomeId).OrderBy(id => id)),
+                ParentOptions: string.Join(",", p.ParentOptions.Select(o => o.ParentOptionId).OrderBy(id => id))
+            ))
+            .SelectMany(group => group.Skip(1));
+    }
+
     private void RemoveDiscreteProbabilities(IEnumerable<DiscreteProbability> discreteProbabilities)
     {
         var discreteProbabilityIds = discreteProbabilities.Select(probability => probability.Id).ToList();