Fix billing downgrade recharge and suspension race

src/Exceptionless.Core/Billing/AutoRenewingLock.cs

@@ -0,0 +1,179 @@
+using System.Runtime.ExceptionServices;
+using Foundatio.Lock;
+
+namespace Exceptionless.Core.Billing;
+
+internal sealed class AutoRenewingLock : ILock
+{
+    private readonly CancellationTokenSource _shutdown = new();
+    private readonly ILock _inner;
+    private readonly TimeProvider _timeProvider;
+    private readonly TimeSpan _renewalDuration;
+    private readonly TimeSpan _renewalInterval;
+    private readonly SemaphoreSlim _renewalLock = new(1, 1);
+    private readonly Task _renewalTask;
+    private ExceptionDispatchInfo? _renewalFailure;
+    private int _renewalFailureObserved;
+    private int _isReleased;
+
+    internal AutoRenewingLock(ILock inner, TimeProvider timeProvider, TimeSpan renewalDuration, TimeSpan renewalInterval)
+    {
+        _inner = inner;
+        _timeProvider = timeProvider;
+        _renewalDuration = renewalDuration;
+        _renewalInterval = renewalInterval;
+        _renewalTask = RenewUntilReleasedAsync();
+    }
+
+    public string LockId => _inner.LockId;
+    public string Resource => _inner.Resource;
+    public DateTime AcquiredTimeUtc => _inner.AcquiredTimeUtc;
+    public TimeSpan TimeWaitedForLock => _inner.TimeWaitedForLock;
+    public int RenewalCount => _inner.RenewalCount;
+
+    public async Task RenewAsync(TimeSpan? timeUntilExpires = null)
+    {
+        ThrowIfReleased();
+        ThrowRenewalFailure();
+        try
+        {
+            await RenewInnerAsync(
+                timeUntilExpires ?? _renewalDuration
+            );
+        }
+        catch (Exception exception)
+        {
+            if (ReferenceEquals(
+                _renewalFailure?.SourceException,
+                exception))
+            {
+                Interlocked.Exchange(
+                    ref _renewalFailureObserved,
+                    1
+                );
+            }
+            throw;
+        }
+        ThrowRenewalFailure();
+    }
+
+    public Task ReleaseAsync() => StopAsync(release: true);
+
+    public async ValueTask DisposeAsync()
+    {
+        await StopAsync(release: false);
+    }
+
+    private async Task RenewUntilReleasedAsync()
+    {
+        try
+        {
+            while (true)
+            {
+                await Task.Delay(_renewalInterval, _timeProvider, _shutdown.Token);
+                await RenewInnerAsync(_renewalDuration);
+            }
+        }
+        catch (OperationCanceledException) when (_shutdown.IsCancellationRequested)
+        {
+        }
+        catch (ObjectDisposedException) when (_shutdown.IsCancellationRequested)
+        {
+        }
+        catch (Exception ex)
+        {
+            CaptureRenewalFailure(ex);
+        }
+    }
+
+    private async Task RenewInnerAsync(TimeSpan renewalDuration)
+    {
+        await _renewalLock.WaitAsync();
+        try
+        {
+            ThrowIfReleased();
+            await _inner.RenewAsync(renewalDuration);
+        }
+        catch (ObjectDisposedException) when (_shutdown.IsCancellationRequested)
+        {
+            throw;
+        }
+        catch (Exception ex)
+        {
+            CaptureRenewalFailure(ex);
+            throw;
+        }
+        finally
+        {
+            _renewalLock.Release();
+        }
+    }
+
+    private async Task StopAsync(bool release)
+    {
+        if (Interlocked.Exchange(ref _isReleased, 1) is not 0)
+            return;
+
+        await _shutdown.CancelAsync();
+        await _renewalTask;
+
+        ExceptionDispatchInfo? cleanupFailure = null;
+        await _renewalLock.WaitAsync();
+        try
+        {
+            if (release)
+                await _inner.ReleaseAsync();
+            else
+                await _inner.DisposeAsync();
+        }
+        catch (Exception ex)
+        {
+            cleanupFailure = ExceptionDispatchInfo.Capture(ex);
+        }
+        finally
+        {
+            _renewalLock.Release();
+            _renewalLock.Dispose();
+            _shutdown.Dispose();
+        }
+
+        bool hasRenewalFailure = _renewalFailure is not null;
+        bool hasUnobservedRenewalFailure = hasRenewalFailure
+            && Volatile.Read(ref _renewalFailureObserved) is 0;
+        if (hasRenewalFailure && cleanupFailure is not null)
+        {
+            Interlocked.Exchange(ref _renewalFailureObserved, 1);
+            throw new AggregateException(
+                _renewalFailure!.SourceException,
+                cleanupFailure.SourceException
+            );
+        }
+
+        cleanupFailure?.Throw();
+        if (hasUnobservedRenewalFailure)
+            ThrowRenewalFailure();
+    }
+
+    private void ThrowIfReleased()
+    {
+        if (Volatile.Read(ref _isReleased) is not 0)
+            throw new ObjectDisposedException(nameof(AutoRenewingLock));
+    }
+
+    private void ThrowRenewalFailure()
+    {
+        var renewalFailure = _renewalFailure;
+        if (renewalFailure is null)
+            return;
+
+        Interlocked.Exchange(ref _renewalFailureObserved, 1);
+        renewalFailure.Throw();
+    }
+
+    private void CaptureRenewalFailure(Exception exception)
+        => Interlocked.CompareExchange(
+            ref _renewalFailure,
+            ExceptionDispatchInfo.Capture(exception),
+            null
+        );
+}

src/Exceptionless.Core/Billing/BillingManager.cs

@@ -2,18 +2,26 @@
 using Exceptionless.Core.Models;
 using Exceptionless.Core.Models.Billing;
 using Exceptionless.Core.Repositories;
+using Foundatio.Lock;
+using Stripe;
 
 namespace Exceptionless.Core.Billing;
 
 public class BillingManager
 {
+    private readonly ILockProvider? _lockProvider;
     private readonly IOrganizationRepository _organizationRepository;
     private readonly IProjectRepository _projectRepository;
     private readonly IUserRepository _userRepository;
     private readonly BillingPlans _plans;
     private readonly TimeProvider _timeProvider;
 
-    public BillingManager(IOrganizationRepository organizationRepository, IProjectRepository projectRepository, IUserRepository userRepository, BillingPlans plans, TimeProvider timeProvider)
+    public BillingManager(
+        IOrganizationRepository organizationRepository,
+        IProjectRepository projectRepository,
+        IUserRepository userRepository,
+        BillingPlans plans,
+        TimeProvider timeProvider)
     {
         _organizationRepository = organizationRepository;
         _projectRepository = projectRepository;
@@ -22,6 +30,53 @@ public BillingManager(IOrganizationRepository organizationRepository, IProjectRe
         _timeProvider = timeProvider;
     }
 
+    public BillingManager(
+        ILockProvider lockProvider,
+        IOrganizationRepository organizationRepository,
+        IProjectRepository projectRepository,
+        IUserRepository userRepository,
+        BillingPlans plans,
+        TimeProvider timeProvider)
+        : this(
+            organizationRepository,
+            projectRepository,
+            userRepository,
+            plans,
+            timeProvider)
+    {
+        _lockProvider = lockProvider;
+    }
+
+    public Task<ILock> AcquireOrganizationLockAsync(string organizationId)
+        => CreateOperationLockProvider(_timeProvider).AcquireAsync(organizationId);
+
+    internal bool ApplyBillingStatus(Organization organization, BillingStatus status, string suspensionNotes)
+    {
+        return BillingStatePolicy.Apply(
+            organization,
+            status,
+            suspensionNotes,
+            _timeProvider.GetUtcNow().UtcDateTime
+        );
+    }
+
+    internal bool ApplySubscription(Organization organization, Subscription subscription)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(subscription.Id);
+
+        var status = subscription.GetBillingStatus()
+            ?? throw new InvalidOperationException($"Unsupported Stripe subscription status \"{subscription.Status}\".");
+        bool hasChanges = !String.Equals(organization.StripeSubscriptionId, subscription.Id, StringComparison.Ordinal);
+        organization.StripeSubscriptionId = subscription.Id;
+
+        hasChanges = ApplyBillingStatus(
+            organization,
+            status,
+            $"Stripe subscription status changed to \"{status}\"."
+        ) || hasChanges;
+        return hasChanges;
+    }
+
     public async Task<bool> CanAddOrganizationAsync(User? user)
     {
         if (user is null)
@@ -94,6 +149,85 @@ public async Task<ChangePlanResult> CanDownGradeAsync(Organization organization,
         return _plans.Plans.Where(p => p.RetentionDays > retentionDays && p.Price > 0).OrderBy(p => p.RetentionDays).ThenBy(p => p.Price).FirstOrDefault();
     }
 
+    public Task<ILock?> TryAcquireOrganizationLockAsync(string organizationId)
+        => CreateOperationLockProvider(_timeProvider).TryAcquireAsync(organizationId);
+
+    internal BillingOperationLockProvider CreateOperationLockProvider(TimeProvider timeProvider)
+        => TryCreateOperationLockProvider(timeProvider)
+            ?? throw new InvalidOperationException(
+                "Billing lock operations require the lock-aware BillingManager constructor."
+            );
+
+    internal BillingOperationLockProvider? TryCreateOperationLockProvider(
+        TimeProvider timeProvider)
+        => _lockProvider is null
+            ? null
+            : new BillingOperationLockProvider(
+                _lockProvider,
+                timeProvider
+            );
+
+    internal Subscription? GetCurrentSubscription(Organization organization, IEnumerable<Subscription> subscriptions)
+    {
+        return GetLiveBillingSubscriptions(subscriptions)
+            .OrderByDescending(subscription => GetSingleBillingPlan(subscription) is not null)
+            .ThenByDescending(GetSubscriptionPriority)
+            .ThenByDescending(subscription => String.Equals(
+                subscription.Id,
+                organization.StripeSubscriptionId,
+                StringComparison.Ordinal
+            ))
+            .ThenByDescending(subscription => subscription.HasPrice(organization.PlanId))
+            .ThenByDescending(subscription => subscription.Created)
+            .ThenByDescending(subscription => subscription.Id, StringComparer.Ordinal)
+            .FirstOrDefault();
+    }
+
+    internal IReadOnlyCollection<Subscription> GetLiveBillingSubscriptions(IEnumerable<Subscription> subscriptions)
+    {
+        var paidPlanIds = _plans.Plans
+            .Where(plan => plan.Price > 0)
+            .Select(plan => plan.Id)
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+        return GetLiveSubscriptions(subscriptions)
+            .Where(subscription => subscription.GetBillingStatus().HasValue
+                && GetBillingSubscriptionItems(subscription, paidPlanIds).Count > 0)
+            .OrderByDescending(subscription => subscription.Created)
+            .ThenByDescending(subscription => subscription.Id, StringComparer.Ordinal)
+            .ToList();
+    }
+
+    internal IReadOnlyCollection<Subscription> GetLiveSubscriptions(IEnumerable<Subscription> subscriptions)
+        => subscriptions.Where(subscription => !subscription.IsEnded()).ToList();
+
+    internal IReadOnlyCollection<SubscriptionItem> GetBillingSubscriptionItems(Subscription subscription)
+    {
+        var paidPlanIds = _plans.Plans
+            .Where(plan => plan.Price > 0)
+            .Select(plan => plan.Id)
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+        return GetBillingSubscriptionItems(subscription, paidPlanIds);
+    }
+
+    internal BillingPlan? GetSingleBillingPlan(Subscription subscription)
+    {
+        if (subscription.Items?.HasMore is true)
+            return null;
+
+        var billingItems = GetBillingSubscriptionItems(subscription);
+        if (billingItems.Count is not 1 || billingItems.Single().Quantity is not 1)
+            return null;
+
+        var billingItem = billingItems.Single();
+        return GetBillingPlan(billingItem.Price.Id);
+    }
+
+    internal bool IsFreePlan(Organization organization)
+        => String.Equals(organization.PlanId, _plans.FreePlan.Id, StringComparison.OrdinalIgnoreCase);
+
+    internal bool IsNonbillablePlan(Organization organization)
+        => GetBillingPlan(organization.PlanId) is { Price: 0 };
+
     public void ApplyBillingPlan(Organization organization, BillingPlan plan, User? user = null, bool updateBillingPrice = true)
     {
         organization.PlanId = plan.Id;
@@ -116,10 +250,49 @@ public void ApplyBillingPlan(Organization organization, BillingPlan plan, User?
         organization.GetCurrentUsage(_timeProvider).Limit = organization.GetMaxEventsPerMonthWithBonus(_timeProvider);
     }
 
+    internal bool IsBillingPlanApplied(
+        Organization organization,
+        BillingPlan plan)
+        => String.Equals(
+                organization.PlanId,
+                plan.Id,
+                StringComparison.OrdinalIgnoreCase)
+            && String.Equals(
+                organization.PlanName,
+                plan.Name,
+                StringComparison.Ordinal)
+            && String.Equals(
+                organization.PlanDescription,
+                plan.Description,
+                StringComparison.Ordinal)
+            && organization.BillingPrice == plan.Price
+            && organization.MaxUsers == plan.MaxUsers
+            && organization.MaxProjects == plan.MaxProjects
+            && organization.RetentionDays == plan.RetentionDays
+            && organization.MaxEventsPerMonth == plan.MaxEventsPerMonth
+            && organization.HasPremiumFeatures == plan.HasPremiumFeatures;
+
     public void ApplyBonus(Organization organization, int bonusEvents, DateTime? expires = null)
     {
         organization.BonusEventsPerMonth = bonusEvents;
         organization.BonusExpiration = expires;
         organization.GetCurrentUsage(_timeProvider).Limit = organization.GetMaxEventsPerMonthWithBonus(_timeProvider);
     }
+
+    private static int GetSubscriptionPriority(Subscription subscription)
+        => subscription.GetBillingStatus() switch
+        {
+            BillingStatus.Active or BillingStatus.Trialing => 3,
+            BillingStatus.PastDue => 2,
+            BillingStatus.Unpaid => 1,
+            _ => 0
+        };
+
+    private static IReadOnlyCollection<SubscriptionItem> GetBillingSubscriptionItems(
+        Subscription subscription,
+        IReadOnlySet<string> paidPlanIds)
+        => (subscription.Items?.Data ?? [])
+            .Where(item => item.Price?.Id is not null && paidPlanIds.Contains(item.Price.Id))
+            .ToList();
+
 }