feat(alerts): auto-resolve cleared alerts

kris6673 · kris6673 · commit 4139ef59954b · 2026-07-16T14:28:06.000+02:00
Fired alerts were written to AlertLastRun but nothing ever removed
them when the underlying condition cleared, so dashboard alerts
stayed active forever. The only workaround was snoozing, which is
the wrong semantic for a resolved condition.

Alert output can't signal "cleared" because it only carries changed
data (it drives notifications, which must not repeat for unchanged
conditions - alerts can recur every 30 minutes). Instead
Write-AlertTrace stamps LastSeen on every run with active items,
and the scheduler clears the stored state when a successful run
stamped nothing.

Clear-AlertTrace only deletes rows carrying LogData: CheckExtension
keeps its last-run watermark in the same table under the same
RowKey format and must survive.

Known trade-off: an alert cmdlet that swallows a dependency failure
internally reads as clean and clears; the alert returns with a
fresh notification on the next successful run.
diff --git a/Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/Push-ExecScheduledCommand.ps1 b/Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/Push-ExecScheduledCommand.ps1
@@ -182,6 +182,7 @@ function Push-ExecScheduledCommand {
     }
 
     $Function = $Command
+    $CommandStartTime = [int64](([datetime]::UtcNow) - (Get-Date '1/1/1970')).TotalSeconds
 
     try {
         $PossibleParams = $Function.Parameters.Keys
@@ -284,6 +285,24 @@ function Push-ExecScheduledCommand {
         Write-Information "Results: $($results | ConvertTo-Json -Depth 10)"
         if ($item.command -like 'Get-CIPPAlert*') {
             Write-Information 'This is an alert task. Processing results as alerts.'
+            if ($State -ne 'Failed') {
+                # Output only carries CHANGED data (unchanged conditions stay silent so
+                # notifications don't repeat), so it can't signal "condition cleared".
+                # Write-AlertTrace instead stamps LastSeen on every run with active items;
+                # no trace row stamped during this run means the condition no longer holds
+                # (or every item is snoozed) - clear the stored state so the alert resolves
+                # on the dashboard instead of lingering forever. Use the resolved command
+                # name: rows are keyed by the function's defined name and table filters
+                # are case-sensitive.
+                $AlertTable = Get-CIPPTable -tablename 'AlertLastRun'
+                $TraceRows = Get-CIPPAzDataTableEntity @AlertTable -Filter "RowKey eq '$Tenant-$($Command.Name)'"
+                $StillActive = @($TraceRows | Where-Object {
+                        -not [string]::IsNullOrWhiteSpace($_.LogData) -and [int64]($_.LastSeen ?? 0) -ge $CommandStartTime
+                    })
+                if ($StillActive.Count -eq 0) {
+                    Clear-AlertTrace -CmdletName $Command.Name -TenantFilter $Tenant
+                }
+            }
             $results = @($results)
             $TaskType = 'Alert'
         } else {
diff --git a/Modules/CIPPCore/Public/GraphHelper/Clear-AlertTrace.ps1 b/Modules/CIPPCore/Public/GraphHelper/Clear-AlertTrace.ps1
@@ -0,0 +1,21 @@
+function Clear-AlertTrace {
+    <#
+    .FUNCTIONALITY
+    Internal function. Removes the stored AlertLastRun state for an alert so a resolved alert stops showing as active.
+    #>
+    param(
+        $CmdletName,
+        $TenantFilter
+    )
+    $Table = Get-CIPPTable -tablename AlertLastRun
+    # Delete every partition for this alert, not just today's: date-partitioned rows from
+    # earlier runs and the fixed partitions (BreachAlert, SecureScore) all key on this RowKey.
+    # Only rows carrying LogData are trace rows - Get-CIPPAlertCheckExtension keeps its
+    # last-run watermark in this table under the same RowKey format, and that must survive.
+    $Rows = Get-CIPPAzDataTableEntity @Table -Filter "RowKey eq '$($TenantFilter)-$($CmdletName)'"
+    $TraceRows = @($Rows | Where-Object { -not [string]::IsNullOrWhiteSpace($_.LogData) })
+    if ($TraceRows.Count -gt 0) {
+        Remove-AzDataTableEntity -Force @Table -Entity $TraceRows | Out-Null
+        Write-Information "Cleared AlertLastRun state for cmdlet '$CmdletName' and tenant '$TenantFilter' ($($TraceRows.Count) row(s))."
+    }
+}
diff --git a/Modules/CIPPCore/Public/GraphHelper/Write-AlertTrace.ps1 b/Modules/CIPPCore/Public/GraphHelper/Write-AlertTrace.ps1
@@ -18,39 +18,36 @@ function Write-AlertTrace {
     }
 
     $Table = Get-CIPPTable -tablename AlertLastRun
-    #Get current row and compare the $logData object. If it's the same, don't write it.
     $Row = Get-CIPPAzDataTableEntity @table -Filter "RowKey eq '$($tenantFilter)-$($cmdletName)' and PartitionKey eq '$PartitionKey'"
-    try {
-        $RowData = $Row.LogData
-        $Compare = Compare-Object $RowData (ConvertTo-Json -InputObject $data -Compress -Depth 10 | Out-String)
-        Write-Host "Comparing new alert data with existing data for cmdlet '$cmdletName' and tenant '$tenantFilter'. Differences: $Compare"
-        if ($Compare) {
-            $LogData = ConvertTo-Json -InputObject $data -Compress -Depth 10 | Out-String
-            $TableRow = @{
-                'PartitionKey' = $PartitionKey
-                'RowKey'       = "$($tenantFilter)-$($cmdletName)"
-                'CmdletName'   = "$cmdletName"
-                'Tenant'       = "$tenantFilter"
-                'LogData'      = [string]$LogData
-                'AlertComment' = [string]$AlertComment
-            }
-            $Table.Entity = $TableRow
-            Add-CIPPAzDataTableEntity @Table -Force | Out-Null
-            return $data
+    $LogData = ConvertTo-Json -InputObject $data -Compress -Depth 10 | Out-String
+    $Changed = $true
+    if ($Row) {
+        try {
+            $Changed = [bool](Compare-Object $Row.LogData $LogData)
+        } catch {
+            $Changed = $true
         }
-    } catch {
-        $LogData = ConvertTo-Json -InputObject $data -Compress -Depth 10 | Out-String
-        $TableRow = @{
-            'PartitionKey' = $PartitionKey
-            'RowKey'       = "$($tenantFilter)-$($cmdletName)"
-            'CmdletName'   = "$cmdletName"
-            'Tenant'       = "$tenantFilter"
-            'LogData'      = [string]$LogData
-            'AlertComment' = [string]$AlertComment
-        }
-        $Table.Entity = $TableRow
-        Add-CIPPAzDataTableEntity @Table -Force | Out-Null
-        return $data
+        Write-Host "Comparing new alert data with existing data for cmdlet '$cmdletName' and tenant '$tenantFilter'. Changed: $Changed"
+    }
+
+    # Written on every run with active items: LastSeen tells the scheduler the alert is
+    # still firing even when the data is unchanged, so it can clear resolved alerts.
+    $TableRow = @{
+        'PartitionKey' = $PartitionKey
+        'RowKey'       = "$($tenantFilter)-$($cmdletName)"
+        'CmdletName'   = "$cmdletName"
+        'Tenant'       = "$tenantFilter"
+        'LogData'      = [string]$LogData
+        'AlertComment' = [string]$AlertComment
+        'LastSeen'     = [string][int64](([datetime]::UtcNow) - (Get-Date '1/1/1970')).TotalSeconds
     }
+    $Table.Entity = $TableRow
+    Add-CIPPAzDataTableEntity @Table -Force | Out-Null
 
+    # Only changed data is returned: the caller's output drives notifications, and an
+    # unchanged persisting condition must not re-notify on every run (alerts can recur
+    # every 30 minutes, and the fixed-partition alerts keep one row across days).
+    if ($Changed) {
+        return $data
+    }
 }
diff --git a/Tests/Private/Clear-AlertTrace.Tests.ps1 b/Tests/Private/Clear-AlertTrace.Tests.ps1
@@ -0,0 +1,65 @@
+# Pester tests for Clear-AlertTrace
+
+BeforeAll {
+    # Resolve by name under Modules/ so the test survives the function moving between modules.
+    $RepoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $PSCommandPath))
+    $FunctionPath = Get-ChildItem -Path (Join-Path $RepoRoot 'Modules') -Recurse -Filter 'Clear-AlertTrace.ps1' -File -ErrorAction SilentlyContinue |
+        Select-Object -First 1 -ExpandProperty FullName
+    if (-not $FunctionPath) { throw 'Could not locate Clear-AlertTrace.ps1 under Modules/' }
+
+    # Stub every CIPP helper the function calls so Pester's Mock has a command to replace.
+    function Get-CIPPAzDataTableEntity { param($TableName, $Filter) }
+    function Get-CIPPTable { param($tablename) }
+    function Remove-AzDataTableEntity { param($TableName, $Entity, [switch]$Force) }
+
+    . $FunctionPath
+}
+
+Describe 'Clear-AlertTrace' {
+    BeforeEach {
+        Mock -CommandName Get-CIPPTable -MockWith { @{ TableName = 'AlertLastRun' } }
+        Mock -CommandName Get-CIPPAzDataTableEntity -MockWith { }
+        Mock -CommandName Remove-AzDataTableEntity -MockWith { }
+    }
+
+    It 'deletes every AlertLastRun trace row for the tenant+cmdlet RowKey' {
+        $Rows = @(
+            [pscustomobject]@{ PartitionKey = '20260715'; RowKey = 'contoso.com-Get-CIPPAlertApnCertExpiry'; LogData = '[{"Message":"expiring"}]' }
+            [pscustomobject]@{ PartitionKey = '20260716'; RowKey = 'contoso.com-Get-CIPPAlertApnCertExpiry'; LogData = '[{"Message":"expiring"}]' }
+        )
+        Mock -CommandName Get-CIPPAzDataTableEntity -MockWith { $Rows }
+
+        Clear-AlertTrace -CmdletName 'Get-CIPPAlertApnCertExpiry' -TenantFilter 'contoso.com'
+
+        Should -Invoke Get-CIPPAzDataTableEntity -Times 1 -Exactly -ParameterFilter {
+            $Filter -eq "RowKey eq 'contoso.com-Get-CIPPAlertApnCertExpiry'"
+        }
+        Should -Invoke Remove-AzDataTableEntity -Times 1 -Exactly -ParameterFilter {
+            @($Entity).Count -eq 2 -and $Force -eq $true
+        }
+    }
+
+    It 'spares rows without LogData, like the CheckExtension last-run watermark' {
+        $Rows = @(
+            [pscustomobject]@{ PartitionKey = 'AlertLastRun'; RowKey = 'contoso.com-Get-CIPPAlertCheckExtension'; LastRunTime = '2026-07-16T00:00:00Z' }
+            [pscustomobject]@{ PartitionKey = '20260716'; RowKey = 'contoso.com-Get-CIPPAlertCheckExtension'; LogData = '[{"Message":"phish"}]' }
+        )
+        Mock -CommandName Get-CIPPAzDataTableEntity -MockWith { $Rows }
+
+        Clear-AlertTrace -CmdletName 'Get-CIPPAlertCheckExtension' -TenantFilter 'contoso.com'
+
+        Should -Invoke Remove-AzDataTableEntity -Times 1 -Exactly -ParameterFilter {
+            @($Entity).Count -eq 1 -and @($Entity)[0].PartitionKey -eq '20260716'
+        }
+    }
+
+    It 'does not call Remove-AzDataTableEntity when only non-trace rows exist' {
+        Mock -CommandName Get-CIPPAzDataTableEntity -MockWith {
+            [pscustomobject]@{ PartitionKey = 'AlertLastRun'; RowKey = 'contoso.com-Get-CIPPAlertCheckExtension'; LastRunTime = '2026-07-16T00:00:00Z' }
+        }
+
+        Clear-AlertTrace -CmdletName 'Get-CIPPAlertCheckExtension' -TenantFilter 'contoso.com'
+
+        Should -Invoke Remove-AzDataTableEntity -Times 0 -Exactly
+    }
+}
diff --git a/Tests/Private/Write-AlertTrace.Tests.ps1 b/Tests/Private/Write-AlertTrace.Tests.ps1
@@ -0,0 +1,79 @@
+# Pester tests for Write-AlertTrace
+
+BeforeAll {
+    # Resolve by name under Modules/ so the test survives the function moving between modules.
+    $RepoRoot = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $PSCommandPath))
+    $FunctionPath = Get-ChildItem -Path (Join-Path $RepoRoot 'Modules') -Recurse -Filter 'Write-AlertTrace.ps1' -File -ErrorAction SilentlyContinue |
+        Select-Object -First 1 -ExpandProperty FullName
+    if (-not $FunctionPath) { throw 'Could not locate Write-AlertTrace.ps1 under Modules/' }
+
+    # Stub every CIPP helper the function calls so Pester's Mock has a command to replace.
+    function Add-CIPPAzDataTableEntity { param($TableName, $Entity, [switch]$Force) }
+    function Get-CIPPAzDataTableEntity { param($TableName, $Filter) }
+    function Get-CIPPTable { param($tablename) }
+    function Remove-SnoozedAlerts { param($Data, $CmdletName, $TenantFilter) }
+
+    . $FunctionPath
+}
+
+Describe 'Write-AlertTrace' {
+    BeforeEach {
+        Mock -CommandName Get-CIPPTable -MockWith { @{ TableName = 'AlertLastRun' } }
+        Mock -CommandName Get-CIPPAzDataTableEntity -MockWith { }
+        Mock -CommandName Add-CIPPAzDataTableEntity -MockWith { }
+        # Pass items through unfiltered (nothing snoozed) unless a test overrides this.
+        Mock -CommandName Remove-SnoozedAlerts -MockWith { $Data }
+
+        $script:AlertData = @([pscustomobject]@{ UserPrincipalName = 'admin@contoso.com'; Message = 'MFA disabled' })
+    }
+
+    It 'writes a new row and returns the data when no row exists for today' {
+        $Result = Write-AlertTrace -cmdletName 'Get-CIPPAlertMFAAdmins' -tenantFilter 'contoso.com' -data $script:AlertData
+
+        Should -Invoke Add-CIPPAzDataTableEntity -Times 1 -Exactly -ParameterFilter {
+            $Entity.RowKey -eq 'contoso.com-Get-CIPPAlertMFAAdmins' -and
+            $Entity.CmdletName -eq 'Get-CIPPAlertMFAAdmins' -and
+            -not [string]::IsNullOrWhiteSpace($Entity.LastSeen)
+        }
+        @($Result).Count | Should -Be 1
+        @($Result)[0].UserPrincipalName | Should -Be 'admin@contoso.com'
+    }
+
+    It 'writes a new row and returns the data when the stored data differs' {
+        $OldData = ConvertTo-Json -InputObject @([pscustomobject]@{ UserPrincipalName = 'other@contoso.com' }) -Compress -Depth 10 | Out-String
+        Mock -CommandName Get-CIPPAzDataTableEntity -MockWith {
+            [pscustomobject]@{ RowKey = 'contoso.com-Get-CIPPAlertMFAAdmins'; LogData = $OldData }
+        }
+
+        $Result = Write-AlertTrace -cmdletName 'Get-CIPPAlertMFAAdmins' -tenantFilter 'contoso.com' -data $script:AlertData
+
+        Should -Invoke Add-CIPPAzDataTableEntity -Times 1 -Exactly
+        @($Result)[0].UserPrincipalName | Should -Be 'admin@contoso.com'
+    }
+
+    It 'refreshes LastSeen but returns nothing when the stored data is unchanged' {
+        # Output drives notifications, so an unchanged persisting condition must stay
+        # silent; the LastSeen stamp is what tells the scheduler the alert still fires.
+        $StoredData = ConvertTo-Json -InputObject $script:AlertData -Compress -Depth 10 | Out-String
+        Mock -CommandName Get-CIPPAzDataTableEntity -MockWith {
+            [pscustomobject]@{ RowKey = 'contoso.com-Get-CIPPAlertMFAAdmins'; LogData = $StoredData }
+        }
+
+        $Result = Write-AlertTrace -cmdletName 'Get-CIPPAlertMFAAdmins' -tenantFilter 'contoso.com' -data $script:AlertData
+
+        Should -Invoke Add-CIPPAzDataTableEntity -Times 1 -Exactly -ParameterFilter {
+            -not [string]::IsNullOrWhiteSpace($Entity.LastSeen)
+        }
+        $Result | Should -BeNullOrEmpty
+    }
+
+    It 'returns null and writes nothing when every item is snoozed' {
+        Mock -CommandName Remove-SnoozedAlerts -MockWith { @() }
+
+        $Result = Write-AlertTrace -cmdletName 'Get-CIPPAlertMFAAdmins' -tenantFilter 'contoso.com' -data $script:AlertData
+
+        $Result | Should -BeNullOrEmpty
+        Should -Invoke Add-CIPPAzDataTableEntity -Times 0 -Exactly
+        Should -Invoke Get-CIPPAzDataTableEntity -Times 0 -Exactly
+    }
+}
