fix(cippcore): harden cache cleanup memory use

Refactor reporting DB writes to keep memory bounded by batch size and switch orphan detection from a run-wide RowKey set to Timestamp-based evidence. This adds a skew margin option, skips undated rows instead of risking mass deletion, and only runs orphan cleanup after successful writes.

Also rework audit-log ingestion tenant discovery to stream directly into HashSets with minimal property projection, removing Group-Object and large intermediate arrays that were triggering OutOfMemoryException under backlog load.

Synced from CyberDrain/CIPP@453928c

Author: github-actions[bot]

Modules/CIPPCore/Public/Add-CIPPDbItem.ps1

@@ -21,13 +21,19 @@ function Add-CIPPDbItem {
 
         [switch]$Count,
         [switch]$AddCount,
-        [switch]$Append
+        [switch]$Append,
+
+        [ValidateRange(0, 60)]
+        [int]$SkewMarginMinutes = 5
     )
 
     begin {
         $Table = Get-CippTable -tablename 'CippReportingDB'
-        $Batch = [System.Collections.Generic.List[hashtable]]::new()
-        $NewRowKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
+        $BatchSize = 100
+        $Batch = [System.Collections.Generic.List[hashtable]]::new($BatchSize)
+        $SeenInBatch = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
+        $RunStartUtc = [DateTimeOffset]::UtcNow.AddMinutes(-$SkewMarginMinutes)
+
         $TotalProcessed = 0
         # Cache regex instances so each row pays only the match cost, not regex compilation.
         # Two passes preserve the original semantics: path/wildcard chars → '_', control chars → stripped.
@@ -54,17 +60,18 @@ function Add-CIPPDbItem {
             if ($null -eq $Item) { continue }
             $ItemId = $Item.ExternalDirectoryObjectId ?? $Item.id ?? $Item.Identity ?? $Item.skuId ?? $Item.userPrincipalName ?? [guid]::NewGuid().ToString()
             $RowKey = $RowKeyControlRegex.Replace($RowKeyPathRegex.Replace("$Type-$ItemId", '_'), '')
-            if ($NewRowKeys.Add($RowKey)) {
+            if ($SeenInBatch.Add($RowKey)) {
                 $Batch.Add(@{
                         PartitionKey = $TenantFilter
                         RowKey       = $RowKey
                         Data         = [string]($Item | ConvertTo-Json -Depth 10 -Compress)
                         Type         = $Type
                     })
-                if ($Batch.Count -ge 500) {
+                if ($Batch.Count -ge $BatchSize) {
                     $null = Add-CIPPAzDataTableEntity @Table -Entity $Batch.ToArray() -Force
                     $TotalProcessed += $Batch.Count
                     $Batch.Clear()
+                    $SeenInBatch.Clear()
                 }
             }
         }
@@ -76,17 +83,22 @@ function Add-CIPPDbItem {
             $TotalProcessed += $Batch.Count
         }
 
-        # Clean up orphaned rows (entities that no longer exist in the new dataset)
-        if (-not $Count.IsPresent -and -not $Append.IsPresent) {
+        # Clean up orphaned rows (entities that no longer exist in the new dataset).
+        if (-not $Count.IsPresent -and -not $Append.IsPresent -and $TotalProcessed -gt 0) {
             $Filter = "PartitionKey eq '{0}' and RowKey ge '{1}-' and RowKey lt '{1}0'" -f $TenantFilter, $Type
-            $Existing = Get-CIPPAzDataTableEntity @Table -Filter $Filter -Property PartitionKey, RowKey, ETag, OriginalEntityId
+            $Existing = Get-CIPPAzDataTableEntity @Table -Filter $Filter -Property PartitionKey, RowKey, ETag, OriginalEntityId, Timestamp
             if ($Existing) {
+                $Undated = 0
                 $Orphans = foreach ($Row in @($Existing)) {
                     if ($Row.RowKey -eq "$Type-Count") { continue }
-                    $ParentKey = $Row.OriginalEntityId ?? $Row.RowKey
-                    if (-not $NewRowKeys.Contains($ParentKey)) {
-                        $Row
-                    }
+
+                    $Stamp = $Row.Timestamp -as [datetimeoffset]
+                    if ($null -eq $Stamp) { $Undated++; continue }
+
+                    if ($Stamp -lt $RunStartUtc) { $Row }
+                }
+                if ($Undated -gt 0) {
+                    Write-LogMessage -API 'CIPPDbItem' -tenant $TenantFilter -sev Warning -message "Skipped $Undated $Type row(s) with no readable Timestamp during orphan cleanup — not deleting without positive evidence"
                 }
                 if ($Orphans) {
                     $null = Remove-AzDataTableEntity @Table -Entity @($Orphans) -Force

Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogIngestionV2.ps1

@@ -28,19 +28,17 @@ function Start-AuditLogIngestionV2 {
         $Now = (Get-Date).ToUniversalTime()
 
         # --- Download tenants: searches awaiting download (State = Created, due) ---
-        $Created = @(Get-CIPPAzDataTableEntity @Ledger -Filter "State eq 'Created'")
-        $DueCreated = $Created | Where-Object { -not $_.NextAttemptUtc -or ([datetimeoffset]$_.NextAttemptUtc).UtcDateTime -le $Now }
         $DownloadTenants = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
-        foreach ($Name in @($DueCreated | Group-Object PartitionKey | ForEach-Object { $_.Name })) {
-            if ($Name) { [void]$DownloadTenants.Add([string]$Name) }
+        foreach ($Row in @(Get-CIPPAzDataTableEntity @Ledger -Filter "State eq 'Created'" -Property @('PartitionKey', 'NextAttemptUtc'))) {
+            if ($Row.NextAttemptUtc -and ([datetimeoffset]$Row.NextAttemptUtc).UtcDateTime -gt $Now) { continue }
+            if ($Row.PartitionKey) { [void]$DownloadTenants.Add([string]$Row.PartitionKey) }
         }
 
         # --- Process-only tenants: rows pending in the webhook cache (downloaded, not yet processed) ---
         $CacheTable = Get-CippTable -TableName 'CacheWebhooks'
-        $CacheRows = @(Get-CIPPAzDataTableEntity @CacheTable -Property @('PartitionKey', 'RowKey'))
         $CacheTenants = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
-        foreach ($Name in @($CacheRows | Group-Object PartitionKey | ForEach-Object { $_.Name })) {
-            if ($Name) { [void]$CacheTenants.Add([string]$Name) }
+        foreach ($Row in @(Get-CIPPAzDataTableEntity @CacheTable -Property @('PartitionKey'))) {
+            if ($Row.PartitionKey) { [void]$CacheTenants.Add([string]$Row.PartitionKey) }
         }
 
         if ($DownloadTenants.Count -eq 0 -and $CacheTenants.Count -eq 0) {