Skip to content

Commit 6c86428

Browse files
KelvinTegelaarKelvinTegelaar
committed
oom management
1 parent e106d4d commit 6c86428

1 file changed

Lines changed: 69 additions & 34 deletions

File tree

‎Modules/CIPPDB/Public/DBCache/Set-CIPPDBCacheSharePointSharingLinks.ps1‎

Lines changed: 69 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -103,28 +103,40 @@ function Set-CIPPDBCacheSharePointSharingLinks {
103103
Write-Host "[SharingLinks][$TenantFilter] Phase 2 (site drives): $($DriveEntries.Count) drives, $FailedDriveLookups site drive lookups failed (+$([math]::Round(((Get-Date) - $StartTime).TotalSeconds,1))s)"
104104

105105
# 3) Delta-scan every drive and keep items that carry the "shared" facet.
106-
$DriveByRequestId = @{}
107-
$RequestId = 0
108-
$DeltaRequests = foreach ($Entry in $DriveEntries) {
109-
$DriveByRequestId["$RequestId"] = $Entry
110-
@{
111-
id = "$RequestId"
112-
method = 'GET'
113-
url = "drives/$($Entry.Drive.id)/root/delta?`$select=id,name,webUrl,parentReference,file,folder,shared,size,lastModifiedDateTime&`$top=999"
114-
}
115-
$RequestId++
116-
}
117-
106+
# Enumerating every driveItem across hundreds of drives in a single bulk call buffers
107+
# the whole tenant's file tree in memory and OOMs the worker on large tenants. Scan a
108+
# small number of drives per bulk call instead, extract only shared items, and release
109+
# the raw item pages between batches to keep peak memory bounded.
110+
$DeltaBatchSize = 20
118111
$SharedItems = [System.Collections.Generic.List[object]]::new()
119112
$FailedDrives = 0
120-
if (@($DeltaRequests).Count -gt 0) {
121-
Write-Host "[SharingLinks][$TenantFilter] Phase 3 (delta scan): scanning $(@($DeltaRequests).Count) drives for shared items"
113+
$DriveTotal = $DriveEntries.Count
114+
Write-Host "[SharingLinks][$TenantFilter] Phase 3 (delta scan): scanning $DriveTotal drives for shared items in batches of $DeltaBatchSize"
115+
for ($Offset = 0; $Offset -lt $DriveTotal; $Offset += $DeltaBatchSize) {
116+
$BatchEntries = @($DriveEntries[$Offset..([Math]::Min($Offset + $DeltaBatchSize - 1, $DriveTotal - 1))])
117+
$DriveByRequestId = @{}
118+
$RequestId = 0
119+
$DeltaRequests = foreach ($Entry in $BatchEntries) {
120+
$DriveByRequestId["$RequestId"] = $Entry
121+
@{
122+
id = "$RequestId"
123+
method = 'GET'
124+
url = "drives/$($Entry.Drive.id)/root/delta?`$select=id,name,webUrl,folder,shared,size,lastModifiedDateTime&`$top=999"
125+
}
126+
$RequestId++
127+
}
128+
122129
try {
123130
$DeltaResponses = New-GraphBulkRequest -tenantid $TenantFilter -Requests @($DeltaRequests) -asapp $true
124131
} catch {
125-
Write-Host "[SharingLinks][$TenantFilter] Phase 3 (delta scan) BULK FAILED after $([math]::Round(((Get-Date) - $StartTime).TotalSeconds,1))s: $($_.Exception.Message)"
126-
throw
132+
# A whole-batch failure (throttle/OOM/token) loses only this batch's drives, not the run.
133+
$FailedDrives += @($BatchEntries).Count
134+
Write-Host "[SharingLinks][$TenantFilter] Phase 3: batch at offset $Offset ($(@($BatchEntries).Count) drives) BULK FAILED: $($_.Exception.Message)"
135+
$DeltaResponses = $null
136+
[System.GC]::Collect()
137+
continue
127138
}
139+
128140
foreach ($Response in $DeltaResponses) {
129141
if ($Response.status -and $Response.status -ne 200) {
130142
$FailedDrives++
@@ -139,33 +151,52 @@ function Set-CIPPDBCacheSharePointSharingLinks {
139151
}
140152
}
141153
}
142-
}
143-
Write-Host "[SharingLinks][$TenantFilter] Phase 3 (delta scan): $($SharedItems.Count) shared items, $FailedDrives drive scans failed (+$([math]::Round(((Get-Date) - $StartTime).TotalSeconds,1))s)"
144154

145-
# 4) Fetch the permissions of every shared item, in bulk.
146-
$SharedItemByRequestId = @{}
147-
$RequestId = 0
148-
$PermissionRequests = foreach ($Entry in $SharedItems) {
149-
$SharedItemByRequestId["$RequestId"] = $Entry
150-
@{
151-
id = "$RequestId"
152-
method = 'GET'
153-
url = "drives/$($Entry.Drive.id)/items/$($Entry.Item.id)/permissions"
154-
}
155-
$RequestId++
155+
# Release this batch's raw item pages before the next batch.
156+
$DeltaResponses = $null
157+
[System.GC]::Collect()
158+
Write-Host "[SharingLinks][$TenantFilter] Phase 3: processed $([Math]::Min($Offset + $DeltaBatchSize, $DriveTotal))/$DriveTotal drives, $($SharedItems.Count) shared so far (+$([math]::Round(((Get-Date) - $StartTime).TotalSeconds,1))s)"
156159
}
160+
# If every drive failed, don't proceed to overwrite good cached data with an empty set.
161+
if ($DriveTotal -gt 0 -and $FailedDrives -ge $DriveTotal) {
162+
throw "All $DriveTotal drive delta scans failed - aborting to preserve existing cache."
163+
}
164+
Write-Host "[SharingLinks][$TenantFilter] Phase 3 (delta scan): $($SharedItems.Count) shared items, $FailedDrives drive scans failed (+$([math]::Round(((Get-Date) - $StartTime).TotalSeconds,1))s)"
157165

158-
# 5) Build one row per sharing link (any scope) or direct grant to an external user.
166+
# 4) Fetch the permissions of every shared item and 5) build one row per sharing link (any
167+
# scope) or direct grant to an external user. Batched for the same memory reason as the
168+
# delta scan above.
169+
$PermissionBatchSize = 100
159170
$Rows = [System.Collections.Generic.List[object]]::new()
160171
$FailedPermissionLookups = 0
161-
if (@($PermissionRequests).Count -gt 0) {
162-
Write-Host "[SharingLinks][$TenantFilter] Phase 4 (permissions): fetching permissions for $(@($PermissionRequests).Count) shared items"
172+
$SharedTotal = $SharedItems.Count
173+
if ($SharedTotal -gt 0) {
174+
Write-Host "[SharingLinks][$TenantFilter] Phase 4 (permissions): fetching permissions for $SharedTotal shared items in batches of $PermissionBatchSize"
175+
}
176+
for ($Offset = 0; $Offset -lt $SharedTotal; $Offset += $PermissionBatchSize) {
177+
$BatchShared = @($SharedItems[$Offset..([Math]::Min($Offset + $PermissionBatchSize - 1, $SharedTotal - 1))])
178+
$SharedItemByRequestId = @{}
179+
$RequestId = 0
180+
$PermissionRequests = foreach ($Entry in $BatchShared) {
181+
$SharedItemByRequestId["$RequestId"] = $Entry
182+
@{
183+
id = "$RequestId"
184+
method = 'GET'
185+
url = "drives/$($Entry.Drive.id)/items/$($Entry.Item.id)/permissions"
186+
}
187+
$RequestId++
188+
}
189+
163190
try {
164191
$PermissionResponses = New-GraphBulkRequest -tenantid $TenantFilter -Requests @($PermissionRequests) -asapp $true
165192
} catch {
166-
Write-Host "[SharingLinks][$TenantFilter] Phase 4 (permissions) BULK FAILED after $([math]::Round(((Get-Date) - $StartTime).TotalSeconds,1))s: $($_.Exception.Message)"
167-
throw
193+
$FailedPermissionLookups += @($BatchShared).Count
194+
Write-Host "[SharingLinks][$TenantFilter] Phase 4: batch at offset $Offset ($(@($BatchShared).Count) items) BULK FAILED: $($_.Exception.Message)"
195+
$PermissionResponses = $null
196+
[System.GC]::Collect()
197+
continue
168198
}
199+
169200
foreach ($Response in $PermissionResponses) {
170201
if ($Response.status -and $Response.status -ne 200) { $FailedPermissionLookups++; continue }
171202
$Entry = $SharedItemByRequestId["$($Response.id)"]
@@ -237,6 +268,10 @@ function Set-CIPPDBCacheSharePointSharingLinks {
237268
})
238269
}
239270
}
271+
272+
$PermissionResponses = $null
273+
[System.GC]::Collect()
274+
Write-Host "[SharingLinks][$TenantFilter] Phase 4: processed $([Math]::Min($Offset + $PermissionBatchSize, $SharedTotal))/$SharedTotal shared items, $($Rows.Count) links so far (+$([math]::Round(((Get-Date) - $StartTime).TotalSeconds,1))s)"
240275
}
241276

242277
Write-Host "[SharingLinks][$TenantFilter] Phase 5 (write): writing $($Rows.Count) rows to cache ($FailedPermissionLookups permission lookups failed)"

0 commit comments

Comments
 (0)