Skip to content

Commit e67c030

Browse files
author
Kamil Bielecki
committed
feat(database): Remove orphaned scan findings from DB
During deletion of old / outdated ORT runs some of child database entities were left in DB. To prevent this situation, that leads to database performance issues, orphaned records are deleted. Deleted record types: - Scan results - Scan summaries - Snippet findings - Snippets - License findings - Copyright findings Signed-off-by: Kamil Bielecki <[email protected]>
1 parent e4d65e7 commit e67c030

File tree

4 files changed

+294
-13
lines changed

4 files changed

+294
-13
lines changed

‎dao/src/main/resources/db/migration/V98__orphanRemovalForFindings.sql

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright (C) 2025 The ORT Server Authors (See <https://github.com/eclipse-apoapsis/ort-server/blob/main/NOTICE>)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* SPDX-License-Identifier: Apache-2.0
17+
* License-Filename: LICENSE
18+
*/
19+
20+
-- This migration script adds cascade deletions on scan_results table and it's children tables.
21+
22+
ALTER TABLE snippet_findings
23+
DROP CONSTRAINT snippet_findings_scan_summary_id_fkey,
24+
ADD CONSTRAINT snippet_findings_scan_summary_id_fkey
25+
FOREIGN KEY (scan_summary_id) REFERENCES scan_summaries (id) ON DELETE CASCADE;
26+
27+
ALTER TABLE snippet_findings_snippets
28+
DROP CONSTRAINT snippet_findings_snippets_snippet_finding_id_fkey,
29+
ADD CONSTRAINT snippet_findings_snippets_snippet_finding_id_fkey
30+
FOREIGN KEY (snippet_finding_id) REFERENCES snippet_findings (id) ON DELETE CASCADE;
31+
32+
ALTER TABLE license_findings
33+
DROP CONSTRAINT license_findings_scan_summary_id_fkey,
34+
ADD CONSTRAINT license_findings_scan_summary_id_fkey
35+
FOREIGN KEY (scan_summary_id) REFERENCES scan_summaries (id) ON DELETE CASCADE;
36+
37+
ALTER TABLE copyright_findings
38+
DROP CONSTRAINT copyright_findings_scan_summary_id_fkey,
39+
ADD CONSTRAINT copyright_findings_scan_summary_id_fkey
40+
FOREIGN KEY (scan_summary_id) REFERENCES scan_summaries (id) ON DELETE CASCADE;

‎services/hierarchy/src/main/kotlin/OrphanRemovalService.kt

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,15 @@ import org.eclipse.apoapsis.ortserver.dao.repositories.repositoryconfiguration.P
3838
import org.eclipse.apoapsis.ortserver.dao.repositories.repositoryconfiguration.PackageCurationDataTable
3939
import org.eclipse.apoapsis.ortserver.dao.repositories.repositoryconfiguration.PackageCurationsTable
4040
import org.eclipse.apoapsis.ortserver.dao.repositories.repositoryconfiguration.PackageLicenseChoicesTable
41+
import org.eclipse.apoapsis.ortserver.dao.repositories.scannerrun.ScannerRunsScanResultsTable
4142
import org.eclipse.apoapsis.ortserver.dao.repositories.scannerrun.ScannerRunsScannersTable
4243
import org.eclipse.apoapsis.ortserver.dao.tables.NestedProvenanceSubRepositoriesTable
4344
import org.eclipse.apoapsis.ortserver.dao.tables.NestedProvenancesTable
4445
import org.eclipse.apoapsis.ortserver.dao.tables.NestedRepositoriesTable
4546
import org.eclipse.apoapsis.ortserver.dao.tables.PackageProvenancesTable
47+
import org.eclipse.apoapsis.ortserver.dao.tables.ScanResultsTable
48+
import org.eclipse.apoapsis.ortserver.dao.tables.ScanSummariesTable
49+
import org.eclipse.apoapsis.ortserver.dao.tables.SnippetFindingsSnippetsTable
4650
import org.eclipse.apoapsis.ortserver.dao.tables.SnippetsTable
4751
import org.eclipse.apoapsis.ortserver.dao.tables.shared.DeclaredLicensesTable
4852
import org.eclipse.apoapsis.ortserver.dao.tables.shared.IdentifiersIssuesTable
@@ -63,6 +67,7 @@ import org.slf4j.LoggerFactory
6367
/**
6468
* Maintenance service to remove orphaned entities.
6569
*/
70+
@Suppress("TooManyFunctions")
6671
class OrphanRemovalService(
6772
private val db: Database
6873
) {
@@ -82,6 +87,9 @@ class OrphanRemovalService(
8287
logger.info("Deleted {} records from {}", deleteOrphanedIdentifiers(), IdentifiersTable.tableName)
8388
logger.info("Deleted {} records from {}", deleteOrphanedVcsInfo(), VcsInfoTable.tableName)
8489
logger.info("Deleted {} records from {}", deleteOrphanedRemoteArtifacts(), RemoteArtifactsTable.tableName)
90+
logger.info("Deleted {} records from {}", deleteOrphanedScanResults(), ScanResultsTable.tableName)
91+
logger.info("Deleted {} records from {}", deleteOrphanedScanSummaries(), ScanSummariesTable.tableName)
92+
logger.info("Deleted {} records from {}", deleteOrphanedSnippets(), SnippetsTable.tableName)
8593

8694
logger.info("Deleting orphaned children of ORT runs finished.")
8795
}
@@ -302,4 +310,33 @@ class OrphanRemovalService(
302310
)
303311
}
304312
}
313+
314+
private suspend fun deleteOrphanedScanResults() =
315+
db.dbQuery {
316+
ScanResultsTable.deleteWhere {
317+
id notInSubQuery (
318+
ScannerRunsScanResultsTable.select(ScannerRunsScanResultsTable.scanResultId)
319+
)
320+
}
321+
}
322+
323+
private suspend fun deleteOrphanedScanSummaries() =
324+
db.dbQuery {
325+
ScanSummariesTable.deleteWhere {
326+
id notInSubQuery (
327+
ScanResultsTable.select(ScanResultsTable.scanSummaryId)
328+
)
329+
}
330+
}
331+
332+
private suspend fun deleteOrphanedSnippets() =
333+
db.dbQuery {
334+
SnippetsTable.deleteWhere {
335+
id notInSubQuery (
336+
SnippetFindingsSnippetsTable
337+
.select(SnippetFindingsSnippetsTable.snippetId.alias("id"))
338+
.where(SnippetFindingsSnippetsTable.snippetId.isNotNull())
339+
)
340+
}
341+
}
305342
}

‎services/hierarchy/src/test/kotlin/OrphanRemovalServiceTest.kt

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,13 @@ import org.eclipse.apoapsis.ortserver.dao.repositories.analyzerrun.PackagesTable
3030
import org.eclipse.apoapsis.ortserver.dao.repositories.analyzerrun.ProcessedDeclaredLicensesTable
3131
import org.eclipse.apoapsis.ortserver.dao.repositories.analyzerrun.ProjectsTable
3232
import org.eclipse.apoapsis.ortserver.dao.repositories.analyzerrun.UnmappedDeclaredLicensesTable
33+
import org.eclipse.apoapsis.ortserver.dao.tables.CopyrightFindingsTable
34+
import org.eclipse.apoapsis.ortserver.dao.tables.LicenseFindingsTable
35+
import org.eclipse.apoapsis.ortserver.dao.tables.ScanResultsTable
36+
import org.eclipse.apoapsis.ortserver.dao.tables.ScanSummariesTable
37+
import org.eclipse.apoapsis.ortserver.dao.tables.SnippetFindingsSnippetsTable
38+
import org.eclipse.apoapsis.ortserver.dao.tables.SnippetFindingsTable
39+
import org.eclipse.apoapsis.ortserver.dao.tables.SnippetsTable
3340
import org.eclipse.apoapsis.ortserver.dao.tables.shared.DeclaredLicensesTable
3441
import org.eclipse.apoapsis.ortserver.dao.tables.shared.IdentifiersTable
3542
import org.eclipse.apoapsis.ortserver.dao.tables.shared.RemoteArtifactsTable
@@ -38,9 +45,11 @@ import org.eclipse.apoapsis.ortserver.dao.test.DatabaseTestExtension
3845
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createAdvisorRunsIdentifiersTableEntry
3946
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createAnalyzerRunTableEntry
4047
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createAuthorsTableEntry
48+
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createCopyrightFindingsTableEntry
4149
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createDeclaredLicensesTableEntry
4250
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createIdentifierTableEntry
4351
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createIdentifiersIssuesTableEntry
52+
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createLicenseFindingsTableEntry
4453
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createMappedDeclaredLicenseTableEntry
4554
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createNestedRepositoriesTableEntry
4655
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createOrtRunTableEntry
@@ -65,7 +74,12 @@ import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.
6574
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createProjectsTableEntry
6675
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createRemoteArtifactsTableEntry
6776
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createRuleViolationsTableEntry
77+
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createScanResultsTableEntry
78+
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createScanSummaryTableEntry
79+
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createScannerRunsScanResultsTableEntry
6880
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createScannerRunsScannersTableEntry
81+
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createSnippetFindingsSnippetsTableEntry
82+
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createSnippetFindingsTableEntry
6983
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createSnippetsTableEntry
7084
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createUnmappedDeclaredLicenseTableEntry
7185
import org.eclipse.apoapsis.ortserver.services.OrphanRemovalServiceTestFixtures.createVcsInfoTableEntry
@@ -484,6 +498,105 @@ class OrphanRemovalServiceTest : WordSpec() {
484498
}
485499
}
486500
}
501+
502+
"delete findings, snippets and scan summaries not related to any other entity" {
503+
db.dbQuery {
504+
505+
// Create scan result not related to scan run.
506+
val scanSummaryId = createScanSummaryTableEntry(hash = "to.delete.1").value
507+
createScanResultsTableEntry(artifactUrl = "to.delete.1", scanSummaryId = scanSummaryId)
508+
createCopyrightFindingsTableEntry(scanSummaryId = scanSummaryId, path = "to.delete.2")
509+
createCopyrightFindingsTableEntry(scanSummaryId = scanSummaryId, path = "to.delete.3")
510+
createLicenseFindingsTableEntry(scanSummaryId = scanSummaryId, path = "to.delete.4")
511+
createLicenseFindingsTableEntry(scanSummaryId = scanSummaryId, path = "to.delete.5")
512+
513+
createSnippetFindingsSnippetsTableEntry(
514+
snippetFindingId = createSnippetFindingsTableEntry(
515+
scanSummaryId = scanSummaryId,
516+
path = "to.delete.6"
517+
).value,
518+
snippetId = createSnippetsTableEntry(purl = "to.delete.7").value
519+
)
520+
521+
createSnippetFindingsSnippetsTableEntry(
522+
snippetFindingId = createSnippetFindingsTableEntry(
523+
scanSummaryId = scanSummaryId,
524+
path = "to.delete.8"
525+
).value,
526+
snippetId = createSnippetsTableEntry(purl = "to.delete.9").value
527+
)
528+
529+
// Create scan result related to scan run.
530+
val scanSummary2Id = createScanSummaryTableEntry(hash = "not.to.delete.1").value
531+
createCopyrightFindingsTableEntry(scanSummaryId = scanSummary2Id, path = "not.to.delete.2")
532+
createCopyrightFindingsTableEntry(scanSummaryId = scanSummary2Id, path = "not.to.delete.3")
533+
createLicenseFindingsTableEntry(scanSummaryId = scanSummary2Id, path = "not.to.delete.4")
534+
createLicenseFindingsTableEntry(scanSummaryId = scanSummary2Id, path = "not.to.delete.5")
535+
536+
createSnippetFindingsSnippetsTableEntry(
537+
snippetFindingId = createSnippetFindingsTableEntry(
538+
scanSummaryId = scanSummary2Id,
539+
path = "not.to.delete.6"
540+
).value,
541+
snippetId = createSnippetsTableEntry(purl = "not.to.delete.7").value
542+
)
543+
544+
createSnippetFindingsSnippetsTableEntry(
545+
snippetFindingId = createSnippetFindingsTableEntry(
546+
scanSummaryId = scanSummary2Id,
547+
path = "not.to.delete.8"
548+
).value,
549+
snippetId = createSnippetsTableEntry(purl = "not.to.delete.9").value
550+
)
551+
552+
createScannerRunsScanResultsTableEntry(
553+
scannerRunId = createScannerRunsScannersTableEntry().value,
554+
scanResultId = createScanResultsTableEntry(
555+
artifactUrl = "not.to.delete.8", scanSummaryId = scanSummary2Id
556+
).value
557+
)
558+
}
559+
560+
db.dbQuery {
561+
ScanSummariesTable.selectAll().count() shouldBe 2
562+
ScanResultsTable.selectAll().count() shouldBe 2
563+
SnippetFindingsTable.selectAll().count() shouldBe 4
564+
LicenseFindingsTable.selectAll().count() shouldBe 4
565+
CopyrightFindingsTable.selectAll().count() shouldBe 4
566+
SnippetsTable.selectAll().count() shouldBe 4
567+
}
568+
569+
service.deleteRunsOrphanedEntities()
570+
571+
db.dbQuery {
572+
ScanSummariesTable.selectAll().count() shouldBe 1
573+
ScanSummariesTable.selectAll().forEach {
574+
it[ScanSummariesTable.hash] shouldStartWith "not.to.delete"
575+
}
576+
ScanResultsTable.selectAll().count() shouldBe 1
577+
ScanResultsTable.selectAll().forEach {
578+
it[ScanResultsTable.artifactUrl] shouldStartWith "not.to.delete"
579+
}
580+
SnippetFindingsSnippetsTable.selectAll().count() shouldBe 2
581+
582+
SnippetFindingsTable.selectAll().count() shouldBe 2
583+
SnippetFindingsTable.selectAll().forEach {
584+
it[SnippetFindingsTable.path] shouldStartWith "not.to.delete"
585+
}
586+
LicenseFindingsTable.selectAll().count() shouldBe 2
587+
LicenseFindingsTable.selectAll().forEach {
588+
it[LicenseFindingsTable.path] shouldStartWith "not.to.delete"
589+
}
590+
CopyrightFindingsTable.selectAll().count() shouldBe 2
591+
CopyrightFindingsTable.selectAll().forEach {
592+
it[CopyrightFindingsTable.path] shouldStartWith "not.to.delete"
593+
}
594+
SnippetsTable.selectAll().count() shouldBe 2
595+
SnippetsTable.selectAll().forEach {
596+
it[SnippetsTable.purl] shouldStartWith "not.to.delete"
597+
}
598+
}
599+
}
487600
}
488601
}
489602
}

0 commit comments

Comments
 (0)