Skip to content

Commit 8023d6d

Browse files
wkl3nkwkl3nk
committed
feat(scanner): Store VCS plugin config in NestedProvenanceStorage
If particular VCS plugin configurations are active during a scan (like `submoduleFetchStrategy=TOP_LEVEL_ONLY`), ensure that VCS plugin configurations are stored alongside nested provenance data. This prevents reuse of cache entries across scans with differing VCS plugin settings, ensuring correctness and reliability of scan results. Signed-off-by: Wolfgang Klenk <[email protected]>
1 parent bb19d45 commit 8023d6d

File tree

6 files changed

+83
-5
lines changed

6 files changed

+83
-5
lines changed

‎dao/src/main/kotlin/tables/NestedProvenancesTable.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,11 @@ object NestedProvenancesTable : LongIdTable("nested_provenances") {
3535

3636
val rootResolvedRevision = text("root_resolved_revision")
3737
val hasOnlyFixedRevisions = bool("has_only_fixed_revisions")
38+
39+
// If specific VCS plugin configurations are used, store a canonical string representation of these configuration
40+
// options in this column. This ensures that results are only reused for scans with identical VCS plugin
41+
// configurations.
42+
val vcsPluginConfigs = text("vcs_plugin_configs").nullable()
3843
}
3944

4045
class NestedProvenanceDao(id: EntityID<Long>) : LongEntity(id) {
@@ -44,6 +49,7 @@ class NestedProvenanceDao(id: EntityID<Long>) : LongEntity(id) {
4449

4550
var rootResolvedRevision by NestedProvenancesTable.rootResolvedRevision
4651
var hasOnlyFixedRevisions by NestedProvenancesTable.hasOnlyFixedRevisions
52+
var vcsPluginConfigs by NestedProvenancesTable.vcsPluginConfigs
4753

4854
val packageProvenances by PackageProvenanceDao optionalReferrersOn PackageProvenancesTable.nestedProvenanceId
4955
val subRepositories by NestedProvenanceSubRepositoryDao referrersOn

‎dao/src/main/resources/db/migration/V105__addNestedProvenancesConfiguration.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
ALTER TABLE nested_provenances
2+
ADD COLUMN vcs_plugin_configs text DEFAULT NULL;

‎workers/scanner/src/main/kotlin/scanner/OrtServerNestedProvenanceStorage.kt‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ import org.ossreviewtoolkit.utils.ort.runBlocking
4242

4343
class OrtServerNestedProvenanceStorage(
4444
private val db: Database,
45-
private val packageProvenanceCache: PackageProvenanceCache
45+
private val packageProvenanceCache: PackageProvenanceCache,
46+
private val vcsPluginConfigs: String?
4647
) : NestedProvenanceStorage {
4748
override fun writeNestedProvenance(
4849
root: RepositoryProvenance,
@@ -64,6 +65,7 @@ class OrtServerNestedProvenanceStorage(
6465
rootVcs = vcsDao
6566
rootResolvedRevision = root.resolvedRevision
6667
hasOnlyFixedRevisions = result.hasOnlyFixedRevisions
68+
vcsPluginConfigs = this@OrtServerNestedProvenanceStorage.vcsPluginConfigs
6769
}
6870

6971
result.nestedProvenance.subRepositories.forEach { (path, repositoryProvenance) ->
@@ -89,7 +91,11 @@ class OrtServerNestedProvenanceStorage(
8991
.where {
9092
VcsInfoTable.type eq resolvedVcs.type.name and
9193
(VcsInfoTable.url eq resolvedVcs.url) and
92-
(VcsInfoTable.revision eq resolvedVcs.revision)
94+
(VcsInfoTable.revision eq resolvedVcs.revision) and
95+
(
96+
NestedProvenancesTable.vcsPluginConfigs eq
97+
this@OrtServerNestedProvenanceStorage.vcsPluginConfigs
98+
)
9399
}.orderBy(NestedProvenancesTable.id to SortOrder.DESC)
94100
.limit(1)
95101
.singleOrNull()

‎workers/scanner/src/main/kotlin/scanner/ScannerRunner.kt‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.eclipse.apoapsis.ortserver.workers.scanner
2121

2222
import org.eclipse.apoapsis.ortserver.model.ScannerJobConfiguration
23+
import org.eclipse.apoapsis.ortserver.model.SubmoduleFetchStrategy
2324
import org.eclipse.apoapsis.ortserver.workers.common.OrtServerFileListStorage
2425
import org.eclipse.apoapsis.ortserver.workers.common.context.WorkerContext
2526
import org.eclipse.apoapsis.ortserver.workers.common.mapToOrt
@@ -33,6 +34,7 @@ import org.ossreviewtoolkit.model.PackageType
3334
import org.ossreviewtoolkit.model.Provenance
3435
import org.ossreviewtoolkit.model.ScannerRun
3536
import org.ossreviewtoolkit.model.SourceCodeOrigin
37+
import org.ossreviewtoolkit.model.VcsType
3638
import org.ossreviewtoolkit.model.config.DownloaderConfiguration
3739
import org.ossreviewtoolkit.model.config.ScannerConfiguration
3840
import org.ossreviewtoolkit.model.utils.FileArchiver
@@ -50,6 +52,18 @@ class ScannerRunner(
5052
private val fileArchiver: FileArchiver,
5153
private val fileListStorage: OrtServerFileListStorage
5254
) {
55+
companion object {
56+
/**
57+
* Convert the VCS plugin configurations to a canonical string representation. If there are no VCS plugin
58+
* configurations, return null.
59+
*/
60+
fun createCanonicalVcsPluginConfigs(vcsPluginConfigs: Map<String, PluginConfig>) =
61+
vcsPluginConfigs.keys.sorted().joinToString(separator = "&") { vcs ->
62+
vcsPluginConfigs[vcs]?.options.orEmpty()
63+
.toSortedMap().entries.joinToString(separator = "&") { (key, value) -> "$vcs/$key/$value" }
64+
}.ifEmpty { null }
65+
}
66+
5367
suspend fun run(
5468
context: WorkerContext,
5569
ortResult: OrtResult,
@@ -73,7 +87,11 @@ class ScannerRunner(
7387

7488
val packageProvenanceCache = PackageProvenanceCache()
7589
val packageProvenanceStorage = OrtServerPackageProvenanceStorage(db, scannerRunId, packageProvenanceCache)
76-
val nestedProvenanceStorage = OrtServerNestedProvenanceStorage(db, packageProvenanceCache)
90+
val nestedProvenanceStorage = OrtServerNestedProvenanceStorage(
91+
db,
92+
packageProvenanceCache,
93+
canonicalVcsPluginConfigs
94+
)
7795
val scanResultStorage = OrtServerScanResultStorage(db, scannerRunId)
7896

7997
val scanStorages = ScanStorages(

‎workers/scanner/src/test/kotlin/OrtServerNestedProvenanceStorageTest.kt‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,11 @@ class OrtServerNestedProvenanceStorageTest : WordSpec() {
6464
packageProvenanceCache = PackageProvenanceCache()
6565
packageProvenanceStorage =
6666
OrtServerPackageProvenanceStorage(dbExtension.db, scannerRun.id, packageProvenanceCache)
67-
nestedProvenanceStorage = OrtServerNestedProvenanceStorage(dbExtension.db, packageProvenanceCache)
67+
nestedProvenanceStorage = OrtServerNestedProvenanceStorage(
68+
dbExtension.db,
69+
packageProvenanceCache,
70+
""
71+
)
6872

6973
packageProvenanceStorage.writeProvenance(id, vcsInfo, packageProvenance)
7074
}
@@ -209,7 +213,11 @@ class OrtServerNestedProvenanceStorageTest : WordSpec() {
209213
packageProvenanceCache = PackageProvenanceCache()
210214
packageProvenanceStorage =
211215
OrtServerPackageProvenanceStorage(dbExtension.db, scannerRun.id, packageProvenanceCache)
212-
nestedProvenanceStorage = OrtServerNestedProvenanceStorage(dbExtension.db, packageProvenanceCache)
216+
nestedProvenanceStorage = OrtServerNestedProvenanceStorage(
217+
dbExtension.db,
218+
packageProvenanceCache,
219+
""
220+
)
213221

214222
val subInfo1 = vcsInfo.copy(path = "sub1")
215223
val subProvenance1 = RepositoryProvenance(subInfo1, subInfo1.revision)

‎workers/scanner/src/test/kotlin/ScannerRunnerTest.kt‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import org.ossreviewtoolkit.model.OrtResult
4444
import org.ossreviewtoolkit.model.Provenance
4545
import org.ossreviewtoolkit.model.RepositoryProvenance
4646
import org.ossreviewtoolkit.model.config.ScannerConfiguration
47+
import org.ossreviewtoolkit.plugins.api.PluginConfig as OrtPluginConfig
4748
import org.ossreviewtoolkit.scanner.LocalPathScannerWrapper
4849
import org.ossreviewtoolkit.scanner.Scanner
4950
import org.ossreviewtoolkit.scanner.ScannerWrapperFactory
@@ -184,6 +185,43 @@ class ScannerRunnerTest : WordSpec({
184185
result.issues shouldBe issuesMap
185186
}
186187
}
188+
189+
"createCanonicalVcsPluginConfigs" should {
190+
"return null if no VCS config plugins are used at all." {
191+
val vcsPluginConfigs = emptyMap<String, OrtPluginConfig>()
192+
193+
val result = ScannerRunner.createCanonicalVcsPluginConfigs(vcsPluginConfigs)
194+
195+
result shouldBe null
196+
}
197+
198+
"return a canonical string of VCS plugin configs." {
199+
val vcsPluginConfigs = mapOf(
200+
"VCS-Z" to OrtPluginConfig(
201+
options = mapOf(
202+
"option-z" to "1",
203+
"option-a" to "2"
204+
),
205+
secrets = mapOf(
206+
"some-secret" to "my-secret"
207+
)
208+
),
209+
"VCS-A" to OrtPluginConfig(
210+
options = mapOf(
211+
"option-x" to "3",
212+
"option-b" to "4"
213+
),
214+
secrets = mapOf(
215+
"some-secret" to "my-secret"
216+
)
217+
)
218+
)
219+
220+
val result = ScannerRunner.createCanonicalVcsPluginConfigs(vcsPluginConfigs)
221+
222+
result shouldBe "VCS-A/option-b/4&VCS-A/option-x/3&VCS-Z/option-a/2&VCS-Z/option-z/1"
223+
}
224+
}
187225
})
188226

189227
private fun mockScannerWrapperFactory(scannerName: String) =

0 commit comments

Comments
 (0)