Skip to content

Commit 2a74519

Browse files
wkl3nkwkl3nk
committed
feat(scanner): Store VCS plugin config in NestedProvenanceStorage
If particular VCS plugin configurations are active during a scan (like `submoduleFetchStrategy=TOP_LEVEL_ONLY`), ensure that VCS plugin configurations are stored alongside nested provenance data. This prevents reuse of cache entries across scans with differing VCS plugin settings, ensuring correctness and reliability of scan results. However, there remains a small risk that other dependencies are not fully resolved in such a scenario where the WorkingTreeCache is limited to the first level of submodules. But this is rather small, because other open source dependencies typically don't use nested submodules at all. Signed-off-by: Wolfgang Klenk <[email protected]>
1 parent 950ab65 commit 2a74519

File tree

6 files changed

+82
-5
lines changed

6 files changed

+82
-5
lines changed

‎dao/src/main/kotlin/tables/NestedProvenancesTable.kt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,11 @@ object NestedProvenancesTable : LongIdTable("nested_provenances") {
3535

3636
val rootResolvedRevision = text("root_resolved_revision")
3737
val hasOnlyFixedRevisions = bool("has_only_fixed_revisions")
38+
39+
// If specific VCS plugin configurations are used, store a canonical string representation of these configuration
40+
// options in this column. This ensures that results are only reused for scans with identical VCS plugin
41+
// configurations.
42+
val vcsPluginConfigs = text("vcs_plugin_configs").nullable()
3843
}
3944

4045
class NestedProvenanceDao(id: EntityID<Long>) : LongEntity(id) {
@@ -44,6 +49,7 @@ class NestedProvenanceDao(id: EntityID<Long>) : LongEntity(id) {
4449

4550
var rootResolvedRevision by NestedProvenancesTable.rootResolvedRevision
4651
var hasOnlyFixedRevisions by NestedProvenancesTable.hasOnlyFixedRevisions
52+
var vcsPluginConfigs by NestedProvenancesTable.vcsPluginConfigs
4753

4854
val packageProvenances by PackageProvenanceDao optionalReferrersOn PackageProvenancesTable.nestedProvenanceId
4955
val subRepositories by NestedProvenanceSubRepositoryDao referrersOn

‎dao/src/main/resources/db/migration/V105__addNestedProvenancesConfiguration.sql

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
ALTER TABLE nested_provenances
2+
ADD COLUMN vcs_plugin_configs text DEFAULT NULL;

‎workers/scanner/src/main/kotlin/scanner/OrtServerNestedProvenanceStorage.kt

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ import org.ossreviewtoolkit.utils.ort.runBlocking
4242

4343
class OrtServerNestedProvenanceStorage(
4444
private val db: Database,
45-
private val packageProvenanceCache: PackageProvenanceCache
45+
private val packageProvenanceCache: PackageProvenanceCache,
46+
private val vcsPluginConfigs: String?
4647
) : NestedProvenanceStorage {
4748
override fun writeNestedProvenance(
4849
root: RepositoryProvenance,
@@ -64,6 +65,7 @@ class OrtServerNestedProvenanceStorage(
6465
rootVcs = vcsDao
6566
rootResolvedRevision = root.resolvedRevision
6667
hasOnlyFixedRevisions = result.hasOnlyFixedRevisions
68+
vcsPluginConfigs = this@OrtServerNestedProvenanceStorage.vcsPluginConfigs
6769
}
6870

6971
result.nestedProvenance.subRepositories.forEach { (path, repositoryProvenance) ->
@@ -89,7 +91,11 @@ class OrtServerNestedProvenanceStorage(
8991
.where {
9092
VcsInfoTable.type eq resolvedVcs.type.name and
9193
(VcsInfoTable.url eq resolvedVcs.url) and
92-
(VcsInfoTable.revision eq resolvedVcs.revision)
94+
(VcsInfoTable.revision eq resolvedVcs.revision) and
95+
(
96+
NestedProvenancesTable.vcsPluginConfigs eq
97+
this@OrtServerNestedProvenanceStorage.vcsPluginConfigs
98+
)
9399
}.orderBy(NestedProvenancesTable.id to SortOrder.DESC)
94100
.limit(1)
95101
.singleOrNull()

‎workers/scanner/src/main/kotlin/scanner/ScannerRunner.kt

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,18 @@ class ScannerRunner(
5252
private val fileArchiver: FileArchiver,
5353
private val fileListStorage: OrtServerFileListStorage
5454
) {
55+
companion object {
56+
/**
57+
* Convert the VCS plugin configurations to a canonical string representation. If there are no VCS plugin
58+
* configurations, return null.
59+
*/
60+
fun createCanonicalVcsPluginConfigs(vcsPluginConfigs: Map<String, PluginConfig>) =
61+
vcsPluginConfigs.keys.sorted().joinToString(separator = "&") { vcs ->
62+
vcsPluginConfigs[vcs]?.options.orEmpty()
63+
.toSortedMap().entries.joinToString(separator = "&") { (key, value) -> "$vcs/$key/$value" }
64+
}.ifEmpty { null }
65+
}
66+
5567
suspend fun run(
5668
context: WorkerContext,
5769
ortResult: OrtResult,
@@ -78,9 +90,14 @@ class ScannerRunner(
7890
)
7991
}
8092

93+
val canonicalVcsPluginConfigs = createCanonicalVcsPluginConfigs(vcsPluginConfigs)
8194
val packageProvenanceCache = PackageProvenanceCache()
8295
val packageProvenanceStorage = OrtServerPackageProvenanceStorage(db, scannerRunId, packageProvenanceCache)
83-
val nestedProvenanceStorage = OrtServerNestedProvenanceStorage(db, packageProvenanceCache)
96+
val nestedProvenanceStorage = OrtServerNestedProvenanceStorage(
97+
db,
98+
packageProvenanceCache,
99+
canonicalVcsPluginConfigs
100+
)
84101
val scanResultStorage = OrtServerScanResultStorage(db, scannerRunId)
85102

86103
val scanStorages = ScanStorages(

‎workers/scanner/src/test/kotlin/OrtServerNestedProvenanceStorageTest.kt

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,11 @@ class OrtServerNestedProvenanceStorageTest : WordSpec() {
6464
packageProvenanceCache = PackageProvenanceCache()
6565
packageProvenanceStorage =
6666
OrtServerPackageProvenanceStorage(dbExtension.db, scannerRun.id, packageProvenanceCache)
67-
nestedProvenanceStorage = OrtServerNestedProvenanceStorage(dbExtension.db, packageProvenanceCache)
67+
nestedProvenanceStorage = OrtServerNestedProvenanceStorage(
68+
dbExtension.db,
69+
packageProvenanceCache,
70+
""
71+
)
6872

6973
packageProvenanceStorage.writeProvenance(id, vcsInfo, packageProvenance)
7074
}
@@ -209,7 +213,11 @@ class OrtServerNestedProvenanceStorageTest : WordSpec() {
209213
packageProvenanceCache = PackageProvenanceCache()
210214
packageProvenanceStorage =
211215
OrtServerPackageProvenanceStorage(dbExtension.db, scannerRun.id, packageProvenanceCache)
212-
nestedProvenanceStorage = OrtServerNestedProvenanceStorage(dbExtension.db, packageProvenanceCache)
216+
nestedProvenanceStorage = OrtServerNestedProvenanceStorage(
217+
dbExtension.db,
218+
packageProvenanceCache,
219+
""
220+
)
213221

214222
val subInfo1 = vcsInfo.copy(path = "sub1")
215223
val subProvenance1 = RepositoryProvenance(subInfo1, subInfo1.revision)

‎workers/scanner/src/test/kotlin/ScannerRunnerTest.kt

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import org.ossreviewtoolkit.model.OrtResult
4444
import org.ossreviewtoolkit.model.Provenance
4545
import org.ossreviewtoolkit.model.RepositoryProvenance
4646
import org.ossreviewtoolkit.model.config.ScannerConfiguration
47+
import org.ossreviewtoolkit.plugins.api.PluginConfig as OrtPluginConfig
4748
import org.ossreviewtoolkit.scanner.LocalPathScannerWrapper
4849
import org.ossreviewtoolkit.scanner.Scanner
4950
import org.ossreviewtoolkit.scanner.ScannerWrapperFactory
@@ -184,6 +185,43 @@ class ScannerRunnerTest : WordSpec({
184185
result.issues shouldBe issuesMap
185186
}
186187
}
188+
189+
"createCanonicalVcsPluginConfigs" should {
190+
"return null if no VCS config plugins are used at all." {
191+
val vcsPluginConfigs = emptyMap<String, OrtPluginConfig>()
192+
193+
val result = ScannerRunner.createCanonicalVcsPluginConfigs(vcsPluginConfigs)
194+
195+
result shouldBe null
196+
}
197+
198+
"return a canonical string of VCS plugin configs." {
199+
val vcsPluginConfigs = mapOf(
200+
"VCS-Z" to OrtPluginConfig(
201+
options = mapOf(
202+
"option-z" to "1",
203+
"option-a" to "2"
204+
),
205+
secrets = mapOf(
206+
"some-secret" to "my-secret"
207+
)
208+
),
209+
"VCS-A" to OrtPluginConfig(
210+
options = mapOf(
211+
"option-x" to "3",
212+
"option-b" to "4"
213+
),
214+
secrets = mapOf(
215+
"some-secret" to "my-secret"
216+
)
217+
)
218+
)
219+
220+
val result = ScannerRunner.createCanonicalVcsPluginConfigs(vcsPluginConfigs)
221+
222+
result shouldBe "VCS-A/option-b/4&VCS-A/option-x/3&VCS-Z/option-a/2&VCS-Z/option-z/1"
223+
}
224+
}
187225
})
188226

189227
private fun mockScannerWrapperFactory(scannerName: String) =

0 commit comments

Comments
 (0)