feat(scanner): Add submodule fetch strategy for nested repositories

Introduce `submoduleFetchStrategy` config to control how the Scanner
fetches Git submodules. When set to `TOP_LEVEL_ONLY`, only top-level
submodules are fetched, avoiding timeouts on deeply nested repos.

This mirrors the behavior already available in the Analyzer and
allows to resolve nested provenances even in this kind of repositories
with a vast amount of nested submodules.

Signed-off-by: Wolfgang Klenk <[email protected]>

Author: wkl3nk

api/v1/mapping/src/commonMain/kotlin/Mappings.kt

@@ -578,7 +578,8 @@ fun ScannerJobConfiguration.mapToApi() = ApiScannerJobConfiguration(
     skipExcluded,
     sourceCodeOrigins?.map { it.mapToApi() },
     config?.mapValues { it.value.mapToApi() },
-    keepAliveWorker
+    keepAliveWorker,
+    submoduleFetchStrategy.mapToApi()
 )
 
 fun ApiScannerJobConfiguration.mapToModel() = ScannerJobConfiguration(
@@ -590,7 +591,8 @@ fun ApiScannerJobConfiguration.mapToModel() = ScannerJobConfiguration(
     skipExcluded,
     sourceCodeOrigins?.map { it.mapToModel() },
     config?.mapValues { it.value.mapToModel() },
-    keepAliveWorker
+    keepAliveWorker,
+    submoduleFetchStrategy.mapToModel()
 )
 
 fun Secret.mapToApi() = ApiSecret(name, description)

api/v1/model/src/commonMain/kotlin/JobConfigurations.kt

@@ -187,7 +187,14 @@ data class ScannerJobConfiguration(
      * Keep the worker alive after it has finished. This is useful for manual problem analysis directly
      * within the pod's execution environment.
      */
-    val keepAliveWorker: Boolean = false
+    val keepAliveWorker: Boolean = false,
+
+    /**
+     * Specifies how submodules are fetched when resolving provenances. Currently supported only for Git repositories.
+     * If set to [SubmoduleFetchStrategy.FULLY_RECURSIVE] (default), all submodules are fetched recursively. If set
+     * to [SubmoduleFetchStrategy.TOP_LEVEL_ONLY], only the top-level submodules are fetched.
+     */
+    val submoduleFetchStrategy: SubmoduleFetchStrategy = SubmoduleFetchStrategy.FULLY_RECURSIVE
 )
 
 /**

model/src/commonMain/kotlin/JobConfigurations.kt

@@ -196,7 +196,14 @@ data class ScannerJobConfiguration(
      * Keep the worker alive after it has finished. This is useful for manual problem analysis directly
      * within the pod's execution environment.
      */
-    val keepAliveWorker: Boolean = false
+    val keepAliveWorker: Boolean = false,
+
+    /**
+     * Specifies how submodules are fetched when resolving provenances. Currently supported only for Git repositories.
+     * If set to [SubmoduleFetchStrategy.FULLY_RECURSIVE] (default), all submodules are fetched recursively. If set
+     * to [SubmoduleFetchStrategy.TOP_LEVEL_ONLY], only the top-level submodules are fetched.
+     */
+    val submoduleFetchStrategy: SubmoduleFetchStrategy = SubmoduleFetchStrategy.FULLY_RECURSIVE
 )
 
 /**

workers/scanner/src/main/kotlin/scanner/ScannerRunner.kt

@@ -20,6 +20,7 @@
 package org.eclipse.apoapsis.ortserver.workers.scanner
 
 import org.eclipse.apoapsis.ortserver.model.ScannerJobConfiguration
+import org.eclipse.apoapsis.ortserver.model.SubmoduleFetchStrategy
 import org.eclipse.apoapsis.ortserver.workers.common.OrtServerFileListStorage
 import org.eclipse.apoapsis.ortserver.workers.common.context.WorkerContext
 import org.eclipse.apoapsis.ortserver.workers.common.mapToOrt
@@ -33,6 +34,7 @@ import org.ossreviewtoolkit.model.PackageType
 import org.ossreviewtoolkit.model.Provenance
 import org.ossreviewtoolkit.model.ScannerRun
 import org.ossreviewtoolkit.model.SourceCodeOrigin
+import org.ossreviewtoolkit.model.VcsType
 import org.ossreviewtoolkit.model.config.DownloaderConfiguration
 import org.ossreviewtoolkit.model.config.ScannerConfiguration
 import org.ossreviewtoolkit.model.utils.FileArchiver
@@ -58,6 +60,24 @@ class ScannerRunner(
     ): OrtScannerResult {
         val pluginConfigs = context.resolvePluginConfigSecrets(config.config)
 
+        // If the submodule fetch strategy is set to TOP_LEVEL_ONLY, for git use a plugin config that prevents that
+        // submodules are fetched recursively.
+        val vcsPluginConfigs = if (config.submoduleFetchStrategy == SubmoduleFetchStrategy.TOP_LEVEL_ONLY) {
+            mapOf(
+                VcsType.GIT.toString() to PluginConfig(
+                    options = mapOf("updateNestedSubmodules" to "false")
+                )
+            )
+        } else {
+            emptyMap()
+        }
+
+        if (config.submoduleFetchStrategy == SubmoduleFetchStrategy.DISABLED) {
+            throw ScannerException(
+                "Scanner job configuration option SubmoduleFetchStrategy.DISABLED is not supported."
+            )
+        }
+
         val packageProvenanceCache = PackageProvenanceCache()
         val packageProvenanceStorage = OrtServerPackageProvenanceStorage(db, scannerRunId, packageProvenanceCache)
         val nestedProvenanceStorage = OrtServerNestedProvenanceStorage(db, packageProvenanceCache)
@@ -85,7 +105,8 @@ class ScannerRunner(
                 ?: listOf(SourceCodeOrigin.ARTIFACT, SourceCodeOrigin.VCS)
         )
 
-        val workingTreeCache = DefaultWorkingTreeCache()
+        val workingTreeCache = DefaultWorkingTreeCache().addVcsPluginConfigs(vcsPluginConfigs)
+
         val provenanceDownloader = DefaultProvenanceDownloader(downloaderConfig, workingTreeCache)
         val packageProvenanceResolver = DefaultPackageProvenanceResolver(
             scanStorages.packageProvenanceStorage,