feat(scanner): Add submodule fetch strategy for nested repositories

Introduce `submoduleFetchStrategy` config to control how the Scanner
fetches Git submodules. When set to `TOP_LEVEL_ONLY`, only top-level
submodules are fetched, avoiding timeouts on deeply nested repos.

This mirrors the behavior already available in the Analyzer and
allows to resolve nested provenances even in this kind of repositories
with a vast amount of nested submodules.

Signed-off-by: Wolfgang Klenk <[email protected]>

Author: wkl3nk

api/v1/mapping/src/commonMain/kotlin/Mappings.kt

@@ -578,7 +578,8 @@ fun ScannerJobConfiguration.mapToApi() = ApiScannerJobConfiguration(
     skipExcluded,
     sourceCodeOrigins?.map { it.mapToApi() },
     config?.mapValues { it.value.mapToApi() },
-    keepAliveWorker
+    keepAliveWorker,
+    submoduleFetchStrategy.mapToApi()
 )
 
 fun ApiScannerJobConfiguration.mapToModel() = ScannerJobConfiguration(
@@ -590,7 +591,8 @@ fun ApiScannerJobConfiguration.mapToModel() = ScannerJobConfiguration(
     skipExcluded,
     sourceCodeOrigins?.map { it.mapToModel() },
     config?.mapValues { it.value.mapToModel() },
-    keepAliveWorker
+    keepAliveWorker,
+    submoduleFetchStrategy.mapToModel()
 )
 
 fun Secret.mapToApi() = ApiSecret(name, description)

api/v1/model/src/commonMain/kotlin/JobConfigurations.kt

@@ -187,7 +187,14 @@ data class ScannerJobConfiguration(
      * Keep the worker alive after it has finished. This is useful for manual problem analysis directly
      * within the pod's execution environment.
      */
-    val keepAliveWorker: Boolean = false
+    val keepAliveWorker: Boolean = false,
+
+    /**
+     * Specifies how submodules are fetched when resolving provenances. Currently supported only for Git repositories.
+     * If set to [SubmoduleFetchStrategy.FULLY_RECURSIVE] (default), all submodules are fetched recursively. If set
+     * to [SubmoduleFetchStrategy.TOP_LEVEL_ONLY], only the top-level submodules are fetched.
+     */
+    val submoduleFetchStrategy: SubmoduleFetchStrategy = SubmoduleFetchStrategy.FULLY_RECURSIVE
 )
 
 /**

model/src/commonMain/kotlin/JobConfigurations.kt

@@ -196,7 +196,14 @@ data class ScannerJobConfiguration(
      * Keep the worker alive after it has finished. This is useful for manual problem analysis directly
      * within the pod's execution environment.
      */
-    val keepAliveWorker: Boolean = false
+    val keepAliveWorker: Boolean = false,
+
+    /**
+     * Specifies how submodules are fetched when resolving provenances. Currently supported only for Git repositories.
+     * If set to [SubmoduleFetchStrategy.FULLY_RECURSIVE] (default), all submodules are fetched recursively. If set
+     * to [SubmoduleFetchStrategy.TOP_LEVEL_ONLY], only the top-level submodules are fetched.
+     */
+    val submoduleFetchStrategy: SubmoduleFetchStrategy = SubmoduleFetchStrategy.FULLY_RECURSIVE
 )
 
 /**

workers/scanner/src/main/kotlin/scanner/ScannerRunner.kt

@@ -58,6 +58,19 @@ class ScannerRunner(
     ): OrtScannerResult {
         val pluginConfigs = context.resolvePluginConfigSecrets(config.config)
 
+        // If the submodule fetch strategy is set to TOP_LEVEL_ONLY, for git use a plugin config that prevents that
+        // submodules are fetched recursively.
+        val vcsPluginConfigs = if (config.submoduleFetchStrategy == SubmoduleFetchStrategy.TOP_LEVEL_ONLY) {
+            mapOf(
+                VcsType.GIT.toString() to PluginConfig(
+                    options = mapOf("updateNestedSubmodules" to "false")
+                )
+            )
+        } else {
+            emptyMap()
+        }
+        val canonicalVcsPluginConfigs = createCanonicalVcsPluginConfigs(vcsPluginConfigs)
+
         val packageProvenanceCache = PackageProvenanceCache()
         val packageProvenanceStorage = OrtServerPackageProvenanceStorage(db, scannerRunId, packageProvenanceCache)
         val nestedProvenanceStorage = OrtServerNestedProvenanceStorage(db, packageProvenanceCache)
@@ -85,7 +98,8 @@ class ScannerRunner(
                 ?: listOf(SourceCodeOrigin.ARTIFACT, SourceCodeOrigin.VCS)
         )
 
-        val workingTreeCache = DefaultWorkingTreeCache()
+        val workingTreeCache = DefaultWorkingTreeCache().addVcsPluginConfigs(vcsPluginConfigs)
+
         val provenanceDownloader = DefaultProvenanceDownloader(downloaderConfig, workingTreeCache)
         val packageProvenanceResolver = DefaultPackageProvenanceResolver(
             scanStorages.packageProvenanceStorage,