diff --git a/plugins/commands/migrate/build.gradle.kts b/plugins/commands/migrate/build.gradle.kts index a85b0667ab8b7..778aa3f9661df 100644 --- a/plugins/commands/migrate/build.gradle.kts +++ b/plugins/commands/migrate/build.gradle.kts @@ -27,7 +27,7 @@ dependencies { implementation(jacksonLibs.jacksonModuleKotlin) implementation(libs.clikt) - implementation(projects.plugins.packageCurationProviders.ortConfigPackageCurationProvider) + implementation(projects.plugins.packageCurationProviders.gitPackageCurationProvider) implementation(projects.plugins.packageManagers.nugetPackageManager) implementation(projects.utils.commonUtils) diff --git a/plugins/commands/migrate/src/main/kotlin/MigrateCommand.kt b/plugins/commands/migrate/src/main/kotlin/MigrateCommand.kt index 444e5b8a61a22..6eec51bdbae56 100644 --- a/plugins/commands/migrate/src/main/kotlin/MigrateCommand.kt +++ b/plugins/commands/migrate/src/main/kotlin/MigrateCommand.kt @@ -37,7 +37,7 @@ import org.ossreviewtoolkit.plugins.api.OrtPlugin import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.plugins.commands.api.OrtCommand import org.ossreviewtoolkit.plugins.commands.api.OrtCommandFactory -import org.ossreviewtoolkit.plugins.packagecurationproviders.ortconfig.toCurationPath +import org.ossreviewtoolkit.plugins.packagecurationproviders.git.toCurationPath import org.ossreviewtoolkit.plugins.packagemanagers.nuget.utils.getIdentifierWithNamespace import org.ossreviewtoolkit.utils.common.div import org.ossreviewtoolkit.utils.common.expandTilde diff --git a/plugins/package-curation-providers/ort-config/build.gradle.kts b/plugins/package-curation-providers/git/build.gradle.kts similarity index 88% rename from plugins/package-curation-providers/ort-config/build.gradle.kts rename to plugins/package-curation-providers/git/build.gradle.kts index 17645cd370803..1978460818e89 100644 --- a/plugins/package-curation-providers/ort-config/build.gradle.kts +++ b/plugins/package-curation-providers/git/build.gradle.kts @@ -25,10 +25,8 @@ plugins { dependencies { api(projects.plugins.packageCurationProviders.packageCurationProviderApi) - implementation(projects.downloader) + implementation(projects.plugins.versionControlSystems.gitVersionControlSystem) implementation(projects.utils.commonUtils) - funTestImplementation(projects.plugins.versionControlSystems.gitVersionControlSystem) - ksp(projects.plugins.packageCurationProviders.packageCurationProviderApi) } diff --git a/plugins/package-curation-providers/git/src/funTest/kotlin/GitPackageCurationProviderFunTest.kt b/plugins/package-curation-providers/git/src/funTest/kotlin/GitPackageCurationProviderFunTest.kt new file mode 100644 index 0000000000000..ce7dc9813ab56 --- /dev/null +++ b/plugins/package-curation-providers/git/src/funTest/kotlin/GitPackageCurationProviderFunTest.kt @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2026 The ORT Project Copyright Holders + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagecurationproviders.git + +import io.kotest.core.annotation.Tags +import io.kotest.core.spec.style.WordSpec +import io.kotest.matchers.collections.beEmpty +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNot + +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.GitFactory + +/** A fixed revision to ensure that the test is not affected by changes to the repository. */ +private const val REVISION = "6fd0972895b8c10d075d8aab0c854f91157a7d0e" + +@Tags("RequiresExternalTool") +class GitPackageCurationProviderFunTest : WordSpec({ + "create()" should { + "clone the correct revision" { + val provider = GitPackageCurationProviderFactory.create( + repositoryUrl = ORT_CONFIG_REPOSITORY_URL, + revision = REVISION + ) + + val workingTree = GitFactory.create().getWorkingTree(provider.repositoryDir) + + workingTree.getRevision() shouldBe REVISION + } + + "clone the default branch if no revision is provided" { + val provider = GitPackageCurationProviderFactory.create(ORT_CONFIG_REPOSITORY_URL) + + val git = GitFactory.create() + val workingTree = git.getWorkingTree(provider.repositoryDir) + val clonedRevision = workingTree.getRevision() + + git.updateWorkingTree(workingTree, "main") + + clonedRevision shouldBe workingTree.getRevision() + } + } + + "getCurationsFor()" should { + val provider = GitPackageCurationProviderFactory.create( + repositoryUrl = ORT_CONFIG_REPOSITORY_URL, + revision = REVISION, + path = "curations" + ) + + "return the curations from the configured path" { + val azureCore = Identifier("NuGet:Azure:Core:1.22.0") + val azureCoreAmqp = Identifier("NuGet:Azure.Core:Amqp:1.2.0") + val packages = createPackagesFromIds(azureCore, azureCoreAmqp) + + val curations = provider.getCurationsFor(packages) + + curations.filter { it.isApplicable(azureCore) } shouldNot beEmpty() + curations.filter { it.isApplicable(azureCoreAmqp) } shouldNot beEmpty() + } + + "return curations that match the namespace of a package" { + val xrd4j = Identifier("Maven:org.niis.xrd4j:foo:0.0.0") + val packages = createPackagesFromIds(xrd4j) + + val curations = provider.getCurationsFor(packages) + + curations.filter { it.isApplicable(xrd4j) } shouldNot beEmpty() + } + + "return an empty result for packages which have no curations" { + val packages = createPackagesFromIds(Identifier("Some:Bogus:Package:Id")) + + val curations = provider.getCurationsFor(packages) + + curations should beEmpty() + } + } +}) diff --git a/plugins/package-curation-providers/ort-config/src/funTest/kotlin/OrtConfigPackageCurationProviderFunTest.kt b/plugins/package-curation-providers/git/src/funTest/kotlin/OrtConfigPackageCurationProviderFunTest.kt similarity index 93% rename from plugins/package-curation-providers/ort-config/src/funTest/kotlin/OrtConfigPackageCurationProviderFunTest.kt rename to plugins/package-curation-providers/git/src/funTest/kotlin/OrtConfigPackageCurationProviderFunTest.kt index 160a1f4142a68..9818ce937c26d 100644 --- a/plugins/package-curation-providers/ort-config/src/funTest/kotlin/OrtConfigPackageCurationProviderFunTest.kt +++ b/plugins/package-curation-providers/git/src/funTest/kotlin/OrtConfigPackageCurationProviderFunTest.kt @@ -17,7 +17,7 @@ * License-Filename: LICENSE */ -package org.ossreviewtoolkit.plugins.packagecurationproviders.ortconfig +package org.ossreviewtoolkit.plugins.packagecurationproviders.git import io.kotest.core.annotation.Tags import io.kotest.core.spec.style.StringSpec @@ -61,4 +61,4 @@ class OrtConfigPackageCurationProviderFunTest : StringSpec({ } }) -private fun createPackagesFromIds(vararg ids: Identifier) = ids.map { Package.EMPTY.copy(id = it) } +internal fun createPackagesFromIds(vararg ids: Identifier) = ids.map { Package.EMPTY.copy(id = it) } diff --git a/plugins/package-curation-providers/git/src/main/kotlin/GitPackageCurationProvider.kt b/plugins/package-curation-providers/git/src/main/kotlin/GitPackageCurationProvider.kt new file mode 100644 index 0000000000000..0310c4c025aeb --- /dev/null +++ b/plugins/package-curation-providers/git/src/main/kotlin/GitPackageCurationProvider.kt @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2026 The ORT Project Copyright Holders + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagecurationproviders.git + +import java.io.File +import java.io.IOException + +import org.apache.logging.log4j.kotlin.logger + +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.model.Package +import org.ossreviewtoolkit.model.PackageCuration +import org.ossreviewtoolkit.model.VcsInfo +import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.readValue +import org.ossreviewtoolkit.plugins.api.OrtPlugin +import org.ossreviewtoolkit.plugins.api.OrtPluginOption +import org.ossreviewtoolkit.plugins.api.PluginDescriptor +import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProvider +import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProviderFactory +import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.GitFactory +import org.ossreviewtoolkit.utils.common.div +import org.ossreviewtoolkit.utils.common.encodeOr +import org.ossreviewtoolkit.utils.common.fileSystemEncode +import org.ossreviewtoolkit.utils.common.safeMkdirs +import org.ossreviewtoolkit.utils.ort.ortDataDirectory + +data class GitPackageCurationProviderConfig( + /** The URL of the repository containing the curations. */ + val repositoryUrl: String, + + /** The optional revision to use. If not specified, the default branch is used. */ + val revision: String?, + + /** The path that contains the package curations. */ + @OrtPluginOption(defaultValue = "curations") + val path: String +) + +/** + * A [PackageCurationProvider] that loads [PackageCuration]s from the configured Git repository. + * + * ### Path layout + * + * The provider requires that the curation files follow the same path layout as in the + * [ort-config repository](https://github.com/oss-review-toolkit/ort-config#curations): + * + * * Files with curations for a specific package must be located at `[type]/[namespace]/[name].yml`, based on the + * identifier of the package. If a component of the identifier is empty, `_` is used as a placeholder. For example, + * for the package `NuGet::Azure.Core:1.2.0`, the curation file must be located at `NuGet/_/Azure.Core.yml`. + * * Files with curations that match all packages within a namespace must be located at `[type]/[namespace]/_.yml`. + * + * Namespace-scoped curations are loaded before package-scoped curations, so that the latter can override the former. + */ +@OrtPlugin( + displayName = "Git Repository", + summary = "A package curation provider that loads package curations from a Git repository.", + factory = PackageCurationProviderFactory::class +) +open class GitPackageCurationProvider( + override val descriptor: PluginDescriptor = GitPackageCurationProviderFactory.descriptor, + private val config: GitPackageCurationProviderConfig +) : PackageCurationProvider { + init { + require(config.repositoryUrl.isNotBlank()) { "The repository URL must not be blank." } + } + + internal val repositoryDir by lazy { + // Use a stable cache path to clone the repository to speed up subsequent runs. + (ortDataDirectory / "cache" / "git-package-curation-provider" / config.repositoryUrl.fileSystemEncode()).also { + updateRepository(it) + } + } + + private val curationsDir by lazy { repositoryDir / config.path } + + override fun getCurationsFor(packages: Collection): Set = + packages.flatMapTo(mutableSetOf()) { pkg -> getCurationsFor(pkg.id) } + + private fun getCurationsFor(pkgId: Identifier): List { + // The Git repository has to follow path layout conventions, so curations can be looked up directly. + val packageCurationsFile = curationsDir / pkgId.toCurationPath() + + // Also consider curations for all packages in a namespace. + val namespaceCurationsFile = packageCurationsFile.resolveSibling("_.yml") + + // Return namespace-level curations before package-level curations to allow overriding the former. + return listOf(namespaceCurationsFile, packageCurationsFile).filter { it.isFile }.flatMap { file -> + runCatching { + file.readValue>().filter { it.isApplicable(pkgId) } + }.getOrElse { + throw IOException("Failed parsing package curation from '${file.absolutePath}'.", it) + } + } + } + + private fun updateRepository(dir: File) { + val vcsInfo = VcsInfo.EMPTY.copy(type = VcsType.GIT, url = config.repositoryUrl) + dir.safeMkdirs() + + GitFactory.create().apply { + val workingTree = initWorkingTree(dir, vcsInfo) + val revision = config.revision ?: getDefaultBranchName(config.repositoryUrl) + val clonedRevision = updateWorkingTree(workingTree, revision).getOrThrow() + + logger.info { + buildString { + append("Successfully cloned revision $clonedRevision ") + if (revision != clonedRevision) append("(from $revision) ") + append("of ${config.repositoryUrl}.") + } + } + } + } +} + +/** + * The path must be aligned with the + * [conventions for the ort-config repository](https://github.com/oss-review-toolkit/ort-config#curations). + */ +fun Identifier.toCurationPath() = "${type.encodeOr("_")}/${namespace.encodeOr("_")}/${name.encodeOr("_")}.yml" diff --git a/plugins/package-curation-providers/git/src/main/kotlin/OrtConfigPackageCurationProvider.kt b/plugins/package-curation-providers/git/src/main/kotlin/OrtConfigPackageCurationProvider.kt new file mode 100644 index 0000000000000..98334987598d4 --- /dev/null +++ b/plugins/package-curation-providers/git/src/main/kotlin/OrtConfigPackageCurationProvider.kt @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2022 The ORT Project Copyright Holders + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagecurationproviders.git + +import org.ossreviewtoolkit.model.PackageCuration +import org.ossreviewtoolkit.plugins.api.OrtPlugin +import org.ossreviewtoolkit.plugins.api.PluginDescriptor +import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProvider +import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProviderFactory + +private const val ORT_CONFIG_REPOSITORY_BRANCH = "main" +internal const val ORT_CONFIG_REPOSITORY_URL = "https://github.com/oss-review-toolkit/ort-config.git" + +/** + * A [PackageCurationProvider] that provides [PackageCuration]s loaded from the + * [ort-config repository](https://github.com/oss-review-toolkit/ort-config). + */ +@OrtPlugin( + id = "ORTConfig", + displayName = "ORT Config Repository", + summary = "A package curation provider that loads package curations from the ort-config repository.", + factory = PackageCurationProviderFactory::class +) +class OrtConfigPackageCurationProvider( + descriptor: PluginDescriptor = OrtConfigPackageCurationProviderFactory.descriptor +) : GitPackageCurationProvider( + descriptor = descriptor, + config = GitPackageCurationProviderConfig( + repositoryUrl = ORT_CONFIG_REPOSITORY_URL, + revision = ORT_CONFIG_REPOSITORY_BRANCH, + path = "curations" + ) +) diff --git a/plugins/package-curation-providers/ort-config/src/main/kotlin/OrtConfigPackageCurationProvider.kt b/plugins/package-curation-providers/ort-config/src/main/kotlin/OrtConfigPackageCurationProvider.kt deleted file mode 100644 index 96a3ae92098a9..0000000000000 --- a/plugins/package-curation-providers/ort-config/src/main/kotlin/OrtConfigPackageCurationProvider.kt +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (C) 2022 The ORT Project Copyright Holders - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.plugins.packagecurationproviders.ortconfig - -import java.io.File -import java.io.IOException - -import org.apache.logging.log4j.kotlin.logger - -import org.ossreviewtoolkit.downloader.VersionControlSystem -import org.ossreviewtoolkit.model.Identifier -import org.ossreviewtoolkit.model.Package -import org.ossreviewtoolkit.model.PackageCuration -import org.ossreviewtoolkit.model.VcsInfo -import org.ossreviewtoolkit.model.VcsType -import org.ossreviewtoolkit.model.readValue -import org.ossreviewtoolkit.plugins.api.OrtPlugin -import org.ossreviewtoolkit.plugins.api.PluginDescriptor -import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProvider -import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProviderFactory -import org.ossreviewtoolkit.utils.common.div -import org.ossreviewtoolkit.utils.common.encodeOr -import org.ossreviewtoolkit.utils.common.safeMkdirs -import org.ossreviewtoolkit.utils.ort.ortDataDirectory - -private const val ORT_CONFIG_REPOSITORY_BRANCH = "main" -private const val ORT_CONFIG_REPOSITORY_URL = "https://github.com/oss-review-toolkit/ort-config.git" - -/** - * A [PackageCurationProvider] that provides [PackageCuration]s loaded from the - * [ort-config repository](https://github.com/oss-review-toolkit/ort-config). - */ -@OrtPlugin( - id = "ORTConfig", - displayName = "ORT Config Repository", - summary = "A package curation provider that loads package curations from the ort-config repository.", - factory = PackageCurationProviderFactory::class -) -open class OrtConfigPackageCurationProvider( - override val descriptor: PluginDescriptor = OrtConfigPackageCurationProviderFactory.descriptor -) : PackageCurationProvider { - private val curationsDir by lazy { - ortDataDirectory.resolve("ort-config").also { - updateOrtConfig(it) - } - } - - override fun getCurationsFor(packages: Collection): Set = - packages.flatMapTo(mutableSetOf()) { pkg -> getCurationsFor(pkg.id) } - - private fun getCurationsFor(pkgId: Identifier): List { - // The ORT config repository follows path layout conventions, so curations can be looked up directly. - val packageCurationsFile = curationsDir / "curations" / pkgId.toCurationPath() - - // Also consider curations for all packages in a namespace. - val namespaceCurationsFile = packageCurationsFile.resolveSibling("_.yml") - - // Return namespace-level curations before package-level curations to allow overriding the former. - return listOf(namespaceCurationsFile, packageCurationsFile).filter { it.isFile }.flatMap { file -> - runCatching { - file.readValue>().filter { it.isApplicable(pkgId) } - }.getOrElse { - throw IOException("Failed parsing package curation from '${file.absolutePath}'.", it) - } - } - } -} - -/** - * The path must be aligned with the - * [conventions for the ort-config repository](https://github.com/oss-review-toolkit/ort-config#curations). - */ -fun Identifier.toCurationPath() = "${type.encodeOr("_")}/${namespace.encodeOr("_")}/${name.encodeOr("_")}.yml" - -private fun updateOrtConfig(dir: File) { - val vcsInfo = VcsInfo.EMPTY.copy(type = VcsType.GIT, url = ORT_CONFIG_REPOSITORY_URL) - val vcs = checkNotNull(VersionControlSystem.forType(vcsInfo.type)) { - "No applicable VersionControlSystem implementation found for ${vcsInfo.type}." - } - - dir.safeMkdirs() - - vcs.apply { - val workingTree = initWorkingTree(dir, vcsInfo) - val revision = updateWorkingTree(workingTree, ORT_CONFIG_REPOSITORY_BRANCH).getOrThrow() - logger.info { - "Successfully cloned $revision from $ORT_CONFIG_REPOSITORY_URL." - } - } -}