Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion plugins/commands/migrate/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ dependencies {

implementation(jacksonLibs.jacksonModuleKotlin)
implementation(libs.clikt)
implementation(projects.plugins.packageCurationProviders.ortConfigPackageCurationProvider)
implementation(projects.plugins.packageCurationProviders.gitPackageCurationProvider)
implementation(projects.plugins.packageManagers.nugetPackageManager)
implementation(projects.utils.commonUtils)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import org.ossreviewtoolkit.plugins.api.OrtPlugin
import org.ossreviewtoolkit.plugins.api.PluginDescriptor
import org.ossreviewtoolkit.plugins.commands.api.OrtCommand
import org.ossreviewtoolkit.plugins.commands.api.OrtCommandFactory
import org.ossreviewtoolkit.plugins.packagecurationproviders.ortconfig.toCurationPath
import org.ossreviewtoolkit.plugins.packagecurationproviders.git.toCurationPath
import org.ossreviewtoolkit.plugins.packagemanagers.nuget.utils.getIdentifierWithNamespace
import org.ossreviewtoolkit.utils.common.div
import org.ossreviewtoolkit.utils.common.expandTilde
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,8 @@ plugins {
dependencies {
api(projects.plugins.packageCurationProviders.packageCurationProviderApi)

implementation(projects.downloader)
implementation(projects.plugins.versionControlSystems.gitVersionControlSystem)
implementation(projects.utils.commonUtils)

funTestImplementation(projects.plugins.versionControlSystems.gitVersionControlSystem)

ksp(projects.plugins.packageCurationProviders.packageCurationProviderApi)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/*
* Copyright (C) 2026 The ORT Project Copyright Holders <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.plugins.packagecurationproviders.git

import io.kotest.core.annotation.Tags
import io.kotest.core.spec.style.WordSpec
import io.kotest.matchers.collections.beEmpty
import io.kotest.matchers.should

Check warning on line 25 in plugins/package-curation-providers/git/src/funTest/kotlin/GitPackageCurationProviderFunTest.kt

View workflow job for this annotation

GitHub Actions / qodana-scan

Unused import directive

Unused import directive

Check warning

Code scanning / QDJVM

Unused import directive Warning

Unused import directive
import io.kotest.matchers.shouldBe
import io.kotest.matchers.shouldNot

import org.ossreviewtoolkit.model.Identifier
import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.GitFactory

/** A fixed revision to ensure that the test is not affected by changes to the repository. */
private const val REVISION = "6fd0972895b8c10d075d8aab0c854f91157a7d0e"

@Tags("RequiresExternalTool")
class GitPackageCurationProviderFunTest : WordSpec({

Check warning on line 36 in plugins/package-curation-providers/git/src/funTest/kotlin/GitPackageCurationProviderFunTest.kt

View workflow job for this annotation

GitHub Actions / qodana-scan

Unused symbol

Class "GitPackageCurationProviderFunTest" is never used

Check warning

Code scanning / QDJVM

Unused symbol Warning

Class "GitPackageCurationProviderFunTest" is never used
Comment thread
sschuberth marked this conversation as resolved.
Dismissed
"create()" should {
"clone the correct revision" {
val provider = GitPackageCurationProviderFactory.create(
repositoryUrl = ORT_CONFIG_REPOSITORY_URL,
revision = REVISION
)

val workingTree = GitFactory.create().getWorkingTree(provider.repositoryDir)

workingTree.getRevision() shouldBe REVISION
}

"clone the default branch if no revision is provided" {
val provider = GitPackageCurationProviderFactory.create(ORT_CONFIG_REPOSITORY_URL)

val git = GitFactory.create()
val workingTree = git.getWorkingTree(provider.repositoryDir)
val clonedRevision = workingTree.getRevision()

git.updateWorkingTree(workingTree, "main")

clonedRevision shouldBe workingTree.getRevision()
}
}

"getCurationsFor()" should {
val provider = GitPackageCurationProviderFactory.create(
repositoryUrl = ORT_CONFIG_REPOSITORY_URL,
revision = REVISION,
path = "curations"
)

"return the curations from the configured path" {
val azureCore = Identifier("NuGet:Azure:Core:1.22.0")
val azureCoreAmqp = Identifier("NuGet:Azure.Core:Amqp:1.2.0")
val packages = createPackagesFromIds(azureCore, azureCoreAmqp)

val curations = provider.getCurationsFor(packages)

curations.filter { it.isApplicable(azureCore) } shouldNot beEmpty()
curations.filter { it.isApplicable(azureCoreAmqp) } shouldNot beEmpty()
}

"return curations that match the namespace of a package" {
val xrd4j = Identifier("Maven:org.niis.xrd4j:foo:0.0.0")
val packages = createPackagesFromIds(xrd4j)

val curations = provider.getCurationsFor(packages)

curations.filter { it.isApplicable(xrd4j) } shouldNot beEmpty()
}

"return an empty result for packages which have no curations" {
val packages = createPackagesFromIds(Identifier("Some:Bogus:Package:Id"))

val curations = provider.getCurationsFor(packages)

curations should beEmpty()
}
}
})
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.plugins.packagecurationproviders.ortconfig
package org.ossreviewtoolkit.plugins.packagecurationproviders.git

import io.kotest.core.annotation.Tags
import io.kotest.core.spec.style.StringSpec
Expand All @@ -29,7 +29,7 @@
import org.ossreviewtoolkit.model.Package

@Tags("RequiresExternalTool")
class OrtConfigPackageCurationProviderFunTest : StringSpec({

Check warning on line 32 in plugins/package-curation-providers/git/src/funTest/kotlin/OrtConfigPackageCurationProviderFunTest.kt

View workflow job for this annotation

GitHub Actions / qodana-scan

Unused symbol

Class "OrtConfigPackageCurationProviderFunTest" is never used
fun createProvider() = OrtConfigPackageCurationProviderFactory.create()

"The provider succeeds to return known curations for packages" {
Expand Down Expand Up @@ -61,4 +61,4 @@
}
})

private fun createPackagesFromIds(vararg ids: Identifier) = ids.map { Package.EMPTY.copy(id = it) }
internal fun createPackagesFromIds(vararg ids: Identifier) = ids.map { Package.EMPTY.copy(id = it) }
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
/*
* Copyright (C) 2026 The ORT Project Copyright Holders <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.plugins.packagecurationproviders.git

import java.io.File
import java.io.IOException

import org.apache.logging.log4j.kotlin.logger

import org.ossreviewtoolkit.model.Identifier
import org.ossreviewtoolkit.model.Package
import org.ossreviewtoolkit.model.PackageCuration
import org.ossreviewtoolkit.model.VcsInfo
import org.ossreviewtoolkit.model.VcsType
import org.ossreviewtoolkit.model.readValue
import org.ossreviewtoolkit.plugins.api.OrtPlugin
import org.ossreviewtoolkit.plugins.api.OrtPluginOption
import org.ossreviewtoolkit.plugins.api.PluginDescriptor
import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProvider
import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProviderFactory
import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.GitFactory
import org.ossreviewtoolkit.utils.common.div
import org.ossreviewtoolkit.utils.common.encodeOr
import org.ossreviewtoolkit.utils.common.fileSystemEncode
import org.ossreviewtoolkit.utils.common.safeMkdirs
import org.ossreviewtoolkit.utils.ort.ortDataDirectory

data class GitPackageCurationProviderConfig(
/** The URL of the repository containing the curations. */
val repositoryUrl: String,

/** The optional revision to use. If not specified, the default branch is used. */
val revision: String?,

/** The path that contains the package curations. */
@OrtPluginOption(defaultValue = "curations")
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if this should really be the default, or just something that OrtConfigPackageCurationProvider sets.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I chose it because it seems people generally use the ort-config repo as template, so why not make their lives a bit easier.

val path: String
)

/**
* A [PackageCurationProvider] that loads [PackageCuration]s from the configured Git repository.
*
* ### Path layout
*
* The provider requires that the curation files follow the same path layout as in the
* [ort-config repository](https://github.com/oss-review-toolkit/ort-config#curations):
*
* * Files with curations for a specific package must be located at `[type]/[namespace]/[name].yml`, based on the
* identifier of the package. If a component of the identifier is empty, `_` is used as a placeholder. For example,
* for the package `NuGet::Azure.Core:1.2.0`, the curation file must be located at `NuGet/_/Azure.Core.yml`.
* * Files with curations that match all packages within a namespace must be located at `[type]/[namespace]/_.yml`.
*
* Namespace-scoped curations are loaded before package-scoped curations, so that the latter can override the former.
*/
@OrtPlugin(
displayName = "Git Repository",
summary = "A package curation provider that loads package curations from a Git repository.",
factory = PackageCurationProviderFactory::class
)
open class GitPackageCurationProvider(
override val descriptor: PluginDescriptor = GitPackageCurationProviderFactory.descriptor,
private val config: GitPackageCurationProviderConfig
) : PackageCurationProvider {
init {
require(config.repositoryUrl.isNotBlank()) { "The repository URL must not be blank." }
}

internal val repositoryDir by lazy {
// Use a stable cache path to clone the repository to speed up subsequent runs.
(ortDataDirectory / "cache" / "git-package-curation-provider" / config.repositoryUrl.fileSystemEncode()).also {
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure about "cache" here. Should we put Git working directories into a more specifically named directory? Which BTW makes me wonder why DefaultWorkingTreeCache uses val dir = createOrtTempDir() instead.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just want some stable directory because it's a nice optimization when running ORT locally. "cache" made sense to me to signal that this can be deleted without causing problems (even though there is no automatic cleanup). Do you have any specific proposals for alternatives? It can also easily be changed later, if needed.

Which BTW makes me wonder why DefaultWorkingTreeCache uses val dir = createOrtTempDir() instead.

IIRC this is because the cache was made specifically for the requirements of the scanner and you would not want to permanently archive the source code of all scanned packages when running ORT locally.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you have any specific proposals for alternatives?

Well, maybe any of "git", "worktree", "clone", "plugin"... but you can also keep it at "cache".

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I kind of like "plugin" because it clarifies the scope, should I change it? Or maybe we can revisit this when doing the same change for the package configuration provider.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or maybe we can revisit this when doing the same change for the package configuration provider.

Yes, let's do that.

updateRepository(it)
}
}

private val curationsDir by lazy { repositoryDir / config.path }

override fun getCurationsFor(packages: Collection<Package>): Set<PackageCuration> =
packages.flatMapTo(mutableSetOf()) { pkg -> getCurationsFor(pkg.id) }

private fun getCurationsFor(pkgId: Identifier): List<PackageCuration> {
// The Git repository has to follow path layout conventions, so curations can be looked up directly.
val packageCurationsFile = curationsDir / pkgId.toCurationPath()

// Also consider curations for all packages in a namespace.
val namespaceCurationsFile = packageCurationsFile.resolveSibling("_.yml")

// Return namespace-level curations before package-level curations to allow overriding the former.
return listOf(namespaceCurationsFile, packageCurationsFile).filter { it.isFile }.flatMap { file ->
runCatching {
file.readValue<List<PackageCuration>>().filter { it.isApplicable(pkgId) }
}.getOrElse {
throw IOException("Failed parsing package curation from '${file.absolutePath}'.", it)
}
}
}

private fun updateRepository(dir: File) {
val vcsInfo = VcsInfo.EMPTY.copy(type = VcsType.GIT, url = config.repositoryUrl)
dir.safeMkdirs()

GitFactory.create().apply {
val workingTree = initWorkingTree(dir, vcsInfo)
val revision = config.revision ?: getDefaultBranchName(config.repositoryUrl)
val clonedRevision = updateWorkingTree(workingTree, revision).getOrThrow()

logger.info {
buildString {
append("Successfully cloned revision $clonedRevision ")
if (revision != clonedRevision) append("(from $revision) ")
append("of ${config.repositoryUrl}.")
}
}
}
}
}

/**
* The path must be aligned with the
* [conventions for the ort-config repository](https://github.com/oss-review-toolkit/ort-config#curations).
*/
fun Identifier.toCurationPath() = "${type.encodeOr("_")}/${namespace.encodeOr("_")}/${name.encodeOr("_")}.yml"
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
* Copyright (C) 2022 The ORT Project Copyright Holders <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.plugins.packagecurationproviders.git

import org.ossreviewtoolkit.model.PackageCuration
import org.ossreviewtoolkit.plugins.api.OrtPlugin
import org.ossreviewtoolkit.plugins.api.PluginDescriptor
import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProvider
import org.ossreviewtoolkit.plugins.packagecurationproviders.api.PackageCurationProviderFactory

private const val ORT_CONFIG_REPOSITORY_BRANCH = "main"
internal const val ORT_CONFIG_REPOSITORY_URL = "https://github.com/oss-review-toolkit/ort-config.git"

/**
* A [PackageCurationProvider] that provides [PackageCuration]s loaded from the
* [ort-config repository](https://github.com/oss-review-toolkit/ort-config).
*/
@OrtPlugin(
id = "ORTConfig",
displayName = "ORT Config Repository",
summary = "A package curation provider that loads package curations from the ort-config repository.",
factory = PackageCurationProviderFactory::class
)
class OrtConfigPackageCurationProvider(

Check warning on line 41 in plugins/package-curation-providers/git/src/main/kotlin/OrtConfigPackageCurationProvider.kt

View workflow job for this annotation

GitHub Actions / qodana-scan

Unused symbol

Class "OrtConfigPackageCurationProvider" is never used

Check warning

Code scanning / QDJVM

Unused symbol Warning

Class "OrtConfigPackageCurationProvider" is never used
Comment thread
sschuberth marked this conversation as resolved.
Dismissed
descriptor: PluginDescriptor = OrtConfigPackageCurationProviderFactory.descriptor
) : GitPackageCurationProvider(
descriptor = descriptor,
config = GitPackageCurationProviderConfig(
repositoryUrl = ORT_CONFIG_REPOSITORY_URL,
revision = ORT_CONFIG_REPOSITORY_BRANCH,
path = "curations"
)
)
Loading
Loading