diff --git a/api/v1/mapping/src/commonMain/kotlin/Mappings.kt b/api/v1/mapping/src/commonMain/kotlin/Mappings.kt index e3512fa5f8..cee76ed00c 100644 --- a/api/v1/mapping/src/commonMain/kotlin/Mappings.kt +++ b/api/v1/mapping/src/commonMain/kotlin/Mappings.kt @@ -27,6 +27,7 @@ import org.eclipse.apoapsis.ortserver.api.v1.model.AnalyzerJob as ApiAnalyzerJob import org.eclipse.apoapsis.ortserver.api.v1.model.AnalyzerJobConfiguration as ApiAnalyzerJobConfiguration import org.eclipse.apoapsis.ortserver.api.v1.model.ComparisonOperator as ApiComparisonOperator import org.eclipse.apoapsis.ortserver.api.v1.model.CredentialsType as ApiCredentialsType +import org.eclipse.apoapsis.ortserver.api.v1.model.Curation as ApiCuration import org.eclipse.apoapsis.ortserver.api.v1.model.EcosystemStats as ApiEcosystemStats import org.eclipse.apoapsis.ortserver.api.v1.model.EnvironmentConfig as ApiEnvironmentConfig import org.eclipse.apoapsis.ortserver.api.v1.model.EnvironmentVariableDeclaration as ApiEnvironmentVariableDeclaration @@ -145,7 +146,7 @@ import org.eclipse.apoapsis.ortserver.model.runs.Issue import org.eclipse.apoapsis.ortserver.model.runs.OrtRuleViolation import org.eclipse.apoapsis.ortserver.model.runs.PackageFilters import org.eclipse.apoapsis.ortserver.model.runs.PackageManagerConfiguration -import org.eclipse.apoapsis.ortserver.model.runs.PackageWithShortestDependencyPaths +import org.eclipse.apoapsis.ortserver.model.runs.PackageRunData import org.eclipse.apoapsis.ortserver.model.runs.ProcessedDeclaredLicense import org.eclipse.apoapsis.ortserver.model.runs.Project import org.eclipse.apoapsis.ortserver.model.runs.RemoteArtifact @@ -153,6 +154,7 @@ import org.eclipse.apoapsis.ortserver.model.runs.ShortestDependencyPath import org.eclipse.apoapsis.ortserver.model.runs.VcsInfo import org.eclipse.apoapsis.ortserver.model.runs.advisor.Vulnerability import org.eclipse.apoapsis.ortserver.model.runs.advisor.VulnerabilityReference +import org.eclipse.apoapsis.ortserver.model.runs.repository.PackageCurationData import org.eclipse.apoapsis.ortserver.model.util.ComparisonOperator import org.eclipse.apoapsis.ortserver.model.util.FilterOperatorAndValue import org.eclipse.apoapsis.ortserver.model.util.ListQueryParameters @@ -857,7 +859,7 @@ fun ShortestDependencyPath.mapToApi() = ApiShortestDependencyPath( path = path.map { it.mapToApi() } ) -fun PackageWithShortestDependencyPaths.mapToApi() = ApiPackage( +fun PackageRunData.mapToApi() = ApiPackage( pkg.identifier.mapToApi(), pkg.purl, pkg.cpe, @@ -872,7 +874,17 @@ fun PackageWithShortestDependencyPaths.mapToApi() = ApiPackage( pkg.vcsProcessed.mapToApi(), pkg.isMetadataOnly, pkg.isModified, - shortestDependencyPaths.map { it.mapToApi() } + shortestDependencyPaths.map { it.mapToApi() }, + curations.map { it.mapToApi() } +) + +fun PackageCurationData.mapToApi(): ApiCuration = ApiCuration( + concludedLicense ?: "", + comment ?: "", + description ?: "", + homepageUrl ?: "", + authors?.toList() ?: emptyList(), + declaredLicenseMapping ) fun ApiPackageFilters.mapToModel(): PackageFilters = diff --git a/api/v1/model/src/commonMain/kotlin/Curation.kt b/api/v1/model/src/commonMain/kotlin/Curation.kt new file mode 100644 index 0000000000..de51e6e020 --- /dev/null +++ b/api/v1/model/src/commonMain/kotlin/Curation.kt @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2025 The ORT Server Authors (See ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.eclipse.apoapsis.ortserver.api.v1.model + +import kotlinx.serialization.Serializable + +@Serializable +data class Curation( + val concludedLicense: String, + val comment: String, + val description: String, + val homepageUrl: String, + val authors: List, + val declaredLicenseMapping: Map +) diff --git a/api/v1/model/src/commonMain/kotlin/Package.kt b/api/v1/model/src/commonMain/kotlin/Package.kt index 05ca32bb17..778c4b4c08 100644 --- a/api/v1/model/src/commonMain/kotlin/Package.kt +++ b/api/v1/model/src/commonMain/kotlin/Package.kt @@ -37,7 +37,8 @@ data class Package( val vcsProcessed: VcsInfo, val isMetadataOnly: Boolean = false, val isModified: Boolean = false, - val shortestDependencyPaths: List + val shortestDependencyPaths: List, + val curations: List ) /** diff --git a/core/src/main/kotlin/api/RunsRoute.kt b/core/src/main/kotlin/api/RunsRoute.kt index 6821c6bb3f..d8396a907c 100644 --- a/core/src/main/kotlin/api/RunsRoute.kt +++ b/core/src/main/kotlin/api/RunsRoute.kt @@ -76,7 +76,7 @@ import org.eclipse.apoapsis.ortserver.model.authorization.RepositoryPermission import org.eclipse.apoapsis.ortserver.model.repositories.OrtRunRepository import org.eclipse.apoapsis.ortserver.model.runs.Issue import org.eclipse.apoapsis.ortserver.model.runs.OrtRuleViolation -import org.eclipse.apoapsis.ortserver.model.runs.PackageWithShortestDependencyPaths +import org.eclipse.apoapsis.ortserver.model.runs.PackageRunData import org.eclipse.apoapsis.ortserver.model.runs.Project import org.eclipse.apoapsis.ortserver.services.IssueService import org.eclipse.apoapsis.ortserver.services.OrtRunService @@ -245,7 +245,7 @@ fun Route.runs() = route("runs") { .listForOrtRunId(ortRun.id, pagingOptions.mapToModel(), filters.mapToModel()) val pagedResponse = packagesForOrtRun - .mapToApi(PackageWithShortestDependencyPaths::mapToApi) + .mapToApi(PackageRunData::mapToApi) .toSearchResponse(filters) call.respond(HttpStatusCode.OK, pagedResponse) diff --git a/core/src/main/kotlin/apiDocs/RunsDocs.kt b/core/src/main/kotlin/apiDocs/RunsDocs.kt index 70ae10b781..c5586e7f6d 100644 --- a/core/src/main/kotlin/apiDocs/RunsDocs.kt +++ b/core/src/main/kotlin/apiDocs/RunsDocs.kt @@ -28,6 +28,7 @@ import kotlin.time.Duration.Companion.minutes import kotlinx.datetime.Clock import org.eclipse.apoapsis.ortserver.api.v1.model.ComparisonOperator +import org.eclipse.apoapsis.ortserver.api.v1.model.Curation import org.eclipse.apoapsis.ortserver.api.v1.model.EcosystemStats import org.eclipse.apoapsis.ortserver.api.v1.model.FilterOperatorAndValue import org.eclipse.apoapsis.ortserver.api.v1.model.Identifier @@ -428,6 +429,16 @@ val getPackagesByRunId: RouteConfig.() -> Unit = { Identifier("Maven", "org.example", "other", "1.0") ) ) + ), + curations = listOf( + Curation( + concludedLicense = "MIT", + comment = "A comment", + description = "", + homepageUrl = "https://example.com/namespace/name", + authors = listOf("John Doe "), + declaredLicenseMapping = emptyMap() + ) ) ) ), diff --git a/core/src/test/kotlin/api/RunsRouteIntegrationTest.kt b/core/src/test/kotlin/api/RunsRouteIntegrationTest.kt index 468ec9ca1a..46abb1e7a2 100644 --- a/core/src/test/kotlin/api/RunsRouteIntegrationTest.kt +++ b/core/src/test/kotlin/api/RunsRouteIntegrationTest.kt @@ -27,6 +27,7 @@ import io.kotest.assertions.ktor.client.shouldHaveStatus import io.kotest.engine.spec.tempdir import io.kotest.inspectors.forAll import io.kotest.matchers.collections.shouldBeSingleton +import io.kotest.matchers.collections.shouldContain import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder import io.kotest.matchers.collections.shouldHaveSize import io.kotest.matchers.file.aFile @@ -126,6 +127,13 @@ import org.eclipse.apoapsis.ortserver.model.runs.advisor.AdvisorResult import org.eclipse.apoapsis.ortserver.model.runs.advisor.Vulnerability import org.eclipse.apoapsis.ortserver.model.runs.advisor.VulnerabilityReference import org.eclipse.apoapsis.ortserver.model.runs.reporter.Report +import org.eclipse.apoapsis.ortserver.model.runs.repository.Curations +import org.eclipse.apoapsis.ortserver.model.runs.repository.Excludes +import org.eclipse.apoapsis.ortserver.model.runs.repository.LicenseChoices +import org.eclipse.apoapsis.ortserver.model.runs.repository.PackageCuration +import org.eclipse.apoapsis.ortserver.model.runs.repository.PackageCurationData +import org.eclipse.apoapsis.ortserver.model.runs.repository.RepositoryAnalyzerConfiguration +import org.eclipse.apoapsis.ortserver.model.runs.repository.Resolutions import org.eclipse.apoapsis.ortserver.model.util.asPresent import org.eclipse.apoapsis.ortserver.services.DefaultAuthorizationService import org.eclipse.apoapsis.ortserver.services.OrganizationService @@ -993,6 +1001,119 @@ class RunsRouteIntegrationTest : AbstractIntegrationTest({ val identifier1 = Identifier("Maven", "com.example", "example", "1.0") val identifier2 = Identifier("Maven", "com.example", "example2", "1.0") + dbExtension.fixtures.repositoryConfigurationRepository.create( + ortRunId = ortRun.id, + curations = Curations( + packages = listOf( + PackageCuration( + id = identifier1, + data = PackageCurationData( + comment = "comment1_a", + description = "description1_a", + concludedLicense = "license1_a", + authors = setOf("auth1a_a", "auth1b_a") + ) + ), + PackageCuration( + id = identifier1, + data = PackageCurationData( + comment = "comment1_b", + description = "description1_b", + concludedLicense = "license1_b", + authors = setOf("auth1a_b", "auth1b_b") + ) + ) + ) + ), + analyzerConfig = RepositoryAnalyzerConfiguration(), + excludes = Excludes(), + resolutions = Resolutions(), + packageConfigurations = listOf(), + licenseChoices = LicenseChoices(), + provenanceSnippetChoices = listOf() + ) + + val package1 = Package( + identifier1, + purl = "pkg:maven/com.example/example@1.0", + cpe = null, + authors = setOf("Author One", "Author Two"), + declaredLicenses = setOf("License1", "License2", "License3"), + ProcessedDeclaredLicense( + spdxExpression = "Expression", + mappedLicenses = mapOf( + "License 1" to "Mapped License 1", + "License 2" to "Mapped License 2", + ), + unmappedLicenses = setOf("License 1", "License 2", "License 3", "License 4") + ), + description = "An example package", + homepageUrl = "https://example.com", + binaryArtifact = RemoteArtifact( + "https://example.com/example-1.0.jar", + "sha1:value", + "SHA-1" + ), + sourceArtifact = RemoteArtifact( + "https://example.com/example-1.0-sources.jar", + "sha1:value", + "SHA-1" + ), + vcs = VcsInfo( + RepositoryType("GIT"), + "https://example.com/git", + "revision", + "path" + ), + vcsProcessed = VcsInfo( + RepositoryType("GIT"), + "https://example.com/git", + "revision", + "path" + ), + isMetadataOnly = false, + isModified = false + ) + + val package2 = Package( + identifier2, + purl = "pkg:maven/com.example/example2@1.0", + cpe = null, + authors = emptySet(), + declaredLicenses = emptySet(), + ProcessedDeclaredLicense( + spdxExpression = "Expression", + mappedLicenses = emptyMap(), + unmappedLicenses = emptySet() + ), + description = "Another example package", + homepageUrl = "https://example.com", + binaryArtifact = RemoteArtifact( + "https://example.com/example2-1.0.jar", + "sha1:value", + "SHA-1" + ), + sourceArtifact = RemoteArtifact( + "https://example.com/example2-1.0-sources.jar", + "sha1:value", + "SHA-1" + ), + vcs = VcsInfo( + RepositoryType("GIT"), + "https://example.com/git", + "revision", + "path" + ), + vcsProcessed = VcsInfo( + RepositoryType("GIT"), + "https://example.com/git", + "revision", + "path" + ), + isMetadataOnly = false, + isModified = false + ) + dbExtension.fixtures.analyzerRunRepository.create( analyzerJobId = analyzerJob.id, startTime = Clock.System.now().toDatabasePrecision(), @@ -1014,87 +1135,7 @@ class RunsRouteIntegrationTest : AbstractIntegrationTest({ skipExcluded = true ), projects = setOf(project), - packages = setOf( - Package( - identifier1, - purl = "pkg:maven/com.example/example@1.0", - cpe = null, - authors = setOf("Author One", "Author Two"), - declaredLicenses = setOf("License1", "License2", "License3"), - ProcessedDeclaredLicense( - spdxExpression = "Expression", - mappedLicenses = mapOf( - "License 1" to "Mapped License 1", - "License 2" to "Mapped License 2", - ), - unmappedLicenses = setOf("License 1", "License 2", "License 3", "License 4") - ), - description = "An example package", - homepageUrl = "https://example.com", - binaryArtifact = RemoteArtifact( - "https://example.com/example-1.0.jar", - "sha1:value", - "SHA-1" - ), - sourceArtifact = RemoteArtifact( - "https://example.com/example-1.0-sources.jar", - "sha1:value", - "SHA-1" - ), - vcs = VcsInfo( - RepositoryType("GIT"), - "https://example.com/git", - "revision", - "path" - ), - vcsProcessed = VcsInfo( - RepositoryType("GIT"), - "https://example.com/git", - "revision", - "path" - ), - isMetadataOnly = false, - isModified = false - ), - Package( - identifier2, - purl = "pkg:maven/com.example/example2@1.0", - cpe = null, - authors = emptySet(), - declaredLicenses = emptySet(), - ProcessedDeclaredLicense( - spdxExpression = "Expression", - mappedLicenses = emptyMap(), - unmappedLicenses = emptySet() - ), - description = "Another example package", - homepageUrl = "https://example.com", - binaryArtifact = RemoteArtifact( - "https://example.com/example2-1.0.jar", - "sha1:value", - "SHA-1" - ), - sourceArtifact = RemoteArtifact( - "https://example.com/example2-1.0-sources.jar", - "sha1:value", - "SHA-1" - ), - vcs = VcsInfo( - RepositoryType("GIT"), - "https://example.com/git", - "revision", - "path" - ), - vcsProcessed = VcsInfo( - RepositoryType("GIT"), - "https://example.com/git", - "revision", - "path" - ), - isMetadataOnly = false, - isModified = false - ) - ), + packages = setOf(package1, package2), issues = emptyList(), dependencyGraphs = emptyMap(), shortestDependencyPaths = mapOf( @@ -1135,14 +1176,39 @@ class RunsRouteIntegrationTest : AbstractIntegrationTest({ it.scope shouldBe "compileClassPath" it.path shouldBe emptyList() } + + curations shouldHaveSize 2 + + val curr1 = curations.find { it.comment == "comment1_a" } + curr1.shouldNotBeNull() + curr1.comment shouldBe "comment1_a" + curr1.description shouldBe "description1_a" + curr1.concludedLicense shouldBe "license1_a" + curr1.authors shouldHaveSize 2 + curr1.authors shouldContain "auth1a_a" + curr1.authors shouldContain "auth1b_a" + + val curr2 = curations.find { it.comment == "comment1_b" } + curr2.shouldNotBeNull() + curr2.comment shouldBe "comment1_b" + curr2.description shouldBe "description1_b" + curr2.concludedLicense shouldBe "license1_b" + curr2.authors shouldHaveSize 2 + curr2.authors shouldContain "auth1a_b" + curr2.authors shouldContain "auth1b_b" } - last().shortestDependencyPaths.shouldBeSingleton { - it.projectIdentifier shouldBe project.identifier.mapToApi() - it.scope shouldBe "compileClassPath" - it.path shouldBe listOf(identifier1.mapToApi()) + with(last()) { + identifier.name shouldBe "example2" + + shortestDependencyPaths.shouldBeSingleton { + it.projectIdentifier shouldBe project.identifier.mapToApi() + it.scope shouldBe "compileClassPath" + it.path shouldBe listOf(identifier1.mapToApi()) + } + + curations shouldHaveSize(0) } - last().identifier.name shouldBe "example2" } } } diff --git a/dao/src/testFixtures/kotlin/Fixtures.kt b/dao/src/testFixtures/kotlin/Fixtures.kt index 36d57e432c..c1a3d81f1b 100644 --- a/dao/src/testFixtures/kotlin/Fixtures.kt +++ b/dao/src/testFixtures/kotlin/Fixtures.kt @@ -77,6 +77,7 @@ import org.jetbrains.exposed.sql.Database * A helper class to manage test fixtures. It provides default instances as well as helper functions to create custom * instances. */ +@Suppress("TooManyFunctions") class Fixtures(private val db: Database) { val advisorJobRepository = DaoAdvisorJobRepository(db) val advisorRunRepository = DaoAdvisorRunRepository(db) diff --git a/model/src/commonMain/kotlin/runs/PackageWithShortestDependencyPaths.kt b/model/src/commonMain/kotlin/runs/PackageRunData.kt similarity index 80% rename from model/src/commonMain/kotlin/runs/PackageWithShortestDependencyPaths.kt rename to model/src/commonMain/kotlin/runs/PackageRunData.kt index 64a3c1691a..d24849324f 100644 --- a/model/src/commonMain/kotlin/runs/PackageWithShortestDependencyPaths.kt +++ b/model/src/commonMain/kotlin/runs/PackageRunData.kt @@ -19,11 +19,13 @@ package org.eclipse.apoapsis.ortserver.model.runs +import org.eclipse.apoapsis.ortserver.model.runs.repository.PackageCurationData + /** * A data class representing a package, and the shortest dependency path that the package is found in (relative to a * project found in a run). */ -data class PackageWithShortestDependencyPaths( +data class PackageRunData( /** A package. */ val pkg: Package, @@ -31,5 +33,8 @@ data class PackageWithShortestDependencyPaths( val pkgId: Long, /** The shortest dependency path for the package. */ - val shortestDependencyPaths: List + val shortestDependencyPaths: List, + + /** The curations for the package. */ + val curations: List = emptyList() ) diff --git a/services/hierarchy/src/main/kotlin/PackageService.kt b/services/hierarchy/src/main/kotlin/PackageService.kt index ccc57fc287..2823575735 100644 --- a/services/hierarchy/src/main/kotlin/PackageService.kt +++ b/services/hierarchy/src/main/kotlin/PackageService.kt @@ -29,27 +29,27 @@ import org.eclipse.apoapsis.ortserver.dao.repositories.analyzerrun.PackagesTable import org.eclipse.apoapsis.ortserver.dao.repositories.analyzerrun.ProcessedDeclaredLicensesTable import org.eclipse.apoapsis.ortserver.dao.repositories.analyzerrun.ShortestDependencyPathDao import org.eclipse.apoapsis.ortserver.dao.repositories.analyzerrun.ShortestDependencyPathsTable +import org.eclipse.apoapsis.ortserver.dao.repositories.repositoryconfiguration.PackageCurationDataDao +import org.eclipse.apoapsis.ortserver.dao.repositories.repositoryconfiguration.PackageCurationDataTable +import org.eclipse.apoapsis.ortserver.dao.repositories.repositoryconfiguration.PackageCurationsTable +import org.eclipse.apoapsis.ortserver.dao.repositories.repositoryconfiguration.RepositoryConfigurationsPackageCurationsTable +import org.eclipse.apoapsis.ortserver.dao.repositories.repositoryconfiguration.RepositoryConfigurationsTable import org.eclipse.apoapsis.ortserver.dao.tables.shared.IdentifiersTable -import org.eclipse.apoapsis.ortserver.dao.utils.applyFilterNullable -import org.eclipse.apoapsis.ortserver.dao.utils.applyILike -import org.eclipse.apoapsis.ortserver.dao.utils.listCustomQueryCustomOrders import org.eclipse.apoapsis.ortserver.model.EcosystemStats +import org.eclipse.apoapsis.ortserver.model.runs.Identifier +import org.eclipse.apoapsis.ortserver.model.runs.Package import org.eclipse.apoapsis.ortserver.model.runs.PackageFilters -import org.eclipse.apoapsis.ortserver.model.runs.PackageWithShortestDependencyPaths +import org.eclipse.apoapsis.ortserver.model.runs.PackageRunData +import org.eclipse.apoapsis.ortserver.model.runs.repository.PackageCurationData import org.eclipse.apoapsis.ortserver.model.util.ListQueryParameters import org.eclipse.apoapsis.ortserver.model.util.ListQueryResult +import org.eclipse.apoapsis.ortserver.model.util.OrderDirection +import org.eclipse.apoapsis.ortserver.model.util.OrderField -import org.jetbrains.exposed.sql.Case import org.jetbrains.exposed.sql.Count import org.jetbrains.exposed.sql.Database -import org.jetbrains.exposed.sql.Expression -import org.jetbrains.exposed.sql.Op -import org.jetbrains.exposed.sql.ResultRow -import org.jetbrains.exposed.sql.SortOrder -import org.jetbrains.exposed.sql.SqlExpressionBuilder.concat -import org.jetbrains.exposed.sql.SqlExpressionBuilder.neq -import org.jetbrains.exposed.sql.and -import org.jetbrains.exposed.sql.stringLiteral +import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq +import org.jetbrains.exposed.sql.andWhere /** * A service to interact with packages. @@ -59,81 +59,204 @@ class PackageService(private val db: Database) { ortRunId: Long, parameters: ListQueryParameters = ListQueryParameters.DEFAULT, filters: PackageFilters = PackageFilters() - ): ListQueryResult = db.dbQuery { - val orders = mutableListOf, SortOrder>>() - - parameters.sortFields.forEach { - val sortOrder = it.direction.toSortOrder() - when (it.name) { - "identifier" -> { - orders += IdentifiersTable.type to sortOrder - orders += IdentifiersTable.namespace to sortOrder - orders += IdentifiersTable.name to sortOrder - orders += IdentifiersTable.version to sortOrder - } - "purl" -> orders += PackagesTable.purl to sortOrder - "processedDeclaredLicense" -> orders += ProcessedDeclaredLicensesTable.spdxExpression to sortOrder - else -> throw QueryParametersException("Unsupported field for sorting: '${it.name}'.") - } - } + ): ListQueryResult = db.dbQuery { + val packages = PackagesTable.joinAnalyzerTables() + .innerJoin(IdentifiersTable) + .innerJoin(ProcessedDeclaredLicensesTable) + .select(PackagesTable.columns) + .where { AnalyzerJobsTable.ortRunId eq ortRunId } + + val curations = RepositoryConfigurationsTable + .innerJoin(RepositoryConfigurationsPackageCurationsTable) + .innerJoin(PackageCurationsTable) + .innerJoin(PackageCurationDataTable) + .innerJoin(IdentifiersTable) + .innerJoin(PackagesTable) + .select(listOf(PackagesTable.id, PackageCurationsTable.id) + PackageCurationDataTable.columns) + .where(RepositoryConfigurationsTable.ortRunId eq ortRunId) + .andWhere { PackagesTable.id inList (packages.map { it[PackagesTable.id].value }) } + .distinct() + .groupBy { it[PackagesTable.id].value } + .mapValues { rows -> rows.value.map { PackageCurationDataDao.wrapRow(it).mapToModel() } } + + val shortestPaths = ShortestDependencyPathsTable + .innerJoin(PackagesTable) + .innerJoin(AnalyzerRunsTable) + .innerJoin(AnalyzerJobsTable) + .select(ShortestDependencyPathsTable.columns.plus(PackagesTable.id)) + .where(AnalyzerJobsTable.ortRunId eq ortRunId) + .andWhere { PackagesTable.id inList (packages.map { it[PackagesTable.id].value }) } + .groupBy { it[PackagesTable.id].value } + .mapValues { rows -> rows.value.map { ShortestDependencyPathDao.wrapRow(it).mapToModel() } } + + val packageResults = packages.map { pkg -> + val pkgId = pkg[PackagesTable.id].value + PackageRunData( + pkgId = pkgId, + pkg = applyCuration( + pkg = PackageDao.wrapRow(pkg).mapToModel(), + curationData = curations.getOrDefault(pkgId, emptyList()).firstOrNull() ?: PackageCurationData() + ), + shortestDependencyPaths = shortestPaths.getOrDefault(pkgId, emptyList()) + ) + }.filter(filters).sort(parameters.sortFields) + + ListQueryResult( + limitPackageResults(parameters.limit, parameters.offset, packageResults), + parameters, + packageResults.size.toLong() + ) + } - var condition: Op = Op.TRUE + private fun applyCuration(pkg: Package, curationData: PackageCurationData): Package { + return Package( + purl = curationData.purl ?: pkg.purl, + cpe = curationData.cpe ?: pkg.cpe, + authors = curationData.authors ?: pkg.authors, + declaredLicenses = pkg.declaredLicenses, + description = curationData.description ?: pkg.description, + homepageUrl = curationData.homepageUrl ?: pkg.homepageUrl, + binaryArtifact = curationData.binaryArtifact ?: pkg.binaryArtifact, + sourceArtifact = curationData.sourceArtifact ?: pkg.sourceArtifact, + vcs = pkg.vcs, + vcsProcessed = pkg.vcsProcessed, + isMetadataOnly = curationData.isMetadataOnly ?: pkg.isMetadataOnly, + isModified = curationData.isModified ?: pkg.isModified, + identifier = pkg.identifier, + processedDeclaredLicense = pkg.processedDeclaredLicense + ) + } + + private fun List.filter(filters: PackageFilters): List { + var filtered = this filters.purl?.let { - condition = condition and PackagesTable.purl.applyILike(it.value) + filtered = filtered.filter{ pkg -> + pkg.pkg.purl.contains(filters.purl?.value?.trim().toString(), ignoreCase = true) + } } filters.identifier?.let { - val namespaceWithSlash = Case() - .When( - IdentifiersTable.namespace neq stringLiteral(""), - concat(IdentifiersTable.namespace, stringLiteral("/")) - ) - .Else(stringLiteral("")) - - val concatenatedIdentifier = concat( - IdentifiersTable.type, - stringLiteral(":"), - namespaceWithSlash, - IdentifiersTable.name, - stringLiteral("@"), - IdentifiersTable.version - ) - - condition = condition and concatenatedIdentifier.applyILike(it.value) + filtered = filtered.filter { pkg -> + pkg.pkg.identifier + .concatenate() + .contains(filters.identifier?.value?.trim().toString(), ignoreCase = true) + } } filters.processedDeclaredLicense?.let { - condition = condition and ProcessedDeclaredLicensesTable.spdxExpression.applyFilterNullable( - it.operator, - it.value - ) + filtered = filtered.filter { pkg -> + filters.processedDeclaredLicense?.value?.joinToString(prefix = "(?i)", separator = "|")?.toRegex() + ?.let { regex -> + pkg.pkg.processedDeclaredLicense.spdxExpression?.contains(regex)?.or(false) + } == true + } } + return filtered + } - val listQueryResult = - listCustomQueryCustomOrders(parameters, orders, ResultRow::toPackageWithShortestDependencyPaths) { - PackagesTable.joinAnalyzerTables() - .innerJoin(IdentifiersTable) - .innerJoin(ProcessedDeclaredLicensesTable) - .select(PackagesTable.columns) - .where { (AnalyzerJobsTable.ortRunId eq ortRunId) and condition } + private fun List.sort(sortFields: List): List { + val comparators = mutableListOf>() + sortFields.forEach { sortParam -> + when (sortParam.name) { + "purl" -> comparators.add(getPurlComparator(sortParam.direction)) + "identifier" -> comparators.addAll(getIdentifierComparators(sortParam.direction)) + "processedDeclaredLicense" -> comparators.add( + getProcessedDeclaredLicenseComparator(sortParam.direction) + ) + else -> throw QueryParametersException("Unsupported field for sorting: '${sortParam.name}'.") } + } + return this.sortedWith(getMultistageComparator(comparators)) + } - val data = listQueryResult.data.map { pkg -> - val shortestPaths = ShortestDependencyPathsTable - .innerJoin(PackagesTable) - .innerJoin(AnalyzerRunsTable) - .innerJoin(AnalyzerJobsTable) - .select(ShortestDependencyPathsTable.columns) - .where { (AnalyzerJobsTable.ortRunId eq ortRunId) and (PackagesTable.id eq pkg.pkgId) } - .map { ShortestDependencyPathDao.wrapRow(it).mapToModel() } - - pkg.copy( - shortestDependencyPaths = shortestPaths - ) + private fun limitPackageResults( + limit: Int?, + offset: Long?, + packageResults: List + ): List { + val listOffset = (offset ?: 0).toInt() + val listLimit = (limit ?: packageResults.size).toInt() + listOffset + return packageResults.subList(listOffset, listLimit) + } + + private fun Identifier.concatenate(): String = + "${type}:${if (namespace.isEmpty()) "" else "$namespace/"}" + + "${name}@${version}" + + private fun getNullComparator(): Comparator { + return Comparator { a, b -> 0 } + } + + private fun getPurlComparator(dir: OrderDirection): Comparator { + return Comparator { a, b -> + when (dir) { + OrderDirection.ASCENDING -> a.pkg.purl.compareTo(b.pkg.purl, ignoreCase = true) + OrderDirection.DESCENDING -> b.pkg.purl.compareTo(a.pkg.purl, ignoreCase = true) + } } + } + + private fun getIdentifierComparators(dir: OrderDirection): List> { + when (dir) { + OrderDirection.ASCENDING -> { + return listOf( + Comparator { a, b -> + a.pkg.identifier.type.compareTo(b.pkg.identifier.type, ignoreCase = true) + }, + Comparator { a, b -> + a.pkg.identifier.namespace.compareTo(b.pkg.identifier.namespace, ignoreCase = true) + }, + Comparator { a, b -> + a.pkg.identifier.name.compareTo(b.pkg.identifier.name, ignoreCase = true) + }, + Comparator { a, b -> + a.pkg.identifier.version.compareTo(b.pkg.identifier.version, ignoreCase = true) + } + ) + } - ListQueryResult(data, parameters, listQueryResult.totalCount) + OrderDirection.DESCENDING -> { + return listOf( + Comparator { a, b -> + b.pkg.identifier.type.compareTo(a.pkg.identifier.type, ignoreCase = true) + }, + Comparator { a, b -> + b.pkg.identifier.namespace.compareTo(a.pkg.identifier.namespace, ignoreCase = true) + }, + Comparator { a, b -> + b.pkg.identifier.name.compareTo(a.pkg.identifier.name, ignoreCase = true) + }, + Comparator { a, b -> + b.pkg.identifier.version.compareTo(a.pkg.identifier.version, ignoreCase = true) + } + ) + } + } + } + + private fun getProcessedDeclaredLicenseComparator(dir: OrderDirection): Comparator { + return Comparator { a, b -> + when (dir) { + OrderDirection.ASCENDING -> a.pkg.processedDeclaredLicense.spdxExpression.toString() + .compareTo(b.pkg.processedDeclaredLicense.spdxExpression.toString(), ignoreCase = true) + OrderDirection.DESCENDING -> b.pkg.processedDeclaredLicense.spdxExpression.toString() + .compareTo(a.pkg.processedDeclaredLicense.spdxExpression.toString(), ignoreCase = true) + } + } + } + + private fun getMultistageComparator(comparators: List>): Comparator { + if (comparators.isEmpty()) { + return getNullComparator() + } else { + val multiStageComparator = comparators[0] + for (comparator in comparators.drop(1)) { + multiStageComparator.then(comparator) + } + + return multiStageComparator + } } /** Count packages found in provided ORT runs. */ @@ -175,14 +298,6 @@ class PackageService(private val db: Database) { } } -private fun ResultRow.toPackageWithShortestDependencyPaths(): PackageWithShortestDependencyPaths = - PackageWithShortestDependencyPaths( - pkg = PackageDao.wrapRow(this).mapToModel(), - pkgId = get(PackagesTable.id).value, - // Temporarily set the shortestDependencyPaths into an empty list, as they will be added in a subsequent step. - shortestDependencyPaths = emptyList() - ) - private fun PackagesTable.joinAnalyzerTables() = innerJoin(PackagesAnalyzerRunsTable) .innerJoin(AnalyzerRunsTable) diff --git a/services/hierarchy/src/test/kotlin/PackageServiceTest.kt b/services/hierarchy/src/test/kotlin/PackageServiceTest.kt index fe3677d65d..711bd9ddd7 100644 --- a/services/hierarchy/src/test/kotlin/PackageServiceTest.kt +++ b/services/hierarchy/src/test/kotlin/PackageServiceTest.kt @@ -24,6 +24,7 @@ import io.kotest.matchers.collections.beEmpty import io.kotest.matchers.collections.containExactlyInAnyOrder import io.kotest.matchers.collections.shouldContainInOrder import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.nulls.shouldNotBeNull import io.kotest.matchers.should import io.kotest.matchers.shouldBe @@ -39,6 +40,13 @@ import org.eclipse.apoapsis.ortserver.model.runs.ProcessedDeclaredLicense import org.eclipse.apoapsis.ortserver.model.runs.Project import org.eclipse.apoapsis.ortserver.model.runs.RemoteArtifact import org.eclipse.apoapsis.ortserver.model.runs.ShortestDependencyPath +import org.eclipse.apoapsis.ortserver.model.runs.repository.Curations +import org.eclipse.apoapsis.ortserver.model.runs.repository.Excludes +import org.eclipse.apoapsis.ortserver.model.runs.repository.LicenseChoices +import org.eclipse.apoapsis.ortserver.model.runs.repository.PackageCuration +import org.eclipse.apoapsis.ortserver.model.runs.repository.PackageCurationData +import org.eclipse.apoapsis.ortserver.model.runs.repository.RepositoryAnalyzerConfiguration +import org.eclipse.apoapsis.ortserver.model.runs.repository.Resolutions import org.eclipse.apoapsis.ortserver.model.util.ComparisonOperator import org.eclipse.apoapsis.ortserver.model.util.FilterOperatorAndValue import org.eclipse.apoapsis.ortserver.model.util.ListQueryParameters @@ -148,8 +156,8 @@ class PackageServiceTest : WordSpec() { results.data shouldHaveSize 2 results.totalCount shouldBe 3 - results.data.first().pkg.identifier.name shouldBe "example3" - results.data.last().pkg.identifier.name shouldBe "example2" + results.data.first().pkg.identifier.name shouldBe "example" + results.data.last().pkg.identifier.name shouldBe "example3" } "allow sorting by identifier" { @@ -163,9 +171,9 @@ class PackageServiceTest : WordSpec() { val ortRunId = createAnalyzerRunWithPackages( setOf( - fixtures.generatePackage(identifier1), fixtures.generatePackage(identifier2), fixtures.generatePackage(identifier3), + fixtures.generatePackage(identifier1), fixtures.generatePackage(identifier4), fixtures.generatePackage(identifier5) ) @@ -289,33 +297,110 @@ class PackageServiceTest : WordSpec() { results.data shouldHaveSize 2 with(results.data.first()) { - pkg.identifier shouldBe identifier2 + pkg.identifier shouldBe identifier1 shortestDependencyPaths shouldBe listOf( ShortestDependencyPath( project1.identifier, "compileClassPath", - listOf(identifier1) + emptyList() + ), + ShortestDependencyPath( + project2.identifier, + "compileClassPath", + emptyList() ) ) } with(results.data.last()) { - pkg.identifier shouldBe identifier1 + pkg.identifier shouldBe identifier2 shortestDependencyPaths shouldBe listOf( ShortestDependencyPath( project1.identifier, "compileClassPath", - emptyList() - ), - ShortestDependencyPath( - project2.identifier, - "compileClassPath", - emptyList() + listOf(identifier1) ) ) } } + "return package curations" { + val service = PackageService(db) + + val project1 = fixtures.getProject() + val project2 = fixtures.getProject(Identifier("Gradle", "", "project2", "1.0")) + + val identifier1 = Identifier("Maven", "com.example", "example", "1.0") + val identifier2 = Identifier("Maven", "com.example", "example2", "1.0") + + val ortRunId = createAnalyzerRunWithPackages( + projects = setOf(project1, project2), + packages = setOf( + fixtures.generatePackage(identifier1), + fixtures.generatePackage(identifier2) + ) + ).id + + fixtures.repositoryConfigurationRepository.create( + ortRunId = ortRunId, + curations = Curations( + packages = listOf( + PackageCuration( + id = identifier1, + data = PackageCurationData( + comment = "comment1_a", + description = "description1_a", + concludedLicense = "license1_a", + authors = setOf("auth1a_a", "auth1b_a") + ) + ), + PackageCuration( + id = identifier1, + data = PackageCurationData( + comment = "comment1_b", + description = "description1_b", + concludedLicense = "license1_b", + authors = setOf("auth1a_b", "auth1b_b") + ) + ) + ) + ), + analyzerConfig = RepositoryAnalyzerConfiguration(), + excludes = Excludes(), + resolutions = Resolutions(), + packageConfigurations = listOf(), + licenseChoices = LicenseChoices(), + provenanceSnippetChoices = listOf() + ) + + val results = service.listForOrtRunId( + ortRunId, + ListQueryParameters(listOf(OrderField("purl", OrderDirection.DESCENDING))) + ) + + results.data shouldHaveSize 2 + + with(results.data.first()) { + pkg.identifier shouldBe identifier1 + curations shouldHaveSize 0 + } + + with(results.data.last()) { + pkg.identifier shouldBe identifier2 + val curr1 = curations.find { it.comment == "comment1_a" } + curr1.shouldNotBeNull() + curr1.comment shouldBe "comment1_a" + curr1.description shouldBe "description1_a" + curr1.concludedLicense shouldBe "license1_a" + + val curr2 = curations.find { it.comment == "comment1_b" } + curr2.shouldNotBeNull() + curr2.comment shouldBe "comment1_b" + curr2.description shouldBe "description1_b" + curr2.concludedLicense shouldBe "license1_b" + } + } + "allow filtering by identifier" { val service = PackageService(db)