Skip to content

Commit 1700252

Browse files
dkrasnoffdkrasnoff
committed
KTL-4388: Moved out to a separate table maven artifact data.
Updated the package's dependency counting with this new database hierarchy.
1 parent 6fa7f4c commit 1700252

32 files changed

Lines changed: 980 additions & 160 deletions

‎app/src/main/kotlin/io/klibs/app/indexing/PackageIndexingService.kt‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@ import io.klibs.app.util.toIndexRequest
66
import io.klibs.core.pckg.dto.PackageDTO
77
import io.klibs.core.pckg.entity.IndexingRequestEntity
88
import io.klibs.core.pckg.enums.IndexingRequestStatus
9+
import io.klibs.core.pckg.dto.MavenCoordinatesDTO
910
import io.klibs.core.pckg.enums.VersionType
1011
import io.klibs.core.pckg.repository.IndexingRequestRepository
1112
import io.klibs.core.pckg.repository.PackageRepository
13+
import io.klibs.core.pckg.service.MavenArtifactService
1214
import io.klibs.core.pckg.service.PackageService
1315
import io.klibs.core.project.ProjectEntity
1416
import io.klibs.core.scm.repository.ScmRepositoryEntity
@@ -45,6 +47,7 @@ class PackageIndexingService(
4547
private val indexingRequestRepository: IndexingRequestRepository,
4648
private val packageService: PackageService,
4749
private val packageRepository: PackageRepository,
50+
private val mavenArtifactService: MavenArtifactService,
4851
private val selfProvider: ObjectProvider<PackageIndexingService>
4952
) {
5053

@@ -176,7 +179,10 @@ class PackageIndexingService(
176179
?: error("Unable to update a non-existing artifact: $mavenArtifact")
177180
updated.id
178181
} else {
179-
packageRepository.save(packageDto.toEntity()).id
182+
val mavenArtifactDto = mavenArtifactService.resolveOrCreate(
183+
MavenCoordinatesDTO(pom.groupId, pom.artifactId, pom.version)
184+
)
185+
packageRepository.save(packageDto.toEntity(mavenArtifactDto)).id
180186
}
181187

182188
logger.trace("Extracting dependencies for {}", indexRequest)

‎app/src/main/kotlin/io/klibs/app/indexing/PomIndexingService.kt‎

Lines changed: 22 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,15 @@ package io.klibs.app.indexing
33
import io.klibs.app.util.ANDROIDX_OWNER_AND_GITHUB_REPOSITORY
44
import io.klibs.app.util.isAndroidxProject
55
import io.klibs.app.util.parseGitHubLink
6-
import io.klibs.core.pckg.entity.PackageDependencyEntity
7-
import io.klibs.core.pckg.entity.PackageDependencyKey
6+
import io.klibs.core.pckg.dto.MavenCoordinatesDTO
87
import io.klibs.core.pckg.model.PackageDeveloper
98
import io.klibs.core.pckg.model.PackageLicense
10-
import io.klibs.core.pckg.repository.PackageDependencyRepository
9+
import io.klibs.core.pckg.repository.PackageRepository
10+
import io.klibs.core.pckg.service.MavenArtifactService
1111
import io.klibs.integration.maven.MavenPom
1212
import org.slf4j.LoggerFactory
1313
import org.springframework.stereotype.Service
14+
import org.springframework.transaction.annotation.Transactional
1415

1516
/**
1617
* Owns everything derived from a package's POM during indexing: pure extractors for fields
@@ -23,7 +24,8 @@ import org.springframework.stereotype.Service
2324
*/
2425
@Service
2526
class PomIndexingService(
26-
private val packageDependencyRepository: PackageDependencyRepository,
27+
private val packageRepository: PackageRepository,
28+
private val mavenArtifactService: MavenArtifactService,
2729
) {
2830

2931
/**
@@ -62,49 +64,43 @@ class PomIndexingService(
6264
}
6365
}
6466

67+
@Transactional
6568
fun indexDependencies(pom: MavenPom, packageId: Long, isReindex: Boolean) {
6669
val dependencies = pom.extractDependencies()
6770

68-
if (isReindex) {
69-
packageDependencyRepository.deleteAllByIdPackageId(packageId)
71+
if (!isReindex && dependencies.isEmpty()) return
72+
73+
val packageEntity = requireNotNull(packageRepository.findById(packageId).orElse(null)) {
74+
"Package with id=$packageId not found while indexing dependencies"
7075
}
7176

72-
if (dependencies.isEmpty()) return
77+
if (isReindex) {
78+
packageEntity.dependencies.clear()
79+
}
7380

74-
val entities = dependencies.map { coords ->
75-
PackageDependencyEntity(
76-
id = PackageDependencyKey(
77-
packageId = packageId,
78-
depGroupId = coords.groupId,
79-
depArtifactId = coords.artifactId,
80-
depVersion = coords.version,
81-
)
82-
)
81+
if (dependencies.isNotEmpty()) {
82+
val artifactsByCoords = mavenArtifactService.resolveOrCreateAll(dependencies)
83+
packageEntity.dependencies.addAll(artifactsByCoords.values.map { it.toEntityRef() })
8384
}
84-
packageDependencyRepository.saveAll(entities)
85-
logger.debug("Saved {} dependencies for package id={}", entities.size, packageId)
85+
86+
packageRepository.save(packageEntity)
87+
logger.debug("Saved {} dependencies for package id={}", packageEntity.dependencies.size, packageId)
8688
}
8789

88-
private fun MavenPom.extractDependencies(): Set<DependencyCoordinates> =
90+
private fun MavenPom.extractDependencies(): Set<MavenCoordinatesDTO> =
8991
dependencies
9092
?.asSequence()
9193
?.mapNotNull { dep ->
9294
val group = dep.groupId?.takeIf { it.isNotBlank() } ?: return@mapNotNull null
9395
val artifact = dep.artifactId?.takeIf { it.isNotBlank() } ?: return@mapNotNull null
9496
val version = dep.version?.takeIf { it.isNotBlank() && !it.contains("\${") }
9597
?: return@mapNotNull null
96-
DependencyCoordinates(groupId = group, artifactId = artifact, version = version)
98+
MavenCoordinatesDTO(groupId = group, artifactId = artifact, version = version)
9799
}
98100
?.filterNot { coords -> groupId == coords.groupId && artifactId == coords.artifactId }
99101
?.toSet()
100102
.orEmpty()
101103

102-
private data class DependencyCoordinates(
103-
val groupId: String,
104-
val artifactId: String,
105-
val version: String,
106-
)
107-
108104
private companion object {
109105
private val logger = LoggerFactory.getLogger(PomIndexingService::class.java)
110106
}

‎app/src/main/resources/application.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ klibs:
6666
rate-limit-refill-period-sec: ${KLIBS_MAVEN_RATE_LIMIT_REFILL_PERIOD:5}
6767
index-endpoint: ${KLIBS_MAVEN_INDEX_ENDPOINT:https://repo1.maven.org/maven2}
6868
index-dir: ${KLIBS_MAVEN_INDEX_CACHE_DIR:${user.dir}/cache/maven-index}
69-
content-endpoint: ${KLIBS_MAVEN_CONTENT_ENDPOINT:https://cache-redirector.jetbrains.com/repo1.maven.org/maven2/}
69+
content-endpoint: ${KLIBS_MAVEN_CONTENT_ENDPOINT:https://repo1.maven.org/maven2/}
7070
github:
7171
personal-access-token: ${KLIBS_GITHUB_TOKEN}
7272
cache:
@@ -76,7 +76,7 @@ klibs:
7676
indexing: ${KLIBS_ENABLE_INDEXING:false}
7777
scheduling:
7878
process-indexing-queue:
79-
enabled: true
79+
enabled: ${PROCESS_INDEXING_QUEUE_ENABLED:true}
8080
indexing-configuration:
8181
executor:
8282
thread-count: 2

‎app/src/main/resources/db/migration/2026-Q2/2026-04-23_create_maven_artifact.yml‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
databaseChangeLog:
2+
- changeSet:
3+
id: create maven_artifact table and seed it from package
4+
author: dmitrii.krasnov@jetbrains.com
5+
preConditions:
6+
- onFail: MARK_RAN
7+
- not:
8+
tableExists:
9+
tableName: maven_artifact
10+
changes:
11+
- createTable:
12+
tableName: maven_artifact
13+
columns:
14+
- column:
15+
name: id
16+
type: BIGINT
17+
constraints:
18+
nullable: false
19+
primaryKey: true
20+
primaryKeyName: pk_maven_artifact
21+
- column:
22+
name: group_id
23+
type: TEXT
24+
constraints:
25+
nullable: false
26+
- column:
27+
name: artifact_id
28+
type: TEXT
29+
constraints:
30+
nullable: false
31+
- column:
32+
name: version
33+
type: TEXT
34+
constraints:
35+
nullable: false
36+
- sql:
37+
sql: |
38+
INSERT INTO maven_artifact (id, group_id, artifact_id, version)
39+
SELECT
40+
row_number() OVER (ORDER BY group_id, artifact_id, version) AS id,
41+
group_id, artifact_id, version
42+
FROM (
43+
SELECT DISTINCT group_id, artifact_id, version FROM package
44+
) g;
45+
46+
- changeSet:
47+
id: add unique index on maven_artifact gav
48+
author: nikita.vlaev@jetbrains.com
49+
preConditions:
50+
- onFail: MARK_RAN
51+
- not:
52+
indexExists:
53+
tableName: maven_artifact
54+
indexName: uindex_maven_artifact_gav
55+
changes:
56+
- createIndex:
57+
tableName: maven_artifact
58+
indexName: uindex_maven_artifact_gav
59+
unique: true
60+
columns:
61+
- column:
62+
name: group_id
63+
- column:
64+
name: artifact_id
65+
- column:
66+
name: version
67+
68+
- changeSet:
69+
id: add ga index on maven_artifact for reverse lookup
70+
author: dmitrii.krasnov@jetbrains.com
71+
preConditions:
72+
- onFail: MARK_RAN
73+
- not:
74+
indexExists:
75+
tableName: maven_artifact
76+
indexName: idx_maven_artifact_ga
77+
changes:
78+
- createIndex:
79+
tableName: maven_artifact
80+
indexName: idx_maven_artifact_ga
81+
columns:
82+
- column:
83+
name: group_id
84+
- column:
85+
name: artifact_id
86+
87+
- changeSet:
88+
id: create maven_artifact sequence
89+
author: dmitrii.krasnov@jetbrains.com
90+
preConditions:
91+
- onFail: MARK_RAN
92+
- not:
93+
sequenceExists:
94+
sequenceName: maven_artifact_id_seq
95+
changes:
96+
- sql:
97+
sql: |
98+
CREATE SEQUENCE maven_artifact_id_seq
99+
AS BIGINT
100+
INCREMENT BY 50
101+
START WITH 1;
102+
103+
SELECT setval('maven_artifact_id_seq',COALESCE((SELECT MAX(id) FROM maven_artifact), 0), true );
104+
105+
- changeSet:
106+
id: add package.maven_artifact_id column
107+
author: dmitrii.krasnov@jetbrains.com
108+
preConditions:
109+
- onFail: MARK_RAN
110+
- not:
111+
columnExists:
112+
tableName: package
113+
columnName: maven_artifact_id
114+
changes:
115+
- addColumn:
116+
tableName: package
117+
columns:
118+
- column:
119+
name: maven_artifact_id
120+
type: BIGINT
121+
constraints:
122+
nullable: true
123+
124+
- changeSet:
125+
id: backfill package.maven_artifact_id from maven_artifact
126+
author: dmitrii.krasnov@jetbrains.com
127+
preConditions:
128+
- onFail: CONTINUE
129+
- tableExists:
130+
tableName: package
131+
- tableExists:
132+
tableName: maven_artifact
133+
- columnExists:
134+
tableName: package
135+
columnName: maven_artifact_id
136+
changes:
137+
- sql:
138+
sql: |
139+
UPDATE package p
140+
SET maven_artifact_id = ma.id
141+
FROM maven_artifact ma
142+
WHERE ma.group_id = p.group_id
143+
AND ma.artifact_id = p.artifact_id
144+
AND ma.version = p.version
145+
AND p.maven_artifact_id IS NULL;
146+
147+
- changeSet:
148+
id: add package.maven_artifact_id foreign key
149+
author: dmitrii.krasnov@jetbrains.com
150+
preConditions:
151+
- onFail: MARK_RAN
152+
- not:
153+
foreignKeyConstraintExists:
154+
foreignKeyName: fk_package_maven_artifact_id
155+
changes:
156+
- addForeignKeyConstraint:
157+
constraintName: fk_package_maven_artifact_id
158+
baseTableName: package
159+
baseColumnNames: maven_artifact_id
160+
referencedTableName: maven_artifact
161+
referencedColumnNames: id
162+
163+
- changeSet:
164+
id: add index on package.maven_artifact_id
165+
author: dmitrii.krasnov@jetbrains.com
166+
preConditions:
167+
- onFail: MARK_RAN
168+
- not:
169+
indexExists:
170+
tableName: package
171+
indexName: idx_package_maven_artifact_id
172+
changes:
173+
- createIndex:
174+
tableName: package
175+
indexName: idx_package_maven_artifact_id
176+
columns:
177+
- column:
178+
name: maven_artifact_id
179+
180+
- changeSet:
181+
id: make package.maven_artifact_id NOT NULL
182+
author: dmitrii.krasnov@jetbrains.com
183+
preConditions:
184+
- onFail: HALT
185+
- columnExists:
186+
tableName: package
187+
columnName: maven_artifact_id
188+
- sqlCheck:
189+
expectedResult: "0"
190+
sql: SELECT COUNT(*) FROM package WHERE maven_artifact_id IS NULL
191+
changes:
192+
- addNotNullConstraint:
193+
tableName: package
194+
columnName: maven_artifact_id
195+
columnDataType: BIGINT

‎app/src/main/resources/db/migration/2026-Q2/2026-04-23_create_package_dependency.yml‎

Lines changed: 9 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -18,24 +18,17 @@ databaseChangeLog:
1818
nullable: false
1919
references: package(id)
2020
foreignKeyName: fk_package_dependency_package_id
21+
deleteCascade: true
2122
- column:
22-
name: dep_group_id
23-
type: TEXT
24-
constraints:
25-
nullable: false
26-
- column:
27-
name: dep_artifact_id
28-
type: TEXT
29-
constraints:
30-
nullable: false
31-
- column:
32-
name: dep_version
33-
type: TEXT
23+
name: dep_maven_artifact_id
24+
type: BIGINT
3425
constraints:
3526
nullable: false
27+
references: maven_artifact(id)
28+
foreignKeyName: fk_package_dependency_dep_maven_artifact_id
3629
- addPrimaryKey:
3730
tableName: package_dependency
38-
columnNames: package_id, dep_group_id, dep_artifact_id, dep_version
31+
columnNames: package_id, dep_maven_artifact_id
3932
constraintName: pk_package_dependency
4033

4134
- changeSet:
@@ -46,13 +39,11 @@ databaseChangeLog:
4639
- not:
4740
indexExists:
4841
tableName: package_dependency
49-
indexName: idx_package_dependency_target
42+
indexName: idx_package_dependency_dep_maven_artifact_id
5043
changes:
5144
- createIndex:
5245
tableName: package_dependency
53-
indexName: idx_package_dependency_target
46+
indexName: idx_package_dependency_dep_maven_artifact_id
5447
columns:
5548
- column:
56-
name: dep_group_id
57-
- column:
58-
name: dep_artifact_id
49+
name: dep_maven_artifact_id

0 commit comments

Comments
 (0)