Skip to content

Commit 9e36a0c

Browse files
mlachkarmlachkar
authored
Merge pull request #993 from mlachkar/release-table
Release and releaseDependency tables
2 parents 2f68dd2 + 14ac75f commit 9e36a0c

26 files changed

+298
-64
lines changed

‎modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala

+3
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ case class Artifact(
3838

3939
val mavenReference: Artifact.MavenReference = Artifact.MavenReference(groupId.value, artifactId, version.encode)
4040

41+
def release: Release =
42+
Release(projectRef.organization, projectRef.repository, platform, language, version, releaseDate)
43+
4144
def fullHttpUrl(env: Env): String =
4245
env match {
4346
case Env.Prod => s"https://index.scala-lang.org$artifactHttpPath"

‎modules/core/shared/src/main/scala/scaladex/core/model/Release.scala

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package scaladex.core.model
2+
3+
import java.time.Instant
4+
5+
import scaladex.core.model.Project._
6+
7+
case class Release(
8+
organization: Organization,
9+
repository: Repository,
10+
platform: Platform,
11+
language: Language,
12+
version: SemanticVersion,
13+
releaseDate: Instant
14+
) {
15+
def ref: Project.Reference = Reference(organization, repository)
16+
}

‎modules/core/shared/src/main/scala/scaladex/core/model/ReleaseDependency.scala

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package scaladex.core.model
2+
3+
case class ReleaseDependency(
4+
source: Release,
5+
target: Release,
6+
scope: String
7+
)

‎modules/core/shared/src/main/scala/scaladex/core/service/SchedulerDatabase.scala

+4-1
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,23 @@ import scaladex.core.model.GithubInfo
1111
import scaladex.core.model.GithubStatus
1212
import scaladex.core.model.Project
1313
import scaladex.core.model.ProjectDependency
14+
import scaladex.core.model.ReleaseDependency
1415

1516
trait SchedulerDatabase extends WebDatabase {
1617
def insertProject(project: Project): Future[Unit]
17-
def insertArtifacts(artifacts: Seq[Artifact]): Future[Unit]
18+
def insertArtifacts(artifacts: Seq[Artifact]): Future[Unit] // for init process
1819
def insertDependencies(dependencies: Seq[ArtifactDependency]): Future[Unit]
1920
def getAllProjectsStatuses(): Future[Map[Project.Reference, GithubStatus]]
2021
def getDependencies(projectRef: Project.Reference): Future[Seq[ArtifactDependency]]
2122
def updateArtifacts(artifacts: Seq[Artifact], newRef: Project.Reference): Future[Int]
2223
def updateArtifactReleaseDate(reference: MavenReference, releaseDate: Instant): Future[Int]
2324
def computeProjectDependencies(): Future[Seq[ProjectDependency]]
25+
def computeReleaseDependencies(): Future[Seq[ReleaseDependency]]
2426
def updateGithubInfoAndStatus(ref: Project.Reference, githubInfo: GithubInfo, status: GithubStatus): Future[Unit]
2527
def computeAllProjectsCreationDates(): Future[Seq[(Instant, Project.Reference)]]
2628
def updateProjectCreationDate(ref: Project.Reference, creationDate: Instant): Future[Unit]
2729
def insertProjectDependencies(projectDependencies: Seq[ProjectDependency]): Future[Int]
30+
def insertReleaseDependencies(projectDependencies: Seq[ReleaseDependency]): Future[Int]
2831
def deleteDependenciesOfMovedProject(): Future[Unit]
2932
def getAllGroupIds(): Future[Seq[Artifact.GroupId]]
3033
def getAllMavenReferences(): Future[Seq[Artifact.MavenReference]]

‎modules/core/shared/src/test/scala/scaladex/core/test/InMemoryDatabase.scala

+4
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import scaladex.core.model.GithubResponse
1313
import scaladex.core.model.GithubStatus
1414
import scaladex.core.model.Project
1515
import scaladex.core.model.ProjectDependency
16+
import scaladex.core.model.ReleaseDependency
1617
import scaladex.core.model.UserState
1718
import scaladex.core.service.SchedulerDatabase
1819

@@ -90,6 +91,7 @@ class InMemoryDatabase extends SchedulerDatabase {
9091
override def getAllProjects(): Future[Seq[Project]] = ???
9192

9293
override def computeProjectDependencies(): Future[Seq[ProjectDependency]] = ???
94+
override def computeReleaseDependencies(): Future[Seq[ReleaseDependency]] = ???
9395

9496
override def computeAllProjectsCreationDates(): Future[Seq[(Instant, Project.Reference)]] = ???
9597

@@ -98,6 +100,8 @@ class InMemoryDatabase extends SchedulerDatabase {
98100

99101
override def insertProjectDependencies(projectDependencies: Seq[ProjectDependency]): Future[Int] = ???
100102

103+
override def insertReleaseDependencies(releaseDependency: Seq[ReleaseDependency]): Future[Int] = ???
104+
101105
override def countInverseProjectDependencies(projectRef: Project.Reference): Future[Int] =
102106
// not really implemented
103107
Future.successful(0)

‎modules/data/src/main/scala/scaladex/data/Main.scala

+3-2
Original file line numberDiff line numberDiff line change
@@ -54,14 +54,15 @@ object Main extends LazyLogging {
5454

5555
val dataPaths = DataPaths.from(config.filesystem)
5656
val localStorage = FilesystemStorage(config.filesystem)
57+
val datasource = DoobieUtils.getHikariDataSource(config.database)
5758

5859
def usingDatabase(f: SqlDatabase => Future[Unit]): Unit = {
5960
implicit val cs = IO.contextShift(system.dispatcher)
6061
val transactor: Resource[IO, HikariTransactor[IO]] =
61-
DoobieUtils.transactor(config.database)
62+
DoobieUtils.transactor(datasource)
6263
transactor
6364
.use { xa =>
64-
val database = new SqlDatabase(config.database, xa)
65+
val database = new SqlDatabase(datasource, xa)
6566
IO.fromFuture(IO(f(database)))
6667
}
6768
.unsafeRunSync()

‎modules/data/src/main/scala/scaladex/data/init/Init.scala

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
package scaladex.data.init
2-
32
import scala.concurrent.ExecutionContext
43
import scala.concurrent.Future
54

‎modules/infra/src/main/resources/migrations/V11_1__table_releases_and_release_dependencies.sql

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
CREATE TABLE releases
2+
(
3+
organization VARCHAR(39) NOT NULL,
4+
repository VARCHAR(100) NOT NULL,
5+
platform VARCHAR NOT NULL,
6+
language_version VARCHAR NOT NULL,
7+
version VARCHAR NOT NULL,
8+
release_date TIMESTAMPTZ NOT NULL,
9+
PRIMARY KEY (organization, repository, platform, language_version, version)
10+
);
11+
CREATE TABLE release_dependencies
12+
(
13+
source_organization VARCHAR(39) NOT NULL,
14+
source_repository VARCHAR(100) NOT NULL,
15+
source_platform VARCHAR NOT NULL,
16+
source_language_version VARCHAR NOT NULL,
17+
source_version VARCHAR NOT NULL,
18+
source_release_date TIMESTAMPTZ NOT NULL,
19+
target_organization VARCHAR(39) NOT NULL,
20+
target_repository VARCHAR(100) NOT NULL,
21+
target_platform VARCHAR NOT NULL,
22+
target_language_version VARCHAR NOT NULL,
23+
target_version VARCHAR NOT NULL,
24+
target_release_date TIMESTAMPTZ NOT NULL,
25+
scope VARCHAR NOT NULL,
26+
PRIMARY KEY (source_organization, source_repository, source_platform, source_language_version, source_version,
27+
target_organization, target_repository, target_platform, target_language_version, target_version, scope)
28+
);
29+

‎modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala

+13-4
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import scala.concurrent.Future
88

99
import cats.effect.IO
1010
import com.typesafe.scalalogging.LazyLogging
11+
import com.zaxxer.hikari.HikariDataSource
1112
import doobie.implicits._
1213
import scaladex.core.model.Artifact
1314
import scaladex.core.model.ArtifactDependency
@@ -16,21 +17,22 @@ import scaladex.core.model.GithubResponse
1617
import scaladex.core.model.GithubStatus
1718
import scaladex.core.model.Project
1819
import scaladex.core.model.ProjectDependency
20+
import scaladex.core.model.ReleaseDependency
1921
import scaladex.core.model.UserState
2022
import scaladex.core.service.SchedulerDatabase
21-
import scaladex.infra.config.PostgreSQLConfig
2223
import scaladex.infra.sql.ArtifactDependencyTable
2324
import scaladex.infra.sql.ArtifactTable
2425
import scaladex.infra.sql.DoobieUtils
2526
import scaladex.infra.sql.GithubInfoTable
2627
import scaladex.infra.sql.ProjectDependenciesTable
2728
import scaladex.infra.sql.ProjectSettingsTable
2829
import scaladex.infra.sql.ProjectTable
30+
import scaladex.infra.sql.ReleaseDependenciesTable
31+
import scaladex.infra.sql.ReleaseTable
2932
import scaladex.infra.sql.UserSessionsTable
3033

31-
class SqlDatabase(conf: PostgreSQLConfig, xa: doobie.Transactor[IO]) extends SchedulerDatabase with LazyLogging {
32-
33-
private val flyway = DoobieUtils.flyway(conf)
34+
class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) extends SchedulerDatabase with LazyLogging {
35+
private val flyway = DoobieUtils.flyway(datasource)
3436
def migrate: IO[Unit] = IO(flyway.migrate())
3537
def dropTables: IO[Unit] = IO(flyway.clean())
3638

@@ -43,6 +45,7 @@ class SqlDatabase(conf: PostgreSQLConfig, xa: doobie.Transactor[IO]) extends Sch
4345
for {
4446
isNewProject <- insertProjectRef(artifact.projectRef, unknownStatus)
4547
_ <- run(ArtifactTable.insertIfNotExist.run(artifact))
48+
_ <- run(ReleaseTable.insertIfNotExists.run(artifact.release))
4649
_ <- insertDependencies(dependencies)
4750
} yield isNewProject
4851
}
@@ -166,9 +169,15 @@ class SqlDatabase(conf: PostgreSQLConfig, xa: doobie.Transactor[IO]) extends Sch
166169
override def computeProjectDependencies(): Future[Seq[ProjectDependency]] =
167170
run(ArtifactDependencyTable.computeProjectDependency.to[Seq])
168171

172+
override def computeReleaseDependencies(): Future[Seq[ReleaseDependency]] =
173+
run(ArtifactDependencyTable.computeReleaseDependency.to[Seq])
174+
169175
override def insertProjectDependencies(projectDependencies: Seq[ProjectDependency]): Future[Int] =
170176
run(ProjectDependenciesTable.insertOrUpdate.updateMany(projectDependencies))
171177

178+
override def insertReleaseDependencies(releaseDependency: Seq[ReleaseDependency]): Future[Int] =
179+
run(ReleaseDependenciesTable.insertIfNotExists.updateMany(releaseDependency))
180+
172181
override def countInverseProjectDependencies(projectRef: Project.Reference): Future[Int] =
173182
run(ProjectDependenciesTable.countInverseDependencies.unique(projectRef))
174183

‎modules/infra/src/main/scala/scaladex/infra/migrations/V11_2__add_data_to_the_new_tables.scala

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package scaladex.infra.migrations
2+
3+
import com.typesafe.scalalogging.LazyLogging
4+
import org.flywaydb.core.api.migration.BaseJavaMigration
5+
import org.flywaydb.core.api.migration.Context
6+
import scaladex.infra.sql.ArtifactTable
7+
import scaladex.infra.sql.ReleaseTable
8+
9+
class V11_2__add_data_to_the_new_tables extends BaseJavaMigration with ScaladexBaseMigration with LazyLogging {
10+
override def migrate(context: Context): Unit =
11+
try {
12+
(for {
13+
releases <- run(xa)(ArtifactTable.getReleasesFromArtifacts.to[Seq])
14+
_ <- run(xa)(ReleaseTable.insertIfNotExists.updateMany(releases))
15+
} yield ())
16+
.unsafeRunSync()
17+
18+
} catch {
19+
case e: Throwable =>
20+
logger.info("failed to migrate the database")
21+
throw new Exception(s"failed to migrate the database because of ${e.getMessage}")
22+
}
23+
24+
}

‎modules/infra/src/main/scala/scaladex/infra/sql/ArtifactDependencyTable.scala

+12-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import scaladex.core.model.Artifact
66
import scaladex.core.model.ArtifactDependency
77
import scaladex.core.model.Project
88
import scaladex.core.model.ProjectDependency
9+
import scaladex.core.model.ReleaseDependency
910
import scaladex.infra.sql.DoobieUtils.Mappings._
1011
import scaladex.infra.sql.DoobieUtils._
1112

@@ -66,10 +67,20 @@ object ArtifactDependencyTable {
6667
val computeProjectDependency: Query0[ProjectDependency] =
6768
selectRequest(
6869
fullJoin,
69-
"DISTINCT d.organization, d.repository, t.organization, t.repository",
70+
"d.organization, d.repository, t.organization, t.repository",
7071
groupBy = Seq("d.organization", "d.repository", "t.organization", "t.repository")
7172
)
7273

74+
val computeReleaseDependency: Query0[ReleaseDependency] = {
75+
val sourceReleaseFields = ReleaseTable.primaryKeys.map("d." + _)
76+
val targetReleaseFields = ReleaseTable.primaryKeys.map("t." + _)
77+
selectRequest(
78+
fullJoin,
79+
s"${sourceReleaseFields.mkString(", ")}, MIN(d.release_date), ${targetReleaseFields.mkString(", ")}, MIN(t.release_date), d.scope",
80+
groupBy = (sourceReleaseFields ++ targetReleaseFields) ++ Seq("d.scope")
81+
)
82+
}
83+
7384
val selectDependencyFromProject: Query[Project.Reference, ArtifactDependency] =
7485
selectRequest(
7586
tableWithSourceArtifact,

‎modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala

+8
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import doobie._
66
import doobie.util.update.Update
77
import scaladex.core.model.Artifact
88
import scaladex.core.model.Project
9+
import scaladex.core.model.Release
910
import scaladex.infra.sql.DoobieUtils.Mappings._
1011
import scaladex.infra.sql.DoobieUtils._
1112

@@ -65,4 +66,11 @@ object ArtifactTable {
6566
groupBy = projectReferenceFields
6667
)
6768

69+
val getReleasesFromArtifacts: Query0[Release] =
70+
selectRequest(
71+
table,
72+
"organization, repository, platform , language_version , version , MIN(release_date) as release_date",
73+
groupBy = Seq("organization", "repository ", "platform ", "language_version", "version")
74+
)
75+
6876
}

‎modules/infra/src/main/scala/scaladex/infra/sql/DoobieUtils.scala

+6-5
Original file line numberDiff line numberDiff line change
@@ -39,24 +39,25 @@ object DoobieUtils {
3939

4040
private implicit val cs: ContextShift[IO] =
4141
IO.contextShift(ExecutionContext.global)
42+
4243
def flyway(conf: PostgreSQLConfig): Flyway = {
4344
val datasource = getHikariDataSource(conf)
45+
flyway(datasource)
46+
}
47+
def flyway(datasource: HikariDataSource): Flyway =
4448
Flyway
4549
.configure()
4650
.dataSource(datasource)
4751
.locations("migrations", "scaladex/infra/migrations")
4852
.load()
49-
}
5053

51-
def transactor(conf: PostgreSQLConfig): Resource[IO, HikariTransactor[IO]] = {
52-
val datasource = getHikariDataSource(conf)
54+
def transactor(datasource: HikariDataSource): Resource[IO, HikariTransactor[IO]] =
5355
for {
5456
ce <- ExecutionContexts.fixedThreadPool[IO](32) // our connect EC
5557
be <- Blocker[IO] // our blocking EC
5658
} yield Transactor.fromDataSource[IO](datasource, ce, be)
57-
}
5859

59-
private def getHikariDataSource(conf: PostgreSQLConfig): HikariDataSource = {
60+
def getHikariDataSource(conf: PostgreSQLConfig): HikariDataSource = {
6061
val config: HikariConfig = new HikariConfig()
6162
config.setDriverClassName(conf.driver)
6263
config.setJdbcUrl(conf.url)

‎modules/infra/src/main/scala/scaladex/infra/sql/ReleaseDependenciesTable.scala

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package scaladex.infra.sql
2+
3+
import doobie.util.update.Update
4+
import scaladex.core.model.ReleaseDependency
5+
import scaladex.infra.sql.DoobieUtils.Mappings._
6+
import scaladex.infra.sql.DoobieUtils.insertOrUpdateRequest
7+
8+
object ReleaseDependenciesTable {
9+
private val table: String = "release_dependencies"
10+
private val sourcePrimaryKeys: Seq[String] =
11+
Seq(
12+
"source_organization",
13+
"source_repository",
14+
"source_platform",
15+
"source_language_version",
16+
"source_version"
17+
)
18+
19+
private val targetPrimaryKeys: Seq[String] =
20+
Seq(
21+
"target_organization",
22+
"target_repository",
23+
"target_platform",
24+
"target_language_version",
25+
"target_version"
26+
)
27+
private val fields: Seq[String] =
28+
sourcePrimaryKeys ++ Seq("source_release_date") ++ targetPrimaryKeys ++ Seq("target_release_date", "scope")
29+
30+
private val primaryKeys = (sourcePrimaryKeys ++ targetPrimaryKeys) :+ "scope"
31+
32+
val insertIfNotExists: Update[ReleaseDependency] =
33+
insertOrUpdateRequest(table, fields, primaryKeys)
34+
}

‎modules/infra/src/main/scala/scaladex/infra/sql/ReleaseTable.scala

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package scaladex.infra.sql
2+
3+
import doobie.util.update.Update
4+
import scaladex.core.model.Release
5+
import scaladex.infra.sql.DoobieUtils.Mappings._
6+
import scaladex.infra.sql.DoobieUtils.insertOrUpdateRequest
7+
8+
object ReleaseTable {
9+
private val table: String = "releases"
10+
val primaryKeys: Seq[String] =
11+
Seq(
12+
"organization",
13+
"repository",
14+
"platform",
15+
"language_version",
16+
"version"
17+
)
18+
private val fields: Seq[String] =
19+
primaryKeys ++ Seq(
20+
"release_date"
21+
)
22+
val insertIfNotExists: Update[Release] =
23+
insertOrUpdateRequest(table, fields, primaryKeys)
24+
}

0 commit comments

Comments
 (0)