Skip to content

Commit c49b368

Browse files
mlachkarmlachkar
committed
add new tables
1 parent ce3da52 commit c49b368

File tree

19 files changed

+230
-4
lines changed

19 files changed

+230
-4
lines changed

‎modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ case class Artifact(
3838

3939
val mavenReference: Artifact.MavenReference = Artifact.MavenReference(groupId.value, artifactId, version.encode)
4040

41+
def release: Release =
42+
Release(projectRef.organization, projectRef.repository, platform, language, version, releaseDate)
43+
4144
def fullHttpUrl(env: Env): String =
4245
env match {
4346
case Env.Prod => s"https://index.scala-lang.org$artifactHttpPath"

‎modules/core/shared/src/main/scala/scaladex/core/model/Release.scala

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package scaladex.core.model
2+
3+
import java.time.Instant
4+
5+
import scaladex.core.model.Project._
6+
7+
case class Release(
8+
organization: Organization,
9+
repository: Repository,
10+
platform: Platform,
11+
language: Language,
12+
version: SemanticVersion,
13+
releaseDate: Instant
14+
) {
15+
def ref: Project.Reference = Reference(organization, repository)
16+
}

‎modules/core/shared/src/main/scala/scaladex/core/model/ReleaseDependency.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package scaladex.core.model
2+
3+
case class ReleaseDependency(
4+
source: Release,
5+
target: Release,
6+
scope: String
7+
)

‎modules/core/shared/src/main/scala/scaladex/core/service/SchedulerDatabase.scala

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,23 @@ import scaladex.core.model.GithubInfo
1111
import scaladex.core.model.GithubStatus
1212
import scaladex.core.model.Project
1313
import scaladex.core.model.ProjectDependency
14+
import scaladex.core.model.ReleaseDependency
1415

1516
trait SchedulerDatabase extends WebDatabase {
1617
def insertProject(project: Project): Future[Unit]
17-
def insertArtifacts(artifacts: Seq[Artifact]): Future[Unit]
18+
def insertArtifacts(artifacts: Seq[Artifact]): Future[Unit] // for init process
1819
def insertDependencies(dependencies: Seq[ArtifactDependency]): Future[Unit]
1920
def getAllProjectsStatuses(): Future[Map[Project.Reference, GithubStatus]]
2021
def getDependencies(projectRef: Project.Reference): Future[Seq[ArtifactDependency]]
2122
def updateArtifacts(artifacts: Seq[Artifact], newRef: Project.Reference): Future[Int]
2223
def updateArtifactReleaseDate(reference: MavenReference, releaseDate: Instant): Future[Int]
2324
def computeProjectDependencies(): Future[Seq[ProjectDependency]]
25+
def computeReleaseDependencies(): Future[Seq[ReleaseDependency]]
2426
def updateGithubInfoAndStatus(ref: Project.Reference, githubInfo: GithubInfo, status: GithubStatus): Future[Unit]
2527
def computeAllProjectsCreationDates(): Future[Seq[(Instant, Project.Reference)]]
2628
def updateProjectCreationDate(ref: Project.Reference, creationDate: Instant): Future[Unit]
2729
def insertProjectDependencies(projectDependencies: Seq[ProjectDependency]): Future[Int]
30+
def insertReleaseDependencies(projectDependencies: Seq[ReleaseDependency]): Future[Int]
2831
def deleteDependenciesOfMovedProject(): Future[Unit]
2932
def getAllGroupIds(): Future[Seq[Artifact.GroupId]]
3033
def getAllMavenReferences(): Future[Seq[Artifact.MavenReference]]

‎modules/core/shared/src/test/scala/scaladex/core/test/InMemoryDatabase.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import scaladex.core.model.GithubResponse
1313
import scaladex.core.model.GithubStatus
1414
import scaladex.core.model.Project
1515
import scaladex.core.model.ProjectDependency
16+
import scaladex.core.model.ReleaseDependency
1617
import scaladex.core.model.UserState
1718
import scaladex.core.service.SchedulerDatabase
1819

@@ -90,6 +91,7 @@ class InMemoryDatabase extends SchedulerDatabase {
9091
override def getAllProjects(): Future[Seq[Project]] = ???
9192

9293
override def computeProjectDependencies(): Future[Seq[ProjectDependency]] = ???
94+
override def computeReleaseDependencies(): Future[Seq[ReleaseDependency]] = ???
9395

9496
override def computeAllProjectsCreationDates(): Future[Seq[(Instant, Project.Reference)]] = ???
9597

@@ -98,6 +100,8 @@ class InMemoryDatabase extends SchedulerDatabase {
98100

99101
override def insertProjectDependencies(projectDependencies: Seq[ProjectDependency]): Future[Int] = ???
100102

103+
override def insertReleaseDependencies(releaseDependency: Seq[ReleaseDependency]): Future[Int] = ???
104+
101105
override def countInverseProjectDependencies(projectRef: Project.Reference): Future[Int] =
102106
// not really implemented
103107
Future.successful(0)

‎modules/infra/src/main/resources/migrations/V11_1__table_releases_and_release_dependencies.sql

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
CREATE TABLE releases
2+
(
3+
organization VARCHAR(39) NOT NULL,
4+
repository VARCHAR(100) NOT NULL,
5+
platform VARCHAR NOT NULL,
6+
language_version VARCHAR NOT NULL,
7+
version VARCHAR NOT NULL,
8+
release_date TIMESTAMPTZ NOT NULL,
9+
PRIMARY KEY (organization, repository, platform, language_version, version)
10+
);
11+
CREATE TABLE release_dependencies
12+
(
13+
source_organization VARCHAR(39) NOT NULL,
14+
source_repository VARCHAR(100) NOT NULL,
15+
source_platform VARCHAR NOT NULL,
16+
source_language_version VARCHAR NOT NULL,
17+
source_version VARCHAR NOT NULL,
18+
source_release_date TIMESTAMPTZ NOT NULL,
19+
target_organization VARCHAR(39) NOT NULL,
20+
target_repository VARCHAR(100) NOT NULL,
21+
target_platform VARCHAR NOT NULL,
22+
target_language_version VARCHAR NOT NULL,
23+
target_version VARCHAR NOT NULL,
24+
target_release_date TIMESTAMPTZ NOT NULL,
25+
scope VARCHAR NOT NULL,
26+
PRIMARY KEY (source_organization, source_repository, source_platform, source_language_version, source_version,
27+
target_organization, target_repository, target_platform, target_language_version, target_version, scope)
28+
);
29+

‎modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import scaladex.core.model.GithubResponse
1616
import scaladex.core.model.GithubStatus
1717
import scaladex.core.model.Project
1818
import scaladex.core.model.ProjectDependency
19+
import scaladex.core.model.ReleaseDependency
1920
import scaladex.core.model.UserState
2021
import scaladex.core.service.SchedulerDatabase
2122
import scaladex.infra.config.PostgreSQLConfig
@@ -26,6 +27,8 @@ import scaladex.infra.sql.GithubInfoTable
2627
import scaladex.infra.sql.ProjectDependenciesTable
2728
import scaladex.infra.sql.ProjectSettingsTable
2829
import scaladex.infra.sql.ProjectTable
30+
import scaladex.infra.sql.ReleaseDependenciesTable
31+
import scaladex.infra.sql.ReleaseTable
2932
import scaladex.infra.sql.UserSessionsTable
3033

3134
class SqlDatabase(conf: PostgreSQLConfig, xa: doobie.Transactor[IO]) extends SchedulerDatabase with LazyLogging {
@@ -43,6 +46,7 @@ class SqlDatabase(conf: PostgreSQLConfig, xa: doobie.Transactor[IO]) extends Sch
4346
for {
4447
isNewProject <- insertProjectRef(artifact.projectRef, unknownStatus)
4548
_ <- run(ArtifactTable.insertIfNotExist.run(artifact))
49+
_ <- run(ReleaseTable.insertIfNotExists.run(artifact.release))
4650
_ <- insertDependencies(dependencies)
4751
} yield isNewProject
4852
}
@@ -166,9 +170,15 @@ class SqlDatabase(conf: PostgreSQLConfig, xa: doobie.Transactor[IO]) extends Sch
166170
override def computeProjectDependencies(): Future[Seq[ProjectDependency]] =
167171
run(ArtifactDependencyTable.computeProjectDependency.to[Seq])
168172

173+
override def computeReleaseDependencies(): Future[Seq[ReleaseDependency]] =
174+
run(ArtifactDependencyTable.computeReleaseDependency.to[Seq])
175+
169176
override def insertProjectDependencies(projectDependencies: Seq[ProjectDependency]): Future[Int] =
170177
run(ProjectDependenciesTable.insertOrUpdate.updateMany(projectDependencies))
171178

179+
override def insertReleaseDependencies(releaseDependency: Seq[ReleaseDependency]): Future[Int] =
180+
run(ReleaseDependenciesTable.insertIfNotExists.updateMany(releaseDependency))
181+
172182
override def countInverseProjectDependencies(projectRef: Project.Reference): Future[Int] =
173183
run(ProjectDependenciesTable.countInverseDependencies.unique(projectRef))
174184

‎modules/infra/src/main/scala/scaladex/infra/migrations/V11_2__add_data_to_the_new_tables.scala

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package scaladex.infra.migrations
2+
3+
import com.typesafe.scalalogging.LazyLogging
4+
import org.flywaydb.core.api.migration.BaseJavaMigration
5+
import org.flywaydb.core.api.migration.Context
6+
import scaladex.infra.sql.ArtifactTable
7+
import scaladex.infra.sql.ReleaseTable
8+
9+
class V11_2__add_data_to_the_new_tables extends BaseJavaMigration with ScaladexBaseMigration with LazyLogging {
10+
override def migrate(context: Context): Unit =
11+
try {
12+
(for {
13+
releases <- run(xa)(ArtifactTable.getReleasesFromArtifacts.to[Seq])
14+
_ <- run(xa)(ReleaseTable.insertIfNotExists.updateMany(releases))
15+
} yield ())
16+
.unsafeRunSync()
17+
18+
} catch {
19+
case e: Throwable =>
20+
logger.info("failed to migrate the database")
21+
throw new Exception(s"failed to migrate the database because of ${e.getMessage}")
22+
}
23+
24+
}

‎modules/infra/src/main/scala/scaladex/infra/sql/ArtifactDependencyTable.scala

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import scaladex.core.model.Artifact
66
import scaladex.core.model.ArtifactDependency
77
import scaladex.core.model.Project
88
import scaladex.core.model.ProjectDependency
9+
import scaladex.core.model.ReleaseDependency
910
import scaladex.infra.sql.DoobieUtils.Mappings._
1011
import scaladex.infra.sql.DoobieUtils._
1112

@@ -70,6 +71,16 @@ object ArtifactDependencyTable {
7071
groupBy = Seq("d.organization", "d.repository", "t.organization", "t.repository")
7172
)
7273

74+
val computeReleaseDependency: Query0[ReleaseDependency] = {
75+
val sourceReleaseFields = ReleaseTable.fields.map("d." + _)
76+
val targetReleaseFields = ReleaseTable.fields.map("t." + _)
77+
selectRequest(
78+
fullJoin,
79+
s"DISTINCT ${sourceReleaseFields.mkString(", ")}, ${targetReleaseFields.mkString(", ")}, d.scope",
80+
groupBy = (sourceReleaseFields ++ targetReleaseFields) :+ "d.scope"
81+
)
82+
}
83+
7384
val selectDependencyFromProject: Query[Project.Reference, ArtifactDependency] =
7485
selectRequest(
7586
tableWithSourceArtifact,

‎modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import doobie._
66
import doobie.util.update.Update
77
import scaladex.core.model.Artifact
88
import scaladex.core.model.Project
9+
import scaladex.core.model.Release
910
import scaladex.infra.sql.DoobieUtils.Mappings._
1011
import scaladex.infra.sql.DoobieUtils._
1112

@@ -65,4 +66,11 @@ object ArtifactTable {
6566
groupBy = projectReferenceFields
6667
)
6768

69+
val getReleasesFromArtifacts: Query0[Release] =
70+
selectRequest(
71+
table,
72+
"organization, repository, platform , language_version , version , MIN(release_date) as release_date",
73+
groupBy = Seq("organization", "repository ", "platform ", "language_version", "version")
74+
)
75+
6876
}

‎modules/infra/src/main/scala/scaladex/infra/sql/ReleaseDependenciesTable.scala

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package scaladex.infra.sql
2+
3+
import doobie.util.update.Update
4+
import scaladex.core.model.ReleaseDependency
5+
import scaladex.infra.sql.DoobieUtils.Mappings._
6+
import scaladex.infra.sql.DoobieUtils.insertOrUpdateRequest
7+
8+
object ReleaseDependenciesTable {
9+
private val table: String = "release_dependencies"
10+
private val sourcePrimaryKeys: Seq[String] =
11+
Seq(
12+
"source_organization",
13+
"source_repository",
14+
"source_platform",
15+
"source_language_version",
16+
"source_version"
17+
)
18+
19+
private val targetPrimaryKeys: Seq[String] =
20+
Seq(
21+
"target_organization",
22+
"target_repository",
23+
"target_platform",
24+
"target_language_version",
25+
"target_version"
26+
)
27+
private val fields: Seq[String] =
28+
sourcePrimaryKeys ++ Seq("source_release_date") ++ targetPrimaryKeys ++ Seq("target_release_date", "scope")
29+
30+
private val primaryKeys = (sourcePrimaryKeys ++ targetPrimaryKeys) :+ "scope"
31+
32+
val insertIfNotExists: Update[ReleaseDependency] =
33+
insertOrUpdateRequest(table, fields, primaryKeys)
34+
}

‎modules/infra/src/main/scala/scaladex/infra/sql/ReleaseTable.scala

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package scaladex.infra.sql
2+
3+
import doobie.util.update.Update
4+
import scaladex.core.model.Release
5+
import scaladex.infra.sql.DoobieUtils.Mappings._
6+
import scaladex.infra.sql.DoobieUtils.insertOrUpdateRequest
7+
8+
object ReleaseTable {
9+
private val table: String = "releases"
10+
private val primaryKeys: Seq[String] =
11+
Seq(
12+
"organization",
13+
"repository",
14+
"platform",
15+
"language_version",
16+
"version"
17+
)
18+
val fields: Seq[String] =
19+
primaryKeys ++ Seq(
20+
"release_date"
21+
)
22+
val insertIfNotExists: Update[Release] =
23+
insertOrUpdateRequest(table, fields, primaryKeys)
24+
}

‎modules/infra/src/test/scala/scaladex/infra/sql/ArtifactDependencyTableTests.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,5 @@ class ArtifactDependencyTableTests extends AnyFunSpec with BaseDatabaseSuite wit
1111
it("check selectDirectDependency")(check(ArtifactDependencyTable.selectDirectDependency))
1212
it("check selectReverseDependency")(check(ArtifactDependencyTable.selectReverseDependency))
1313
it("check selectProjectDependency")(check(ArtifactDependencyTable.computeProjectDependency))
14+
it("check computeReleaseDependency")(check(ArtifactDependencyTable.computeReleaseDependency))
1415
}

‎modules/infra/src/test/scala/scaladex/infra/sql/ArtifactTableTests.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,5 @@ class ArtifactTableTests extends AnyFunSpec with BaseDatabaseSuite with Matchers
1414
it("check selectMavenReference")(check(ArtifactTable.selectMavenReference))
1515
it("check updateReleaseDate")(check(ArtifactTable.updateReleaseDate))
1616
it("check selectByMavenReference")(check(ArtifactTable.selectByMavenReference))
17+
it("check getReleasesFromArtifacts")(check(ArtifactTable.getReleasesFromArtifacts))
1718
}

‎modules/infra/src/test/scala/scaladex/infra/sql/ReleaseDependenciesTableTests.scala

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package scaladex.infra.sql
2+
3+
import org.scalatest.funspec.AnyFunSpec
4+
import org.scalatest.matchers.should.Matchers
5+
import scaladex.infra.BaseDatabaseSuite
6+
7+
class ReleaseDependenciesTableTests extends AnyFunSpec with BaseDatabaseSuite with Matchers {
8+
it("check insertIfNotExists")(check(ReleaseDependenciesTable.insertIfNotExists))
9+
}

‎modules/infra/src/test/scala/scaladex/infra/sql/ReleaseTableTests.scala

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package scaladex.infra.sql
2+
3+
import org.scalatest.funspec.AnyFunSpec
4+
import org.scalatest.matchers.should.Matchers
5+
import scaladex.infra.BaseDatabaseSuite
6+
7+
class ReleaseTableTests extends AnyFunSpec with BaseDatabaseSuite with Matchers {
8+
it("check insertIfNotExists")(check(ReleaseTable.insertIfNotExists))
9+
}

‎modules/server/src/main/scala/scaladex/server/service/ProjectDependenciesUpdater.scala

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,19 @@ package scaladex.server.service
33
import scala.concurrent.ExecutionContext
44
import scala.concurrent.Future
55

6+
import com.typesafe.scalalogging.LazyLogging
67
import scaladex.core.service.SchedulerDatabase
78
import scaladex.core.util.ScalaExtensions._
89

9-
class ProjectDependenciesUpdater(database: SchedulerDatabase)(implicit ec: ExecutionContext) {
10+
class ProjectDependenciesUpdater(database: SchedulerDatabase)(implicit ec: ExecutionContext) extends LazyLogging {
11+
1012
def updateAll(): Future[Unit] =
13+
for {
14+
_ <- updateProjectDependencyTable()
15+
_ <- updateReleaseDependencyTable()
16+
} yield ()
17+
18+
def updateProjectDependencyTable(): Future[Unit] =
1119
for {
1220
projectWithDependencies <- database
1321
.computeProjectDependencies()
@@ -16,6 +24,7 @@ class ProjectDependenciesUpdater(database: SchedulerDatabase)(implicit ec: Execu
1624
s"Failed to compute project dependencies because of ${e.getMessage}"
1725
)
1826
)
27+
_ = logger.info(s"will try to insert ${projectWithDependencies.size} projectDependencies")
1928
_ <- database.deleteDependenciesOfMovedProject()
2029
_ <- database
2130
.insertProjectDependencies(projectWithDependencies)
@@ -26,4 +35,29 @@ class ProjectDependenciesUpdater(database: SchedulerDatabase)(implicit ec: Execu
2635
)
2736

2837
} yield ()
38+
39+
def updateReleaseDependencyTable(): Future[Unit] =
40+
for {
41+
releaseDependencies <- database
42+
.computeReleaseDependencies()
43+
.mapFailure(e =>
44+
new Exception(
45+
s"Failed to compute release dependencies because of ${e.getMessage}"
46+
)
47+
)
48+
_ = logger.info(s"will try to insert ${releaseDependencies.size} releaseDependencies")
49+
_ <- releaseDependencies
50+
.grouped(10000)
51+
.map(releaseDependencies =>
52+
database
53+
.insertReleaseDependencies(releaseDependencies)
54+
.mapFailure(e =>
55+
new Exception(
56+
s"Failed to insert release dependencies because of ${e.getMessage}"
57+
)
58+
)
59+
)
60+
.sequence
61+
62+
} yield ()
2963
}

‎modules/server/src/main/scala/scaladex/server/service/SchedulerService.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ class SchedulerService(
2525
val projectDependenciesUpdater = new ProjectDependenciesUpdater(database)
2626

2727
private val schedulers = Seq(
28-
Scheduler("update-project-dependencies", projectDependenciesUpdater.updateAll, 1.hour),
28+
Scheduler("update-dependency-tables", projectDependenciesUpdater.updateAll, 1.hour),
2929
Scheduler("update-project-creation-date", updateProjectCreationDate, 30.minutes),
3030
Scheduler("sync-search", searchSynchronizer.syncAll, 30.minutes),
3131
new MovedArtifactsSynchronizer(database)

‎modules/template/src/main/twirl/scaladex/view/project/project.scala.html

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
@import scaladex.view.html.contributingInfo
22
@import scaladex.core.model.Project
33
@import scaladex.core.model.Artifact
4-
@import akka.http.scaladsl.model.headers.LinkParams.title
54
@import scaladex.core.model.Env
65
@import scaladex.core.model.UserState
76
@import scaladex.core.model.SemanticVersion

0 commit comments

Comments
 (0)