Skip to content

Commit 6573347

Browse files
tgodziktgodzik
authored
Merge pull request #1720 from scalacenter/optimize-scaladex
improvement: Try to fix an issue with performance
2 parents 6fb527e + 5ce6625 commit 6573347

12 files changed

Lines changed: 168 additions & 34 deletions

File tree

‎build.sbt‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,7 @@ inThisBuild(
1414
excludeDependencies ++= Seq(
1515
// from Coursier
1616
ExclusionRule("org.scala-lang.modules", "scala-collection-compat_2.13"),
17-
ExclusionRule("org.scala-lang.modules", "scala-xml_2.13"),
18-
// from doobie-scalatest
19-
ExclusionRule("org.scalatest", "scalatest_2.13"),
20-
ExclusionRule("org.tpolecat", "doobie-core_2.13")
17+
ExclusionRule("org.scala-lang.modules", "scala-xml_2.13")
2118
)
2219
)
2320
)
@@ -82,7 +79,6 @@ lazy val infra = project
8279
"org.tpolecat" %% "doobie-h2" % V.doobie,
8380
"org.tpolecat" %% "doobie-postgres" % V.doobie,
8481
"org.tpolecat" %% "doobie-hikari" % V.doobie,
85-
("org.tpolecat" %% "doobie-scalatest" % V.doobie % Test).cross(CrossVersion.for3Use2_13),
8682
"io.circe" %% "circe-core" % V.circe,
8783
"io.circe" %% "circe-generic" % V.circe,
8884
"io.circe" %% "circe-parser" % V.circe,

‎modules/core/shared/src/main/scala/scaladex/core/service/SchedulerDatabase.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,10 @@ trait SchedulerDatabase extends WebDatabase:
3030
def updateArtifacts(artifacts: Seq[Artifact.Reference], newRef: Project.Reference): Future[Int]
3131
def updateArtifactReleaseDate(ref: Artifact.Reference, releaseDate: Instant): Future[Int]
3232
def getGroupIds(): Future[Seq[Artifact.GroupId]]
33+
def getGroupIds(limit: Int, offset: Int): Future[Seq[Artifact.GroupId]]
3334
def getArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, Artifact.ArtifactId)]]
3435
def getArtifactRefs(): Future[Seq[Artifact.Reference]]
36+
def getArtifactRefs(groupId: Artifact.GroupId): Future[Seq[Artifact.Reference]]
37+
def getArtifactRefs(groupId: Artifact.GroupId, limit: Int, offset: Int): Future[Seq[Artifact.Reference]]
3538
def updateLatestVersion(ref: Project.Reference, artifact: Artifact.Reference): Future[Unit]
3639
end SchedulerDatabase

‎modules/core/shared/src/test/scala/scaladex/core/test/InMemoryDatabase.scala‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,11 @@ class InMemoryDatabase extends SchedulerDatabase:
157157

158158
override def updateArtifacts(allArtifacts: Seq[Artifact.Reference], newRef: Project.Reference): Future[Int] = ???
159159
override def getGroupIds(): Future[Seq[Artifact.GroupId]] = ???
160+
override def getGroupIds(limit: Int, offset: Int): Future[Seq[Artifact.GroupId]] = ???
160161
override def getArtifactRefs(): Future[Seq[Artifact.Reference]] = ???
162+
override def getArtifactRefs(groupId: Artifact.GroupId): Future[Seq[Artifact.Reference]] = ???
163+
override def getArtifactRefs(groupId: Artifact.GroupId, limit: Int, offset: Int): Future[Seq[Artifact.Reference]] =
164+
???
161165
override def insertUser(userId: UUID, userInfo: UserInfo): Future[Unit] = ???
162166
override def updateUser(userId: UUID, userInfo: UserState): Future[Unit] = ???
163167
override def getUser(userId: UUID): Future[Option[UserState]] = ???

‎modules/infra/src/main/resources/migrations/V28__add_performance_indexes.sql‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
-- Add indexes to improve query performance and reduce connection pool pressure
2+
3+
-- Index for is_latest_version queries (used on every page header)
4+
CREATE INDEX IF NOT EXISTS artifact_latest_version_idx
5+
ON artifacts (organization, repository)
6+
WHERE is_latest_version = true;
7+
8+
-- Index for reverse dependency lookups by target
9+
CREATE INDEX IF NOT EXISTS artifact_dep_target_idx
10+
ON artifact_dependencies (target_group_id, target_artifact_id, target_version);

‎modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,12 +206,21 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
206206
override def getGroupIds(): Future[Seq[Artifact.GroupId]] =
207207
run(ArtifactTable.selectGroupIds.to[Seq])
208208

209+
override def getGroupIds(limit: Int, offset: Int): Future[Seq[Artifact.GroupId]] =
210+
run(ArtifactTable.selectGroupIdsPage.to[Seq]((limit.toLong, offset.toLong)))
211+
209212
override def getArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, Artifact.ArtifactId)]] =
210213
run(ArtifactTable.selectArtifactIds.to[Seq](ref))
211214

212215
override def getArtifactRefs(): Future[Seq[Artifact.Reference]] =
213216
run(ArtifactTable.selectReferences.to[Seq])
214217

218+
override def getArtifactRefs(groupId: Artifact.GroupId): Future[Seq[Artifact.Reference]] =
219+
run(ArtifactTable.selectReferencesByGroupId.to[Seq](groupId))
220+
221+
override def getArtifactRefs(groupId: Artifact.GroupId, limit: Int, offset: Int): Future[Seq[Artifact.Reference]] =
222+
run(ArtifactTable.selectReferencesByGroupIdPage.to[Seq]((groupId, limit.toLong, offset.toLong)))
223+
215224
override def insertUser(userId: UUID, userInfo: UserInfo): Future[Unit] =
216225
run(UserSessionsTable.insert.run((userId, userInfo)).map(_ => ()))
217226

‎modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,12 +140,26 @@ object ArtifactTable:
140140
val selectGroupIds: Query0[GroupId] =
141141
selectRequest(table, Seq("DISTINCT group_id"))
142142

143+
/** Paged group IDs: params are (limit, offset). */
144+
val selectGroupIdsPage: Query[(Long, Long), GroupId] =
145+
Query(s"SELECT DISTINCT group_id FROM $table ORDER BY group_id LIMIT ? OFFSET ?")
146+
143147
val selectArtifactIds: Query[Project.Reference, (GroupId, ArtifactId)] =
144148
selectRequest(table, Seq("DISTINCT group_id", "artifact_id"), keys = projectReferenceFields)
145149

146150
val selectReferences: Query0[Reference] =
147151
selectRequest(table, Seq("DISTINCT group_id", "artifact_id", "\"version\""))
148152

153+
val selectReferencesByGroupId: Query[GroupId, Reference] =
154+
selectRequest(table, Seq("DISTINCT group_id", "artifact_id", "\"version\""), keys = Seq("group_id"))
155+
156+
/** Paged refs for a group: params are (groupId, limit, offset). */
157+
val selectReferencesByGroupIdPage: Query[(GroupId, Long, Long), Reference] =
158+
Query(
159+
s"""SELECT DISTINCT group_id, artifact_id, "version" FROM $table
160+
|WHERE group_id = ? ORDER BY artifact_id, "version" LIMIT ? OFFSET ?""".stripMargin
161+
)
162+
149163
val selectReferencesByProject: Query[Project.Reference, Reference] =
150164
selectRequest(table, Seq("DISTINCT group_id", "artifact_id", "\"version\""), keys = projectReferenceFields)
151165

‎modules/infra/src/main/scala/scaladex/infra/sql/DoobieUtils.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ object DoobieUtils:
3434
config.setUsername(conf.user)
3535
config.setPassword(conf.pass.decode)
3636
new HikariDataSource(config)
37+
end getHikariDataSource
3738

3839
def transactor(datasource: HikariDataSource): Resource[IO, HikariTransactor[IO]] =
3940
for

‎modules/infra/src/test/scala/scaladex/infra/BaseDatabaseSuite.scala‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,14 @@
11
package scaladex.infra
22

33
import scala.concurrent.Await
4-
import scala.concurrent.ExecutionContext
54
import scala.concurrent.Future
65
import scala.concurrent.duration.Duration
76

87
import scaladex.infra.config.PostgreSQLConfig
98
import scaladex.infra.sql.DoobieUtils
109

11-
import cats.effect.ContextShift
1210
import cats.effect.IO
1311
import com.zaxxer.hikari.HikariDataSource
14-
import doobie.scalatest.IOChecker
1512
import doobie.util.transactor.Transactor
1613
import org.scalatest.Assertions
1714
import org.scalatest.BeforeAndAfterEach
@@ -20,14 +17,11 @@ import org.scalatest.Suite
2017
trait BaseDatabaseSuite extends IOChecker with BeforeAndAfterEach:
2118
self: Assertions with Suite =>
2219

23-
private given ContextShift[IO] =
24-
IO.contextShift(ExecutionContext.global)
25-
2620
private val config: PostgreSQLConfig = PostgreSQLConfig
2721
.load()
2822
.get
2923

30-
override val transactor: Transactor.Aux[IO, Unit] =
24+
override val transactor: Transactor[IO] =
3125
Transactor
3226
.fromDriverManager[IO](
3327
config.driver,
@@ -48,6 +42,7 @@ trait BaseDatabaseSuite extends IOChecker with BeforeAndAfterEach:
4842
yield ()
4943
reset.unsafeToFuture()
5044
end BaseDatabaseSuite
45+
5146
object BaseDatabaseSuite:
5247
private val config: PostgreSQLConfig = PostgreSQLConfig
5348
.load()

‎modules/infra/src/test/scala/scaladex/infra/Checker.scala‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package scaladex.infra
2+
3+
import scala.concurrent.ExecutionContext
4+
5+
import cats.effect.ContextShift
6+
import cats.effect.IO
7+
import doobie.*
8+
import doobie.implicits.*
9+
import doobie.util.testing.AnalysisArgs
10+
import doobie.util.testing.Analyzable
11+
import doobie.util.testing.formatReport
12+
import doobie.util.transactor.Transactor
13+
import org.scalatest.Assertions
14+
15+
trait IOChecker:
16+
self: Assertions =>
17+
18+
given ContextShift[IO] = IO.contextShift(ExecutionContext.global)
19+
20+
def transactor: Transactor[IO]
21+
22+
def colors: doobie.util.Colors = doobie.util.Colors.Ansi
23+
24+
def check[A](a: A)(using ev: Analyzable[A]): Unit =
25+
checkImpl(Analyzable.unpack(a))
26+
27+
private def checkImpl(args: AnalysisArgs): Unit =
28+
val report = doobie.util.testing
29+
.analyze(args)
30+
.transact(transactor)
31+
.unsafeRunSync()
32+
if !report.succeeded then
33+
fail(
34+
formatReport(args, report, colors)
35+
.padLeft(" ")
36+
.toString
37+
)
38+
end checkImpl
39+
end IOChecker

‎modules/infra/src/test/scala/scaladex/infra/sql/ArtifactTableTests.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,10 @@ class ArtifactTableTests extends AnyFunSpec with BaseDatabaseSuite with Matchers
3030
it("check selectOldestByProject")(check(selectOldestByProject))
3131
it("check updateProjectRef")(check(updateProjectRef))
3232
it("check selectGroupIds")(check(selectGroupIds))
33+
it("check selectGroupIdsPage")(check(selectGroupIdsPage))
3334
it("check selectReferences")(check(selectReferences))
35+
it("check selectReferencesByGroupId")(check(selectReferencesByGroupId))
36+
it("check selectReferencesByGroupIdPage")(check(selectReferencesByGroupIdPage))
3437
it("check selectReferencesByProject")(check(selectReferencesByProject))
3538
it("check updateReleaseDate")(check(updateReleaseDate))
3639
it("check selectByReference")(check(selectByReference))

0 commit comments

Comments
 (0)