Skip to content

Commit ce3da52

Browse files
adpi2adpi2
authored
Merge pull request #991 from mlachkar/github-info
GitHub info
2 parents 91e0d8d + cae342f commit ce3da52

File tree

13 files changed

+146
-87
lines changed

13 files changed

+146
-87
lines changed

‎modules/core/shared/src/main/scala/scaladex/core/model/GithubStatus.scala

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package scaladex.core.model
22

33
import java.time.Instant
44

5-
sealed trait GithubStatus {
5+
sealed trait GithubStatus extends Ordered[GithubStatus] {
66
val updateDate: Instant
77

88
override def toString: String = this match {
@@ -24,11 +24,19 @@ sealed trait GithubStatus {
2424
case GithubStatus.NotFound(_) => true
2525
case _ => false
2626
}
27+
override def compare(that: GithubStatus): Int =
28+
GithubStatus.ordering.compare(this, that)
29+
2730
}
2831
object GithubStatus {
2932
case class Ok(updateDate: Instant) extends GithubStatus
3033
case class Unknown(updateDate: Instant) extends GithubStatus
3134
case class Moved(updateDate: Instant, destination: Project.Reference) extends GithubStatus
3235
case class NotFound(updateDate: Instant) extends GithubStatus
3336
case class Failed(updateDate: Instant, errorCode: Int, errorMessage: String) extends GithubStatus
37+
38+
implicit val ordering: Ordering[GithubStatus] = Ordering.by {
39+
case GithubStatus.Unknown(_) => Instant.MIN
40+
case githubStatus: GithubStatus => githubStatus.updateDate
41+
}
3442
}

‎modules/core/shared/src/main/scala/scaladex/core/service/SchedulerDatabase.scala

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,8 @@ trait SchedulerDatabase extends WebDatabase {
2020
def getDependencies(projectRef: Project.Reference): Future[Seq[ArtifactDependency]]
2121
def updateArtifacts(artifacts: Seq[Artifact], newRef: Project.Reference): Future[Int]
2222
def updateArtifactReleaseDate(reference: MavenReference, releaseDate: Instant): Future[Int]
23-
def updateGithubInfoAndStatus(ref: Project.Reference, githubInfo: GithubInfo, status: GithubStatus): Future[Unit]
24-
def moveProject(ref: Project.Reference, githubInfo: GithubInfo, status: GithubStatus.Moved): Future[Unit]
25-
def updateGithubStatus(ref: Project.Reference, githubStatus: GithubStatus): Future[Unit]
2623
def computeProjectDependencies(): Future[Seq[ProjectDependency]]
24+
def updateGithubInfoAndStatus(ref: Project.Reference, githubInfo: GithubInfo, status: GithubStatus): Future[Unit]
2725
def computeAllProjectsCreationDates(): Future[Seq[(Instant, Project.Reference)]]
2826
def updateProjectCreationDate(ref: Project.Reference, creationDate: Instant): Future[Unit]
2927
def insertProjectDependencies(projectDependencies: Seq[ProjectDependency]): Future[Int]

‎modules/core/shared/src/main/scala/scaladex/core/service/WebDatabase.scala

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,14 @@ import scala.concurrent.Future
77

88
import scaladex.core.model.Artifact
99
import scaladex.core.model.ArtifactDependency
10+
import scaladex.core.model.GithubInfo
11+
import scaladex.core.model.GithubResponse
1012
import scaladex.core.model.Project
1113
import scaladex.core.model.UserState
1214

1315
trait WebDatabase {
14-
def insertArtifact(artifact: Artifact, dependencies: Seq[ArtifactDependency], time: Instant): Future[Unit]
16+
// insertArtifact return a boolean. It's true if the a new project is inserted, false otherwise
17+
def insertArtifact(artifact: Artifact, dependencies: Seq[ArtifactDependency], time: Instant): Future[Boolean]
1518
def updateProjectSettings(ref: Project.Reference, settings: Project.Settings): Future[Unit]
1619
def getAllProjects(): Future[Seq[Project]]
1720
def getProject(projectRef: Project.Reference): Future[Option[Project]]
@@ -25,4 +28,9 @@ trait WebDatabase {
2528
def countArtifacts(): Future[Long]
2629
def insertSession(userId: UUID, userState: UserState): Future[Unit]
2730
def getSession(userId: UUID): Future[Option[UserState]]
31+
def updateGithubInfo(
32+
repo: Project.Reference,
33+
response: GithubResponse[(Project.Reference, GithubInfo)],
34+
now: Instant
35+
): Future[Unit]
2836
}

‎modules/core/shared/src/test/scala/scaladex/core/model/GithubStatusTests.scala

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package scaladex.core.model
2+
3+
import java.time.Instant
4+
5+
import org.scalatest.funspec.AnyFunSpec
6+
import org.scalatest.matchers.should.Matchers
7+
8+
class GithubStatusTests extends AnyFunSpec with Matchers {
9+
val date: Instant = Instant.ofEpochMilli(1475505237265L)
10+
val now: Instant = Instant.now()
11+
describe("githubStatus") {
12+
it("should order correctly") {
13+
val unknown = GithubStatus.Unknown(now)
14+
val ok = GithubStatus.Ok(date)
15+
Seq(ok, unknown).sorted shouldBe Seq(unknown, ok)
16+
}
17+
}
18+
}

‎modules/core/shared/src/test/scala/scaladex/core/test/InMemoryDatabase.scala

Lines changed: 27 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import scala.concurrent.Future
99
import scaladex.core.model.Artifact
1010
import scaladex.core.model.ArtifactDependency
1111
import scaladex.core.model.GithubInfo
12+
import scaladex.core.model.GithubResponse
1213
import scaladex.core.model.GithubStatus
1314
import scaladex.core.model.Project
1415
import scaladex.core.model.ProjectDependency
@@ -26,22 +27,17 @@ class InMemoryDatabase extends SchedulerDatabase {
2627
dependencies.clear()
2728
}
2829

29-
override def moveProject(
30-
ref: Project.Reference,
31-
githubInfo: GithubInfo,
32-
status: GithubStatus.Moved
33-
): Future[Unit] = ???
34-
3530
override def insertArtifact(
3631
artifact: Artifact,
3732
dependencies: Seq[ArtifactDependency],
3833
now: Instant
39-
): Future[Unit] = {
34+
): Future[Boolean] = {
4035
val ref = artifact.projectRef
41-
if (!projects.contains(ref)) projects.addOne(ref -> Project.default(ref, now = now))
36+
val isNewProject = !projects.contains(ref)
37+
if (isNewProject) projects.addOne(ref -> Project.default(ref, now = now))
4238
artifacts.addOne(ref -> (artifacts.getOrElse(ref, Seq.empty) :+ artifact))
4339
dependencies.appendedAll(dependencies)
44-
Future.successful(())
40+
Future.successful(isNewProject)
4541
}
4642

4743
override def insertProject(project: Project): Future[Unit] = ???
@@ -93,17 +89,6 @@ class InMemoryDatabase extends SchedulerDatabase {
9389

9490
override def getAllProjects(): Future[Seq[Project]] = ???
9591

96-
override def updateGithubInfoAndStatus(
97-
ref: Project.Reference,
98-
githubInfo: GithubInfo,
99-
githubStatus: GithubStatus
100-
): Future[Unit] =
101-
Future.successful(
102-
projects.update(ref, projects(ref).copy(githubInfo = Some(githubInfo), githubStatus = githubStatus))
103-
)
104-
105-
override def updateGithubStatus(ref: Project.Reference, githubStatus: GithubStatus): Future[Unit] = ???
106-
10792
override def computeProjectDependencies(): Future[Seq[ProjectDependency]] = ???
10893

10994
override def computeAllProjectsCreationDates(): Future[Seq[(Instant, Project.Reference)]] = ???
@@ -124,4 +109,26 @@ class InMemoryDatabase extends SchedulerDatabase {
124109
override def insertSession(userId: UUID, userState: UserState): Future[Unit] = ???
125110
override def getSession(userId: UUID): Future[Option[UserState]] = ???
126111
override def updateArtifactReleaseDate(reference: Artifact.MavenReference, releaseDate: Instant): Future[Int] = ???
112+
113+
override def updateGithubInfoAndStatus(
114+
ref: Project.Reference,
115+
githubInfo: GithubInfo,
116+
githubStatus: GithubStatus
117+
): Future[Unit] =
118+
Future.successful(
119+
projects.update(ref, projects(ref).copy(githubInfo = Some(githubInfo), githubStatus = githubStatus))
120+
)
121+
122+
override def updateGithubInfo(
123+
repo: Project.Reference,
124+
response: GithubResponse[(Project.Reference, GithubInfo)],
125+
now: Instant
126+
): Future[Unit] =
127+
response match {
128+
case GithubResponse.Ok((repo, githubInfo)) =>
129+
Future.successful(
130+
projects.update(repo, projects(repo).copy(githubInfo = Some(githubInfo), githubStatus = GithubStatus.Ok(now)))
131+
)
132+
case _ => Future.successful(())
133+
}
127134
}

‎modules/infra/src/main/scala/scaladex/infra/GithubClient.scala

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import akka.stream.scaladsl.Flow
2626
import akka.util.ByteString
2727
import com.typesafe.scalalogging.LazyLogging
2828
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport._
29+
import io.circe.Json
2930
import io.circe.syntax._
3031
import scaladex.core.model.GithubInfo
3132
import scaladex.core.model.GithubResponse
@@ -381,7 +382,9 @@ class GithubClient(token: Secret)(implicit val system: ActorSystem)
381382
case other => other
382383
}
383384
case _ @HttpResponse(code, _, entity, _) =>
384-
Unmarshal(entity).to[String].map(errorMessage => GithubResponse.Failed(code.intValue, errorMessage))
385+
if (entity.contentType.mediaType.isApplication) { // we need to parse as json when the mediaType is application/json
386+
Unmarshal(entity).to[Json].map(errorMessage => GithubResponse.Failed(code.intValue, errorMessage.toString()))
387+
} else Unmarshal(entity).to[String].map(errorMessage => GithubResponse.Failed(code.intValue, errorMessage))
385388
}
386389
}
387390

‎modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala

Lines changed: 44 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import doobie.implicits._
1212
import scaladex.core.model.Artifact
1313
import scaladex.core.model.ArtifactDependency
1414
import scaladex.core.model.GithubInfo
15+
import scaladex.core.model.GithubResponse
1516
import scaladex.core.model.GithubStatus
1617
import scaladex.core.model.Project
1718
import scaladex.core.model.ProjectDependency
@@ -37,12 +38,13 @@ class SqlDatabase(conf: PostgreSQLConfig, xa: doobie.Transactor[IO]) extends Sch
3738
artifact: Artifact,
3839
dependencies: Seq[ArtifactDependency],
3940
time: Instant
40-
): Future[Unit] = {
41+
): Future[Boolean] = {
4142
val unknownStatus = GithubStatus.Unknown(time)
42-
val insertArtifactF = insertProjectRef(artifact.projectRef, unknownStatus)
43-
.flatMap(_ => run(ArtifactTable.insertIfNotExist.run(artifact)))
44-
val insertDepsF = insertDependencies(dependencies)
45-
insertArtifactF.flatMap(_ => insertDepsF).map(_ => ())
43+
for {
44+
isNewProject <- insertProjectRef(artifact.projectRef, unknownStatus)
45+
_ <- run(ArtifactTable.insertIfNotExist.run(artifact))
46+
_ <- insertDependencies(dependencies)
47+
} yield isNewProject
4648
}
4749

4850
override def insertProject(project: Project): Future[Unit] =
@@ -84,8 +86,27 @@ class SqlDatabase(conf: PostgreSQLConfig, xa: doobie.Transactor[IO]) extends Sch
8486
override def updateArtifactReleaseDate(reference: Artifact.MavenReference, releaseDate: Instant): Future[Int] =
8587
run(ArtifactTable.updateReleaseDate.run((releaseDate, reference)))
8688

87-
override def updateGithubStatus(ref: Project.Reference, githubStatus: GithubStatus): Future[Unit] =
88-
run(ProjectTable.updateGithubStatus.run(githubStatus, ref)).map(_ => ())
89+
override def updateGithubInfo(
90+
repo: Project.Reference,
91+
response: GithubResponse[(Project.Reference, GithubInfo)],
92+
now: Instant
93+
): Future[Unit] =
94+
response match {
95+
case GithubResponse.Ok((_, info)) =>
96+
val status = GithubStatus.Ok(now)
97+
updateGithubInfoAndStatus(repo, info, status)
98+
99+
case GithubResponse.MovedPermanently((destination, info)) =>
100+
val status = GithubStatus.Moved(now, destination)
101+
logger.info(s"$repo moved to $status")
102+
moveProject(repo, info, status)
103+
104+
case GithubResponse.Failed(code, reason) =>
105+
val status =
106+
if (code == 404) GithubStatus.NotFound(now) else GithubStatus.Failed(now, code, reason)
107+
logger.info(s"Failed to download github info for $repo because of $status")
108+
updateGithubStatus(repo, status)
109+
}
89110

90111
override def updateGithubInfoAndStatus(
91112
ref: Project.Reference,
@@ -97,19 +118,6 @@ class SqlDatabase(conf: PostgreSQLConfig, xa: doobie.Transactor[IO]) extends Sch
97118
_ <- run(GithubInfoTable.insertOrUpdate.run((ref, githubInfo, githubInfo)))
98119
} yield ()
99120

100-
override def moveProject(
101-
ref: Project.Reference,
102-
githubInfo: GithubInfo,
103-
status: GithubStatus.Moved
104-
): Future[Unit] =
105-
for {
106-
oldProject <- getProject(ref)
107-
_ <- updateGithubStatus(ref, status)
108-
_ <- run(ProjectTable.insertIfNotExists.run((status.destination, GithubStatus.Ok(status.updateDate))))
109-
_ <- updateProjectSettings(status.destination, oldProject.map(_.settings).getOrElse(Project.Settings.default))
110-
_ <- run(GithubInfoTable.insertOrUpdate.run(status.destination, githubInfo, githubInfo))
111-
} yield ()
112-
113121
override def updateProjectSettings(ref: Project.Reference, settings: Project.Settings): Future[Unit] =
114122
run(ProjectSettingsTable.insertOrUpdate.run((ref, settings, settings))).map(_ => ())
115123

@@ -189,6 +197,22 @@ class SqlDatabase(conf: PostgreSQLConfig, xa: doobie.Transactor[IO]) extends Sch
189197
override def getSession(userId: UUID): Future[Option[UserState]] =
190198
run(UserSessionsTable.selectUserSessionById.to[Seq](userId)).map(_.headOption)
191199

200+
def moveProject(
201+
ref: Project.Reference,
202+
githubInfo: GithubInfo,
203+
status: GithubStatus.Moved
204+
): Future[Unit] =
205+
for {
206+
oldProject <- getProject(ref)
207+
_ <- updateGithubStatus(ref, status)
208+
_ <- run(ProjectTable.insertIfNotExists.run((status.destination, GithubStatus.Ok(status.updateDate))))
209+
_ <- updateProjectSettings(status.destination, oldProject.map(_.settings).getOrElse(Project.Settings.default))
210+
_ <- run(GithubInfoTable.insertOrUpdate.run(status.destination, githubInfo, githubInfo))
211+
} yield ()
212+
213+
def updateGithubStatus(ref: Project.Reference, githubStatus: GithubStatus): Future[Unit] =
214+
run(ProjectTable.updateGithubStatus.run(githubStatus, ref)).map(_ => ())
215+
192216
private def run[A](v: doobie.ConnectionIO[A]): Future[A] =
193217
v.transact(xa).unsafeToFuture()
194218
}

‎modules/infra/src/main/scala/scaladex/infra/migrations/V7_2__edit_platform_and_language.scala

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,10 @@ class V7_2__edit_platform_and_language extends BaseJavaMigration with ScaladexBa
2424
(for {
2525
oldArtifacts <- run(xa)(selectArtifact.to[Seq])
2626
groupedArtifacts = oldArtifacts.grouped(10000).toSeq
27-
numberOfArtifactsUpdated <- groupedArtifacts
27+
_ <- groupedArtifacts
2828
.map(artifacts => run(xa)(updatePlatformAndLanguage.updateMany(artifacts.map(_.update))))
2929
.sequence
30-
_ = logger.info(s"Updated ${numberOfArtifactsUpdated.sum} artifacts")
3130
_ <- run(xa)(sql"ALTER TABLE artifacts DROP COLUMN binary_version".update.run)
32-
_ = logger.info(s"column binary_version deleted")
3331
} yield ())
3432
.unsafeRunSync()
3533

‎modules/infra/src/main/scala/scaladex/infra/migrations/V9__fix_platform_and_language.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,11 @@ class V9__fix_platform_and_language extends BaseJavaMigration with ScaladexBaseM
1616
(for {
1717
artifactToFix <- run(xa)(selectArtifact.to[Seq])
1818
artifactToFixWithIds = artifactToFix.flatMap(a => Artifact.ArtifactId.parse(a.artifactId).map(a -> _))
19-
numberOfArtifactsUpdated <- run(xa) {
19+
_ <- run(xa) {
2020
updatePlatformAndLanguage.updateMany(artifactToFixWithIds.map {
2121
case (artifact, id) => (id.binaryVersion.platform, id.binaryVersion.language, artifact.mavenReference)
2222
})
2323
}
24-
_ = logger.info(s"Updated $numberOfArtifactsUpdated artifacts")
2524
} yield ())
2625
.unsafeRunSync()
2726

‎modules/server/src/main/scala/scaladex/server/Server.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ object Server extends LazyLogging {
6868
val githubService = config.github.token.map(new GithubClient(_))
6969
val paths = DataPaths.from(config.filesystem)
7070
val filesystem = FilesystemStorage(config.filesystem)
71-
val publishProcess = PublishProcess(paths, filesystem, webDatabase, config.env)(publishPool)
71+
val publishProcess = PublishProcess(paths, filesystem, webDatabase, config.env)(publishPool, system)
7272
val sonatypeClient = new SonatypeClient()
7373
val sonatypeSynchronizer = new SonatypeSynchronizer(schedulerDatabase, sonatypeClient, publishProcess)
7474
val adminTaskService = new AdminTaskService(sonatypeSynchronizer)

‎modules/server/src/main/scala/scaladex/server/service/GithubUpdater.scala

Lines changed: 6 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,6 @@ import scala.concurrent.ExecutionContext
66
import scala.concurrent.Future
77
import scala.concurrent.duration.DurationInt
88

9-
import scaladex.core.model.GithubInfo
10-
import scaladex.core.model.GithubResponse
11-
import scaladex.core.model.GithubStatus
129
import scaladex.core.model.Project
1310
import scaladex.core.service.GithubService
1411
import scaladex.core.service.SchedulerDatabase
@@ -19,7 +16,11 @@ class GithubUpdater(database: SchedulerDatabase, githubService: GithubService)(i
1916
override def run(): Future[Unit] =
2017
database.getAllProjectsStatuses().flatMap { projectStatuses =>
2118
val projectToUpdate =
22-
projectStatuses.collect { case (ref, status) if !status.moved && !status.notFound => ref }.toSeq
19+
projectStatuses
20+
.filter { case (_, status) => !status.moved && !status.notFound }
21+
.toSeq
22+
.sortBy(_._2)
23+
.map(_._1)
2324

2425
logger.info(s"Updating github info of ${projectToUpdate.size} projects")
2526
projectToUpdate.mapSync(update).map(_ => ())
@@ -29,29 +30,7 @@ class GithubUpdater(database: SchedulerDatabase, githubService: GithubService)(i
2930
val now = Instant.now()
3031
for {
3132
response <- githubService.getProjectInfo(ref)
32-
_ <- updateDbAndLog(ref, response, now)
33+
_ <- database.updateGithubInfo(ref, response, now)
3334
} yield ()
3435
}
35-
36-
def updateDbAndLog(
37-
repo: Project.Reference,
38-
response: GithubResponse[(Project.Reference, GithubInfo)],
39-
now: Instant
40-
): Future[Unit] =
41-
response match {
42-
case GithubResponse.Ok((_, info)) =>
43-
val status = GithubStatus.Ok(now)
44-
database.updateGithubInfoAndStatus(repo, info, status)
45-
46-
case GithubResponse.MovedPermanently((destination, info)) =>
47-
val status = GithubStatus.Moved(now, destination)
48-
logger.info(s"$repo moved to $status")
49-
database.moveProject(repo, info, status)
50-
51-
case GithubResponse.Failed(code, reason) =>
52-
val status =
53-
if (code == 404) GithubStatus.NotFound(now) else GithubStatus.Failed(now, code, reason)
54-
logger.info(s"Failed to download github info for $repo because of $status")
55-
database.updateGithubStatus(repo, status)
56-
}
5736
}

0 commit comments

Comments
 (0)