Update metrics scheduler

Author: adpi2

modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala

@@ -1,6 +1,8 @@
 package scaladex.infra
 
 import java.time.Instant
+import java.time.OffsetDateTime
+import java.time.ZoneOffset
 import java.util.UUID
 
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -33,6 +35,7 @@ import scaladex.infra.sql.ProjectTable
 import scaladex.infra.sql.ReleaseDependenciesTable
 import scaladex.infra.sql.ReleaseTable
 import scaladex.infra.sql.UserSessionsTable
+import scaladex.core.model.BinaryVersion
 
 class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) extends SchedulerDatabase with LazyLogging {
   private val flyway = DoobieUtils.flyway(datasource)
@@ -150,6 +153,16 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
   def countProjects(): Future[Long] =
     run(ProjectTable.countProjects.unique)
 
+  def countProjects(year: Int): Future[Long] = {
+    val instant = OffsetDateTime.of(year + 1, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant
+    run(ProjectTable.countProjectsUntil.unique(instant))
+  }
+
+  def getProjectsByYear(year: Int): Future[Seq[(Project.Reference, Language)]] = {
+    val instant = OffsetDateTime.of(year + 1, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant
+    run(ArtifactTable.selectProjectsUntil.to[Seq](instant))
+  }
+
   override def countArtifacts(): Future[Long] =
     run(ArtifactTable.count.unique)
 

modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala

@@ -160,4 +160,12 @@ object ArtifactTable {
       groupBy = Seq("group_id", "artifact_id")
     )
   }
+
+  val selectProjectsUntil: Query[Instant, (Project.Reference, Language)] =
+    selectRequest1[Instant, (Project.Reference, Language)](
+      table,
+      Seq("organization", "repository", "language_version"),
+      where = Seq("release_date < ?"),
+      groupBy = Seq("organization", "repository", "language_version")
+    )
 }

modules/infra/src/main/scala/scaladex/infra/sql/ProjectTable.scala

@@ -45,6 +45,13 @@ object ProjectTable {
   val countProjects: Query0[Long] =
     selectRequest(table, Seq("count(*)"))
 
+  val countProjectsUntil: Query[Instant, Long] =
+    selectRequest1(
+      table,
+      Seq("count(*)"),
+      where = Seq("creation_date < ?", "github_status!='Moved'", "github_status!='NotFound'")
+    )
+
   val selectByReference: Query[Project.Reference, Project] =
     selectRequest(fullTable, allFields, referenceFields.map(f => s"p.$f"))
 

modules/server/src/main/scala/scaladex/server/service/AdminService.scala

@@ -12,15 +12,15 @@ import scaladex.core.model.Project
 import scaladex.core.model.Project.Settings
 import scaladex.core.model.UserState
 import scaladex.core.service.GithubClient
-import scaladex.core.service.SchedulerDatabase
 import scaladex.core.service.SearchEngine
 import scaladex.core.util.ScalaExtensions._
+import scaladex.infra.SqlDatabase
 import scaladex.view.Job
 import scaladex.view.Task
 
 class AdminService(
     env: Env,
-    database: SchedulerDatabase,
+    database: SqlDatabase,
     searchEngine: SearchEngine,
     githubClientOpt: Option[GithubClient],
     sonatypeSynchronizer: SonatypeService
@@ -39,7 +39,8 @@ class AdminService(
       new JobScheduler(Job.projectDependencies, projectDependenciesUpdater.updateAll),
       new JobScheduler(Job.projectCreationDates, updateProjectCreationDate),
       new JobScheduler(Job.moveArtifacts, artifactsService.moveAll),
-      new JobScheduler(Job.userSessions, userSessionService.updateAll)
+      new JobScheduler(Job.userSessions, userSessionService.updateAll),
+      new JobScheduler(Job.metrics, (new Metrics(database)).run)
     ) ++
       githubClientOpt.map { client =>
         val githubUpdater = new GithubUpdater(database, client)

modules/server/src/main/scala/scaladex/server/service/Metrics.scala

@@ -0,0 +1,44 @@
+package scaladex.server.service
+
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+
+import cats.implicits.toTraverseOps
+import com.typesafe.scalalogging.LazyLogging
+import scaladex.infra.SqlDatabase
+import scaladex.core.model.Scala
+
+class Metrics(db: SqlDatabase)(implicit ec: ExecutionContext) extends LazyLogging {
+  def run(): Future[String] = {
+    val years: Seq[Int] = Range.inclusive(2013, 2022)
+    for {
+      projectsByYear <- years.traverse(db.getProjectsByYear)
+      projects <- db.getAllProjects()
+    } yield {
+      val projectMap = projects.map(p => p.reference -> p).toMap
+      years.zip(projectsByYear).foreach { case (year, projects) =>
+        logger.info(s"$year:")
+        val all = projects
+          .groupMap(_._1)(_._2)
+          .view
+          .filterKeys(k => projectMap.get(k).filter(p => !p.githubStatus.isMoved && !p.githubStatus.isNotFound).nonEmpty)
+          .values.map(_.toSet).toSeq
+        val scala3  = all.count(_.contains(Scala.`3`))
+        val scala213  = all.count(ls => ls.contains(Scala.`2.13`) && !ls.contains(Scala.`3`))
+        val scala212 = all.count(ls => ls.contains(Scala.`2.12`) && !ls.contains(Scala.`2.13`) && !ls.contains(Scala.`3`))
+        val scala211 = all.count(ls => ls.contains(Scala.`2.11`) && !ls.contains(Scala.`2.12`) && !ls.contains(Scala.`2.13`) && !ls.contains(Scala.`3`))
+        val scala210 = all.count(ls => ls.contains(Scala.`2.10`) && !ls.contains(Scala.`2.11`) && !ls.contains(Scala.`2.12`) && !ls.contains(Scala.`2.13`) && !ls.contains(Scala.`3`))
+        logger.info(s"  Scala 2.10: $scala210")
+        logger.info(s"  Scala 2.11: $scala211")
+        logger.info(s"  Scala 2.12: $scala212")
+        logger.info(s"  Scala 2.13: $scala213")
+        logger.info(s"  Scala 3: $scala3")
+      }
+      val filteredProjects = projects.filter(p => !p.githubStatus.isMoved && !p.githubStatus.isNotFound)
+      logger.info(s"total projects: ${filteredProjects.size}")
+      val contributors = filteredProjects.flatMap(_.githubInfo).flatMap(_.contributors).map(_.login).distinct.size
+      logger.info(s"total contributors: $contributors")
+      "Success"
+    }
+  }
+}

modules/template/src/main/scala/scaladex/view/Job.scala

@@ -44,6 +44,11 @@ object Job {
     "Find missing artifacts in Maven Central of the known group IDs.",
     24.hours
   )
+  val metrics: Job = Job(
+    "metrics",
+    "Print regular metrics into the logs",
+    24.hours
+  )
 
   case class Status(state: State, results: Seq[Result], progress: Option[Progress]) {
     def isStarted: Boolean = state.isInstanceOf[Started]