Update metrics scheduler

Author: adpi2

modules/core/shared/src/main/scala/scaladex/core/model/GithubInfo.scala

@@ -41,7 +41,8 @@ case class GithubInfo(
     codeOfConduct: Option[Url],
     chatroom: Option[Url],
     openIssues: Seq[GithubIssue], // right now it's all issues, not only beginners issues
-    scalaPercentage: Option[Int]
+    scalaPercentage: Option[Int],
+    scalaLines: Option[Int]
 ) {
   val contributorCount: Int = contributors.size
 
@@ -80,6 +81,7 @@ object GithubInfo {
     codeOfConduct = None,
     chatroom = None,
     openIssues = Seq(),
-    scalaPercentage = None
+    scalaPercentage = None,
+    scalaLines = None
   )
 }

modules/infra/src/main/scala/scaladex/infra/GithubClient.scala

@@ -85,7 +85,7 @@ class GithubClient(token: Secret)(implicit val system: ActorSystem)
       contributors <- getContributors(repo.ref)
       openIssues <- getOpenIssues(repo.ref)
       chatroom <- getGitterChatRoom(repo.ref)
-      scalaPercentage <- getPercentageOfLanguage(repo.ref, language = "Scala")
+      (scalaLines, scalaPercentage) <- getPercentageOfLanguage(repo.ref, language = "Scala")
     } yield GithubInfo(
       homepage = repo.homepage.map(Url),
       description = repo.description,
@@ -103,7 +103,8 @@ class GithubClient(token: Secret)(implicit val system: ActorSystem)
       codeOfConduct = communityProfile.flatMap(_.codeOfConductFile).map(Url),
       chatroom = chatroom.map(Url),
       openIssues = openIssues.map(_.toGithubIssue).toList,
-      scalaPercentage = Option(scalaPercentage)
+      scalaPercentage = Option(scalaPercentage),
+      scalaLines = scalaLines
     )
 
   def getReadme(ref: Project.Reference): Future[Option[String]] = {
@@ -191,14 +192,15 @@ class GithubClient(token: Secret)(implicit val system: ActorSystem)
       .fallbackTo(Future.successful(Seq.empty))
   }
 
-  def getPercentageOfLanguage(ref: Project.Reference, language: String): Future[Int] = {
+  def getPercentageOfLanguage(ref: Project.Reference, language: String): Future[(Option[Int], Int)] = {
     def toPercentage(portion: Int, total: Int): Int =
       ((portion.toFloat / total) * 100).toInt
     val request =
       HttpRequest(uri = s"${repoUrl(ref)}/languages").addHeader(acceptJson).addCredentials(credentials)
     get[Map[String, Int]](request).map { response =>
       val totalNumBytes = response.values.sum
-      response.get(language).fold(0)(toPercentage(_, totalNumBytes))
+      val lines = response.get(language)
+      (lines, lines.fold(0)(toPercentage(_, totalNumBytes)))
     }
   }
 

modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala

@@ -30,6 +30,8 @@ import scaladex.infra.sql.ProjectTable
 import scaladex.infra.sql.ReleaseDependenciesTable
 import scaladex.infra.sql.ReleaseTable
 import scaladex.infra.sql.UserSessionsTable
+import java.time.OffsetDateTime
+import java.time.ZoneOffset
 
 class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) extends SchedulerDatabase with LazyLogging {
   private val flyway = DoobieUtils.flyway(datasource)
@@ -148,6 +150,11 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
   def countProjects(): Future[Long] =
     run(ProjectTable.countProjects.unique)
 
+  def countProjects(year: Int): Future[Long] = {
+    val instant = OffsetDateTime.of(year + 1, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant
+    run(ProjectTable.countProjectsUntil.unique(instant))
+  }
+
   override def countArtifacts(): Future[Long] =
     run(ArtifactTable.count.unique)
 

modules/infra/src/main/scala/scaladex/infra/sql/GithubInfoTable.scala

@@ -26,7 +26,8 @@ object GithubInfoTable {
     "code_of_conduct",
     "chatroom",
     "open_issues",
-    "scala_percentage"
+    "scala_percentage",
+    "scala_lines"
   )
 
   val insert: Update[(Project.Reference, GithubInfo)] =

modules/infra/src/main/scala/scaladex/infra/sql/ProjectTable.scala

@@ -44,6 +44,9 @@ object ProjectTable {
 
   val countProjects: Query0[Long] =
     selectRequest(table, Seq("count(*)"))
+  
+  val countProjectsUntil: Query[Instant, Long] =
+    selectRequest1(table, "count(*)", where = Some("creation_date < ? AND github_status!='Moved' AND github_status!='NotFound'"))
 
   val selectByReference: Query[Project.Reference, Project] =
     selectRequest(fullTable, allFields, referenceFields.map(f => s"p.$f"))

modules/server/src/main/scala/scaladex/server/service/Metrics.scala

@@ -0,0 +1,27 @@
+package scaladex.server.service
+
+import com.typesafe.scalalogging.LazyLogging
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+import scaladex.infra.SqlDatabase
+import cats.implicits.toTraverseOps
+
+class Metrics(db: SqlDatabase)(implicit ec: ExecutionContext) extends LazyLogging {
+  def run(): Future[Unit] = {
+    val years: Seq[Int] = Range.inclusive(2013, 2022)
+    for {
+      countByYear <- years.traverse(db.countProjects)
+      projects <- db.getAllProjects()
+    } yield {
+      years.zip(countByYear).foreach {
+        case (year, count) => logger.info(s"$year: $count")
+      }
+      val filteredProjects = projects.filter(p => !p.githubStatus.moved && !p.githubStatus.notFound)
+      logger.info(s"total projects: ${filteredProjects.size}")
+      val linesOfCode = filteredProjects.flatMap(_.githubInfo).flatMap(_.scalaLines).map(_.toLong).sum
+      logger.info(s"total lines of Scala code: $linesOfCode")
+      val contributors = filteredProjects.flatMap(_.githubInfo).flatMap(_.contributors).map(_.login).distinct.size
+      logger.info(s"total contributors: $contributors")
+    }
+  }
+}

modules/server/src/main/scala/scaladex/server/service/SchedulerService.scala

@@ -11,10 +11,11 @@ import scaladex.core.service.SearchEngine
 import scaladex.core.util.ScalaExtensions._
 import scaladex.infra.GithubClient
 import scaladex.view.SchedulerStatus
+import scaladex.infra.SqlDatabase
 
 class SchedulerService(
     env: Env,
-    database: SchedulerDatabase,
+    database: SqlDatabase,
     searchEngine: SearchEngine,
     githubClientOpt: Option[GithubClient],
     sonatypeSynchronizer: SonatypeSynchronizer,
@@ -30,7 +31,8 @@ class SchedulerService(
     Scheduler("sync-search", searchSynchronizer.syncAll, 30.minutes),
     new MovedArtifactsSynchronizer(database),
     userSessionSynchronizer,
-    new MovedArtifactsSynchronizer(database)
+    new MovedArtifactsSynchronizer(database),
+    Scheduler("metrics", (new Metrics(database)).run, 1.hour),
   ) ++
     Option.when(!env.isLocal)(Scheduler("sync-sonatype-missing-releases", sonatypeSynchronizer.syncAll, 24.hours)) ++
     githubClientOpt.map(client => new GithubUpdater(database, client))