diff --git a/modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala b/modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala index 9301a5c37..00c62422d 100644 --- a/modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala +++ b/modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala @@ -1,6 +1,8 @@ package scaladex.infra import java.time.Instant +import java.time.OffsetDateTime +import java.time.ZoneOffset import java.util.UUID import scala.concurrent.ExecutionContext.Implicits.global @@ -33,6 +35,7 @@ import scaladex.infra.sql.ProjectTable import scaladex.infra.sql.ReleaseDependenciesTable import scaladex.infra.sql.ReleaseTable import scaladex.infra.sql.UserSessionsTable +import scaladex.core.model.BinaryVersion class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) extends SchedulerDatabase with LazyLogging { private val flyway = DoobieUtils.flyway(datasource) @@ -150,6 +153,16 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten def countProjects(): Future[Long] = run(ProjectTable.countProjects.unique) + def countProjects(year: Int): Future[Long] = { + val instant = OffsetDateTime.of(year + 1, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant + run(ProjectTable.countProjectsUntil.unique(instant)) + } + + def getProjectsByYear(year: Int): Future[Seq[(Project.Reference, Language)]] = { + val instant = OffsetDateTime.of(year + 1, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant + run(ArtifactTable.selectProjectsUntil.to[Seq](instant)) + } + override def countArtifacts(): Future[Long] = run(ArtifactTable.count.unique) diff --git a/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala b/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala index f531e7c74..623a1e36b 100644 --- a/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala +++ b/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala @@ -160,4 +160,12 @@ object ArtifactTable { groupBy = Seq("group_id", "artifact_id") ) } + + val selectProjectsUntil: Query[Instant, (Project.Reference, Language)] = + selectRequest1[Instant, (Project.Reference, Language)]( + table, + Seq("organization", "repository", "language_version"), + where = Seq("release_date < ?"), + groupBy = Seq("organization", "repository", "language_version") + ) } diff --git a/modules/infra/src/main/scala/scaladex/infra/sql/ProjectTable.scala b/modules/infra/src/main/scala/scaladex/infra/sql/ProjectTable.scala index 31c6c11c6..dab9a3c20 100644 --- a/modules/infra/src/main/scala/scaladex/infra/sql/ProjectTable.scala +++ b/modules/infra/src/main/scala/scaladex/infra/sql/ProjectTable.scala @@ -45,6 +45,13 @@ object ProjectTable { val countProjects: Query0[Long] = selectRequest(table, Seq("count(*)")) + val countProjectsUntil: Query[Instant, Long] = + selectRequest1( + table, + Seq("count(*)"), + where = Seq("creation_date < ?", "github_status!='Moved'", "github_status!='NotFound'") + ) + val selectByReference: Query[Project.Reference, Project] = selectRequest(fullTable, allFields, referenceFields.map(f => s"p.$f")) diff --git a/modules/server/src/main/scala/scaladex/server/service/AdminService.scala b/modules/server/src/main/scala/scaladex/server/service/AdminService.scala index 6acd34d91..7ac529012 100644 --- a/modules/server/src/main/scala/scaladex/server/service/AdminService.scala +++ b/modules/server/src/main/scala/scaladex/server/service/AdminService.scala @@ -12,15 +12,15 @@ import scaladex.core.model.Project import scaladex.core.model.Project.Settings import scaladex.core.model.UserState import scaladex.core.service.GithubClient -import scaladex.core.service.SchedulerDatabase import scaladex.core.service.SearchEngine import scaladex.core.util.ScalaExtensions._ +import scaladex.infra.SqlDatabase import scaladex.view.Job import scaladex.view.Task class AdminService( env: Env, - database: SchedulerDatabase, + database: SqlDatabase, searchEngine: SearchEngine, githubClientOpt: Option[GithubClient], sonatypeSynchronizer: SonatypeService @@ -39,7 +39,8 @@ class AdminService( new JobScheduler(Job.projectDependencies, projectDependenciesUpdater.updateAll), new JobScheduler(Job.projectCreationDates, updateProjectCreationDate), new JobScheduler(Job.moveArtifacts, artifactsService.moveAll), - new JobScheduler(Job.userSessions, userSessionService.updateAll) + new JobScheduler(Job.userSessions, userSessionService.updateAll), + new JobScheduler(Job.metrics, (new Metrics(database)).run) ) ++ githubClientOpt.map { client => val githubUpdater = new GithubUpdater(database, client) diff --git a/modules/server/src/main/scala/scaladex/server/service/Metrics.scala b/modules/server/src/main/scala/scaladex/server/service/Metrics.scala new file mode 100644 index 000000000..12d367fab --- /dev/null +++ b/modules/server/src/main/scala/scaladex/server/service/Metrics.scala @@ -0,0 +1,44 @@ +package scaladex.server.service + +import scala.concurrent.ExecutionContext +import scala.concurrent.Future + +import cats.implicits.toTraverseOps +import com.typesafe.scalalogging.LazyLogging +import scaladex.infra.SqlDatabase +import scaladex.core.model.Scala + +class Metrics(db: SqlDatabase)(implicit ec: ExecutionContext) extends LazyLogging { + def run(): Future[String] = { + val years: Seq[Int] = Range.inclusive(2013, 2022) + for { + projectsByYear <- years.traverse(db.getProjectsByYear) + projects <- db.getAllProjects() + } yield { + val projectMap = projects.map(p => p.reference -> p).toMap + years.zip(projectsByYear).foreach { case (year, projects) => + logger.info(s"$year:") + val all = projects + .groupMap(_._1)(_._2) + .view + .filterKeys(k => projectMap.get(k).filter(p => !p.githubStatus.isMoved && !p.githubStatus.isNotFound).nonEmpty) + .values.map(_.toSet).toSeq + val scala3 = all.count(_.contains(Scala.`3`)) + val scala213 = all.count(ls => ls.contains(Scala.`2.13`) && !ls.contains(Scala.`3`)) + val scala212 = all.count(ls => ls.contains(Scala.`2.12`) && !ls.contains(Scala.`2.13`) && !ls.contains(Scala.`3`)) + val scala211 = all.count(ls => ls.contains(Scala.`2.11`) && !ls.contains(Scala.`2.12`) && !ls.contains(Scala.`2.13`) && !ls.contains(Scala.`3`)) + val scala210 = all.count(ls => ls.contains(Scala.`2.10`) && !ls.contains(Scala.`2.11`) && !ls.contains(Scala.`2.12`) && !ls.contains(Scala.`2.13`) && !ls.contains(Scala.`3`)) + logger.info(s" Scala 2.10: $scala210") + logger.info(s" Scala 2.11: $scala211") + logger.info(s" Scala 2.12: $scala212") + logger.info(s" Scala 2.13: $scala213") + logger.info(s" Scala 3: $scala3") + } + val filteredProjects = projects.filter(p => !p.githubStatus.isMoved && !p.githubStatus.isNotFound) + logger.info(s"total projects: ${filteredProjects.size}") + val contributors = filteredProjects.flatMap(_.githubInfo).flatMap(_.contributors).map(_.login).distinct.size + logger.info(s"total contributors: $contributors") + "Success" + } + } +} diff --git a/modules/template/src/main/scala/scaladex/view/Job.scala b/modules/template/src/main/scala/scaladex/view/Job.scala index 20ae3f03f..64df86675 100644 --- a/modules/template/src/main/scala/scaladex/view/Job.scala +++ b/modules/template/src/main/scala/scaladex/view/Job.scala @@ -44,6 +44,11 @@ object Job { "Find missing artifacts in Maven Central of the known group IDs.", 24.hours ) + val metrics: Job = Job( + "metrics", + "Print regular metrics into the logs", + 24.hours + ) case class Status(state: State, results: Seq[Result], progress: Option[Progress]) { def isStarted: Boolean = state.isInstanceOf[Started]