improvement: Add an ability to discover new groups

README.md

@@ -55,6 +55,23 @@ If your project is not hosted by Github you should consider creating a mirror of
 
 Do not forget to update the `scmInfo` setting in your build file before the next release.
 
+#### Did you publish via Sonatype Central Portal?
+
+If you published your artifacts using the new Sonatype Central Portal (e.g., via sbt-typelevel's `tlCiRelease`), they may not appear in Scaladex automatically.
+This is because the Central Portal does not notify Scaladex when new artifacts are published, and Scaladex's periodic scan only checks group IDs it already knows about.
+
+To have your artifacts indexed, submit your group ID to the [new-groups.json](https://github.com/scalacenter/scaladex-contrib/blob/master/new-groups.json) file in the [scaladex-contrib](https://github.com/scalacenter/scaladex-contrib) repository.
+The file format is a simple JSON array of group ID strings:
+
+```json
+[
+  "dev.constructive",
+  "com.example.mylib"
+]
+```
+
+Once merged, Scaladex will discover and index all Scala artifacts from your group ID within 6 hours.
+
 ### My project is hard to find in the search page
 
 Read [How to improve the visibility of your project](doc/user/improve-visibility.md).

modules/data/src/main/scala/scaladex/data/cleanup/NewGroupId.scala

@@ -0,0 +1,41 @@
+package scaladex.data
+package cleanup
+
+import java.nio.file.*
+
+import scala.io.Source
+import scala.util.Using
+
+import scaladex.infra.DataPaths
+
+import io.circe.parser.*
+
+/** A group ID submitted by the community for discovery and indexing.
+  *
+  * Library authors publishing via Sonatype Central Portal may find that their artifacts don't appear in Scaladex
+  * because the legacy webhook doesn't fire and the scheduled backfill job only scans group IDs already in the database.
+  *
+  * By submitting their group ID to new-groups.json, library authors can ensure their artifacts are discovered and
+  * indexed.
+  *
+  * @param groupId
+  *   the Maven group ID (e.g., "dev.constructive")
+  */
+case class NewGroupId(groupId: String)
+
+object NewGroupId:
+
+  /** Load new group IDs from the new-groups.json file.
+    *
+    * The file format is a JSON array of group ID strings: ["dev.constructive", "com.example", ...]
+    */
+  def load(paths: DataPaths): List[NewGroupId] =
+    val filePath = paths.newGroups
+    if Files.exists(filePath) then
+      val input = Using.resource(Source.fromFile(filePath.toFile))(_.mkString)
+      decode[List[String]](input) match
+        case Right(groupIds) => groupIds.map(NewGroupId.apply)
+        case Left(_) => List()
+    else List()
+  end load
+end NewGroupId

modules/infra/src/main/scala/scaladex/infra/DataPaths.scala

@@ -14,6 +14,7 @@ a PR.
 scaladex-contrib
 ├── claims.json
 ├── licensesByName.json
+├── new-groups.json
 └── non-standard.json
 
 scaladex-credentials (optionnal)
@@ -28,11 +29,12 @@ case class DataPaths(contrib: Path):
 
   val claims: Path = initJsonFile(contrib, "claims.json")
   val nonStandard: Path = initJsonFile(contrib, "non-standard.json")
+  val newGroups: Path = initJsonFile(contrib, "new-groups.json", "[]")
 
-  private def initJsonFile(parent: Path, name: String): Path =
+  private def initJsonFile(parent: Path, name: String, defaultContent: String = "{}"): Path =
     val file = parent.resolve(name)
     if !Files.exists(file) then
       Files.createFile(file)
-      Files.write(file, "{}".getBytes(StandardCharsets.UTF_8))
+      Files.write(file, defaultContent.getBytes(StandardCharsets.UTF_8))
     file
 end DataPaths

modules/server/src/main/scala/scaladex/server/service/AdminService.scala

@@ -53,7 +53,8 @@ class AdminService(
         if !env.isLocal then
           Seq(
             new JobScheduler(Job.missingMavenArtifacts, mavenCentralService.findMissing),
-            new JobScheduler(Job.nonStandardArtifacts, mavenCentralService.findNonStandard)
+            new JobScheduler(Job.nonStandardArtifacts, mavenCentralService.findNonStandard),
+            new JobScheduler(Job.newGroupIds, mavenCentralService.findNewGroups)
           )
         else Seq.empty
       )

modules/server/src/main/scala/scaladex/server/service/MavenCentralService.scala

@@ -9,6 +9,7 @@ import scaladex.core.model.Project
 import scaladex.core.service.MavenCentralClient
 import scaladex.core.service.SchedulerDatabase
 import scaladex.core.util.ScalaExtensions.*
+import scaladex.data.cleanup.NewGroupId
 import scaladex.data.cleanup.NonStandardLib
 import scaladex.infra.DataPaths
 
@@ -35,6 +36,37 @@ class MavenCentralService(
     yield s"Inserted ${result.sum} missing poms"
   end findNonStandard
 
+  /** Discover and index artifacts from new group IDs submitted by the community.
+    *
+    * This addresses the gap where artifacts published via Sonatype Central Portal don't trigger the legacy webhook, and
+    * the scheduled backfill job only scans group IDs already in the database.
+    *
+    * Library authors can submit their group IDs to new-groups.json in scaladex-contrib to have their artifacts
+    * discovered.
+    */
+  def findNewGroups(): Future[String] =
+    val newGroupIds = NewGroupId.load(dataPaths)
+    if newGroupIds.isEmpty then Future.successful("No new group IDs to discover")
+    else
+      for
+        knownRefs <- database.getArtifactRefs()
+        knownGroupIds = knownRefs.map(_.groupId).toSet
+        // Filter to only process group IDs not already in the database
+        newGroups = newGroupIds.filterNot(ng => knownGroupIds.contains(Artifact.GroupId(ng.groupId)))
+        _ = if newGroups.nonEmpty then
+          logger.info(s"Discovering ${newGroups.size} new group IDs: ${newGroups.map(_.groupId).mkString(", ")}")
+        result <- newGroups.mapSync { ng =>
+          val groupId = Artifact.GroupId(ng.groupId)
+          findAndIndexMissingArtifacts(groupId, None, knownRefs.toSet)
+        }
+      yield
+        val totalInserted = result.sum
+        if totalInserted > 0 then s"Discovered ${newGroups.size} new group IDs, inserted $totalInserted poms"
+        else if newGroups.isEmpty then s"All ${newGroupIds.size} group IDs from new-groups.json are already known"
+        else s"Checked ${newGroups.size} new group IDs, no new artifacts found"
+    end if
+  end findNewGroups
+
   private def findAndIndexMissingArtifacts(
       groupId: GroupId,
       artifactId: ArtifactId,

modules/template/src/main/scala/scaladex/view/Job.scala

@@ -49,6 +49,11 @@ object Job:
     "Find missing non-standard artifacts from Maven Central",
     2.hours
   )
+  val newGroupIds: Job = Job(
+    "new-group-ids",
+    "Discover artifacts from new group IDs submitted via new-groups.json (for Central Portal users)",
+    6.hours
+  )
   val latestArtifacts: Job = Job(
     "latest-artifacts",
     "Update latest version of artifacts",