SQL cleanup, add NM/WNM titles, fix IO purity, and expand test coverage

- V0019 migration: drop redundant cache_key_idx, replace EXCEPTION-based
  AVERAGE with CASE WHEN, convert implicit cross-joins to explicit JOINs
  in federation views, drop 27 redundant indexes on federations_summary,
  document intentional no-FK on player_history.federation_id
- Add NM/WNM to Title enum in both Scala domain and Smithy spec
- Wrap OffsetDateTime.now() in IO in Crawler.fetchAndSave
- Add 40+ new tests: parser edge cases, syncer status logic, player hash
  stability, DB sorting/filtering/batch ops, history rating queries,
  hash-based diffing, and CLI history ingestor integration

Author: lenguyenthanh

modules/api/src/main/smithy/_global.smithy

@@ -109,13 +109,15 @@ integer PageSize
 
 enum Title {
   GM = "GM"
-  WGM = "WGM"
   IM = "IM"
-  WIM = "WIM"
   FM = "FM"
-  WFM = "WFM"
   CM = "CM"
+  NM = "NM"
+  WGM = "WGM"
+  WIM = "WIM"
+  WFM = "WFM"
   WCM = "WCM"
+  WNM = "WNM"
 }
 
 enum OtherTitle {

modules/cli/src/test/scala/HistoryIngestorSuite.scala

@@ -0,0 +1,159 @@
+package fide
+package cli
+package test
+
+import cats.effect.IO
+import cats.effect.kernel.Resource
+import cats.syntax.all.*
+import com.comcast.ip4s.*
+import fide.db.test.Containers
+import fide.db.{ Db, HistoryDb, PostgresConfig }
+import fide.domain.Models.*
+import fide.types.*
+import io.github.iltotore.iron.*
+import org.typelevel.log4cats.Logger
+import org.typelevel.log4cats.noop.NoOpLogger
+import weaver.*
+
+import java.nio.file.{ Files as JFiles, Path }
+
+object HistoryIngestorSuite extends SimpleIOSuite:
+
+  given Logger[IO] = NoOpLogger[IO]
+
+  private val defaultPage = Pagination(PageNumber(1), PageSize(100))
+
+  // Dummy postgres config — never used since HistoryIngestor receives db/historyDb directly
+  private val dummyPg = PostgresConfig(ip"0.0.0.0", port"5432", "", "", "", 1, "fide", false, false)
+
+  private def mkConfig(
+      dir: Path,
+      startMonth: Option[YearMonth] = none,
+      endMonth: Option[YearMonth] = none
+  ): IngestConfig =
+    IngestConfig(dir, startMonth, endMonth, dummyPg, 100)
+
+  private def resourceWithFeds: Resource[IO, (HistoryDb, Db)] =
+    Containers.createResource.map(x => (HistoryDb(x.postgres, 100), Db(x.postgres)))
+
+  private def writeCsv(dir: java.nio.file.Path, filename: String, content: String): IO[Unit] =
+    IO:
+      JFiles.writeString(dir.resolve(filename), content)
+      ()
+
+  private val csvHeader =
+    "id,name,title,womenTitle,otherTitles,standard,standardK,rapid,rapidK,blitz,blitzK,gender,birthYear,active,federationId"
+
+  private def playerRow(
+      id: Int,
+      name: String,
+      fedId: String = "USA",
+      standard: Int = 2700,
+      active: Boolean = true
+  ) =
+    s"$id,$name,GM,,,${standard},40,2600,40,2500,40,M,1990,$active,$fedId"
+
+  // fide-ced: CSV discovery tests
+  test("discovers only yyyy-MM.csv files, ignores others"):
+    resourceWithFeds.use: (historyDb, db) =>
+      IO(JFiles.createTempDirectory("fide-test")).flatMap: tmpDir =>
+        for
+          _ <- writeCsv(tmpDir, "2024-01.csv", s"$csvHeader\n${playerRow(1, "Alice")}")
+          _ <- writeCsv(tmpDir, "notes.csv", s"$csvHeader\n${playerRow(2, "Bob")}")
+          _ <- writeCsv(tmpDir, "backup.csv", s"$csvHeader\n${playerRow(3, "Charlie")}")
+          config   = mkConfig(tmpDir)
+          ingestor = HistoryIngestor(historyDb, db, config)
+          _      <- ingestor.ingest
+          months <- historyDb.availableMonths
+        yield expect(months.size == 1) and
+          expect(months.head == YearMonth(2024, 1))
+
+  test("startMonth filter only ingests files >= startMonth"):
+    resourceWithFeds.use: (historyDb, db) =>
+      IO(JFiles.createTempDirectory("fide-test")).flatMap: tmpDir =>
+        for
+          _ <- writeCsv(tmpDir, "2024-01.csv", s"$csvHeader\n${playerRow(1, "Alice")}")
+          _ <- writeCsv(tmpDir, "2024-02.csv", s"$csvHeader\n${playerRow(1, "Alice")}")
+          _ <- writeCsv(tmpDir, "2024-03.csv", s"$csvHeader\n${playerRow(1, "Alice")}")
+          config   = mkConfig(tmpDir, startMonth = YearMonth(2024, 2).some)
+          ingestor = HistoryIngestor(historyDb, db, config)
+          _      <- ingestor.ingest
+          months <- historyDb.availableMonths
+        yield expect(months.size == 2) and
+          expect(!months.contains(YearMonth(2024, 1)))
+
+  test("endMonth filter only ingests files <= endMonth"):
+    resourceWithFeds.use: (historyDb, db) =>
+      IO(JFiles.createTempDirectory("fide-test")).flatMap: tmpDir =>
+        for
+          _ <- writeCsv(tmpDir, "2024-01.csv", s"$csvHeader\n${playerRow(1, "Alice")}")
+          _ <- writeCsv(tmpDir, "2024-02.csv", s"$csvHeader\n${playerRow(1, "Alice")}")
+          _ <- writeCsv(tmpDir, "2024-03.csv", s"$csvHeader\n${playerRow(1, "Alice")}")
+          config   = mkConfig(tmpDir, endMonth = YearMonth(2024, 2).some)
+          ingestor = HistoryIngestor(historyDb, db, config)
+          _      <- ingestor.ingest
+          months <- historyDb.availableMonths
+        yield expect(months.size == 2) and
+          expect(!months.contains(YearMonth(2024, 3)))
+
+  // fide-u4h: end-to-end integration test
+  test("full pipeline: CSV to queryable history"):
+    resourceWithFeds.use: (historyDb, db) =>
+      IO(JFiles.createTempDirectory("fide-test")).flatMap: tmpDir =>
+        val jan2024 = YearMonth(2024, 1)
+        val feb2024 = YearMonth(2024, 2)
+        for
+          _ <- writeCsv(
+            tmpDir,
+            "2024-01.csv",
+            s"$csvHeader\n${playerRow(1, "Alice")}\n${playerRow(2, "Bob")}"
+          )
+          _ <- writeCsv(tmpDir, "2024-02.csv", s"$csvHeader\n${playerRow(1, "Alice", standard = 2750)}")
+          config   = mkConfig(tmpDir)
+          ingestor = HistoryIngestor(historyDb, db, config)
+          _ <- ingestor.ingest
+          // Verify available months
+          months <- historyDb.availableMonths
+          // Verify player_info populated
+          janPlayers <- historyDb.allPlayers(
+            jan2024,
+            Sorting(SortBy.Name, Order.Asc),
+            defaultPage,
+            PlayerFilter.default
+          )
+          // Verify per-month snapshots
+          febPlayers <- historyDb.allPlayers(
+            feb2024,
+            Sorting(SortBy.Name, Order.Asc),
+            defaultPage,
+            PlayerFilter.default
+          )
+          // Verify playerById
+          alice <- historyDb.playerById(PlayerId(1), feb2024)
+          // Verify federation populated
+          feds <- db.allFederations
+        yield expect(months.size == 2) and
+          expect(janPlayers.size == 2) and
+          expect(janPlayers.head.name == "Alice") and
+          expect(febPlayers.size == 1) and
+          expect(alice.isDefined) and
+          expect(alice.get.standard.contains(Rating(2750))) and
+          expect(feds.exists(_.id == FederationId("USA")))
+
+  test("re-running ingest is idempotent"):
+    resourceWithFeds.use: (historyDb, db) =>
+      IO(JFiles.createTempDirectory("fide-test")).flatMap: tmpDir =>
+        val jan2024 = YearMonth(2024, 1)
+        for
+          _ <- writeCsv(tmpDir, "2024-01.csv", s"$csvHeader\n${playerRow(1, "Alice")}")
+          config   = mkConfig(tmpDir)
+          ingestor = HistoryIngestor(historyDb, db, config)
+          _       <- ingestor.ingest
+          _       <- ingestor.ingest // second run
+          players <- historyDb.allPlayers(
+            jan2024,
+            Sorting(SortBy.Name, Order.Asc),
+            defaultPage,
+            PlayerFilter.default
+          )
+        yield expect(players.size == 1) // no duplicates

modules/crawler/src/main/scala/Crawler.scala

@@ -54,8 +54,8 @@ object Crawler:
           case _ => info"Skipping crawling as the data is up to date".as(CrawlStatus.Skipped)
 
       def fetchAndSave(timestamp: Option[String]): IO[Unit] =
-        val now = OffsetDateTime.now()
         for
+          now          <- IO.realTimeZonedDateTime.map(_.toOffsetDateTime)
           _            <- info"Start crawling"
           startAt      <- IO.monotonic
           cachedHashes <- playerHashCache.get

modules/crawler/src/test/scala/ParserTest.scala

@@ -4,7 +4,7 @@ package test
 
 import cats.effect.IO
 import cats.syntax.all.*
-import fide.domain.OtherTitle
+import fide.domain.{ Gender, OtherTitle, Title }
 import org.typelevel.log4cats.Logger
 import weaver.*
 
@@ -50,4 +50,50 @@ object ParserTest extends SimpleIOSuite:
       expect.same(Downloader.sanitizeName("Amar,, Kahtan"), "Amar Kahtan".some) &&
       expect.same(Downloader.sanitizeName("-,-"), none)
 
+  test("player with no federation"):
+    parse(
+      "1478800        Aagaard, Christian                                               M                                                                       1999      "
+    ).map: result =>
+      expect(result.get.federationId.isEmpty)
+
+  test("player with NON federation is excluded"):
+    parse(
+      "1478800        Aagaard, Christian                                           NON M                                                                       1999      "
+    ).map: result =>
+      expect(result.get.federationId.isEmpty)
+
+  test("player with title and women title"):
+    parse(
+      "10001492       Ojok, Patrick                                                UGA F   GM   WGM                     1638      20 1932      20 1926      20 1974          "
+    ).map: result =>
+      val p = result.get
+      expect(p.title.contains(Title.GM)) and
+        expect(p.womenTitle.contains(Title.WGM))
+
+  test("empty line is skipped"):
+    parse("").map(result => expect(result.isEmpty))
+
+  test("whitespace-only line is skipped"):
+    parse("                                                                       ").map(result =>
+      expect(result.isEmpty)
+    )
+
+  test("birth year below 1000 is excluded"):
+    parse(
+      "10001492       Ojok, Patrick                                                UGA M                                1638  0   20 1932  0   20 1926  0   20 0999      "
+    ).map: result =>
+      expect(result.get.birthYear.isEmpty)
+
+  test("birth year above current year is excluded"):
+    parse(
+      "10001492       Ojok, Patrick                                                UGA M                                1638  0   20 1932  0   20 1926  0   20 9999      "
+    ).map: result =>
+      expect(result.get.birthYear.isEmpty)
+
+  test("gender is parsed"):
+    parse(
+      "10001492       Ojok, Patrick                                                UGA F                                1638  0   20 1932  0   20 1926  0   20 1974      "
+    ).map: result =>
+      expect(result.get.gender.contains(Gender.Female))
+
   private def parse(s: String) = Downloader.parseLine(s).map(_.map(_._1))

modules/crawler/src/test/scala/SyncerTest.scala

@@ -0,0 +1,27 @@
+package fide
+package crawler
+package test
+
+import cats.effect.IO
+import fide.crawler.Syncer.Status
+import weaver.*
+
+object SyncerTest extends SimpleIOSuite:
+
+  test("Status: same local and remote is UpToDate"):
+    IO(expect(Status(Some("a"), Some("a")) == Status.UpToDate))
+
+  test("Status: different local and remote is OutDated"):
+    IO:
+      val status = Status(Some("a"), Some("b"))
+      expect(status == Status.OutDated(Some("b")))
+
+  test("Status: no local, some remote is OutDated"):
+    IO:
+      val status = Status(None, Some("b"))
+      expect(status == Status.OutDated(Some("b")))
+
+  test("Status: no local, no remote is OutDated"):
+    IO:
+      val status = Status(None, None)
+      expect(status == Status.OutDated(None))