chore: improve mill tests (#2049)

Author: hugo-vrijswijk

build.sbt

@@ -57,7 +57,7 @@ lazy val sbtPlugin = (projectMatrix in file("modules") / "sbt")
   .enablePlugins(SbtPlugin)
   .defaultAxes(VirtualAxis.scalaPartialVersion("2.12"), VirtualAxis.jvm)
   .settings(commonSettings, sbtPluginSettings, publishLocalDependsOn(core, testRunner))
-  .dependsOn(core)
+  .dependsOn(core, testkit % Test)
   .jvmPlatform(scalaVersions = Seq(versions.scala3, versions.scala212))
 
 // Mill plugins are compiled with the Scala version of the minimum supported Mill version

modules/mill/src/test/scala/stryker4s/log/MillLoggerTest.scala

@@ -0,0 +1,81 @@
+package mill.api.daemon {
+
+  import java.io.{ByteArrayInputStream, PrintStream}
+  import scala.collection.mutable.ListBuffer
+
+  /** A test double for Mill's [[Logger]]. Lives in `mill.api.daemon` because `prompt` (which backs `debugEnabled` and
+    * color detection) is `private[mill]` and can only be overridden from within the `mill` package.
+    */
+  class FakeMillLogger(debugOn: Boolean, coloredOn: Boolean) extends Logger {
+    val logged: ListBuffer[(String, String)] = ListBuffer.empty
+
+    override val streams: SystemStreams = new SystemStreams(
+      new PrintStream(_ => ()),
+      new PrintStream(_ => ()),
+      new ByteArrayInputStream(Array.empty)
+    )
+
+    override def info(s: String): Unit = logged += (("info", s))
+    override def debug(s: String): Unit = logged += (("debug", s))
+    override def warn(s: String): Unit = logged += (("warn", s))
+    override def error(s: String): Unit = logged += (("error", s))
+    override def ticker(s: String): Unit = logged += (("ticker", s))
+
+    override private[mill] def prompt: Logger.Prompt = new Logger.Prompt.NoOp {
+      override def debugEnabled: Boolean = debugOn
+      override def colored: Boolean = coloredOn
+    }
+  }
+}
+
+package stryker4s.log {
+
+  import mill.api.daemon.FakeMillLogger
+  import stryker4s.testkit.Stryker4sSuite
+
+  class MillLoggerTest extends Stryker4sSuite {
+
+    /** Exposes the `protected` `colorEnabled` for assertions. */
+    private class ProbeLogger(colored: Boolean, env: Map[String, String])
+        extends MillLogger(new FakeMillLogger(debugOn = false, coloredOn = colored), env) {
+      def color: Boolean = colorEnabled
+    }
+
+    private def colorOf(colored: Boolean, env: Map[String, String]): Boolean =
+      new ProbeLogger(colored, env).color
+
+    test("debug messages are only logged when mill has debug enabled") {
+      val enabled = new FakeMillLogger(debugOn = true, coloredOn = false)
+      new MillLogger(enabled, Map.empty).log(Level.Debug, "debug message")
+      assertEquals(enabled.logged.toList, List("debug" -> "debug message"))
+
+      val disabled = new FakeMillLogger(debugOn = false, coloredOn = false)
+      new MillLogger(disabled, Map.empty).log(Level.Debug, "debug message")
+      assertEquals(disabled.logged.toList, List.empty)
+    }
+
+    test("info, warn and error are always forwarded to mill") {
+      val millLogger = new FakeMillLogger(debugOn = false, coloredOn = false)
+      val logger = new MillLogger(millLogger, Map.empty)
+      logger.log(Level.Info, "an info")
+      logger.log(Level.Warn, "a warning")
+      logger.log(Level.Error, "an error")
+      assertEquals(
+        millLogger.logged.toList,
+        List("info" -> "an info", "warn" -> "a warning", "error" -> "an error")
+      )
+    }
+
+    test("color is enabled when mill enables color and NO_COLOR is unset") {
+      assert(colorOf(colored = true, env = Map.empty))
+    }
+
+    test("color is disabled when NO_COLOR is set, even if mill enables color") {
+      assert(!colorOf(colored = true, env = Map("NO_COLOR" -> "")))
+    }
+
+    test("color is disabled when mill disables color, even if NO_COLOR is unset") {
+      assert(!colorOf(colored = false, env = Map.empty))
+    }
+  }
+}

modules/mill/src/test/scala/stryker4s/mill/MillConfigSourceTest.scala

@@ -1,8 +1,12 @@
 package stryker4s.mill
 
 import cats.effect.IO
+import cats.syntax.all.*
+import ciris.ConfigValue
 import fs2.io.file.Path
+import stryker4s.config.*
 import stryker4s.testkit.Stryker4sIOSuite
+import sttp.client4.UriContext
 
 import scala.concurrent.duration.*
 import scala.meta.dialects
@@ -40,15 +44,15 @@ class MillConfigSourceTest extends Stryker4sIOSuite {
     val config = new MillConfigSource[IO](
       baseDirValue = Path("/tmp/project"),
       mutateValue = Some(Seq("src/main/scala/**.scala")),
-      filesValue = None,
+      filesValue = Some(Seq("src/main/scala/**")),
       testFilterValue = Some(Seq("*MyTest")),
       reportersValue = Some(Seq("html", "console")),
       excludedMutationsValue = Some(Seq("BooleanLiteral")),
       thresholdsHighValue = Some(85),
       thresholdsLowValue = Some(60),
       thresholdsBreakValue = Some(0),
-      dashboardBaseUrlValue = None,
-      dashboardReportTypeValue = None,
+      dashboardBaseUrlValue = Some("https://dashboard.stryker-mutator.io"),
+      dashboardReportTypeValue = Some("full"),
       dashboardProjectValue = Some("github.com/foo/bar"),
       dashboardVersionValue = Some("main"),
       dashboardModuleValue = Some("my-module"),
@@ -66,24 +70,69 @@ class MillConfigSourceTest extends Stryker4sIOSuite {
 
     for {
       _ <- config.mutate.load.assertEquals(Seq("src/main/scala/**.scala"))
+      _ <- config.files.load.assertEquals(Seq("src/main/scala/**"))
       _ <- config.testFilter.load.assertEquals(Seq("*MyTest"))
+      _ <- config.reporters.load.assertEquals(Seq[ReporterType](Html, Console))
+      _ <- config.excludedMutations.load.assertEquals(Seq(ExcludedMutation("BooleanLiteral")))
       _ <- config.thresholdsHigh.load.assertEquals(85)
+      _ <- config.thresholdsLow.load.assertEquals(60)
+      _ <- config.thresholdsBreak.load.assertEquals(0)
+      _ <- config.dashboardBaseUrl.load.assertEquals(uri"https://dashboard.stryker-mutator.io")
+      _ <- config.dashboardReportType.load.assertEquals(Full: DashboardReportType)
       _ <- config.dashboardProject.load.assertEquals(Some("github.com/foo/bar"))
+      _ <- config.dashboardVersion.load.assertEquals(Some("main"))
       _ <- config.dashboardModule.load.assertEquals(Some("my-module"))
       _ <- config.timeout.load.assertEquals(5.seconds)
+      _ <- config.timeoutFactor.load.assertEquals(1.5)
+      _ <- config.maxTestRunnerReuse.load.assertEquals(Some(10))
+      _ <- config.scalaDialect.load.assertEquals(dialects.Scala3)
       _ <- config.concurrency.load.assertEquals(4)
+      _ <- config.debugLogTestRunnerStdout.load.assertEquals(true)
+      _ <- config.debugDebugTestRunner.load.assertEquals(false)
+      _ <- config.staticTmpDir.load.assertEquals(false)
       _ <- config.cleanTmpDir.load.assertEquals(true)
+      _ <- config.openReport.load.assertEquals(false)
     } yield ()
   }
 
   test("should make the baseDir absolute") {
     emptyConfigSource.baseDir.load.assertEquals(Path("/tmp/project").absolute)
   }
 
-  test("missing values are reported as missing") {
-    emptyConfigSource.thresholdsHigh.attempt
-      .map(_.leftValue.messages.loneElement)
-      .assertEquals("Missing mill config strykerThresholdsHigh")
+  test("missing values are reported as missing with their mill config key") {
+    val source = emptyConfigSource
+    // Every value-backed config def should report the exact mill build key it reads from when absent
+    val cases: List[(ConfigValue[IO, ?], String)] = List(
+      source.mutate -> "strykerMutate",
+      source.files -> "strykerFiles",
+      source.testFilter -> "strykerTestFilter",
+      source.reporters -> "strykerReporters",
+      source.excludedMutations -> "strykerExcludedMutations",
+      source.thresholdsHigh -> "strykerThresholdsHigh",
+      source.thresholdsLow -> "strykerThresholdsLow",
+      source.thresholdsBreak -> "strykerThresholdsBreak",
+      source.dashboardBaseUrl -> "strykerDashboardBaseUrl",
+      source.dashboardReportType -> "strykerDashboardReportType",
+      source.dashboardProject -> "strykerDashboardProject",
+      source.dashboardVersion -> "strykerDashboardVersion",
+      source.dashboardModule -> "strykerDashboardModule",
+      source.timeout -> "strykerTimeout",
+      source.timeoutFactor -> "strykerTimeoutFactor",
+      source.maxTestRunnerReuse -> "strykerMaxTestRunnerReuse",
+      source.scalaDialect -> "strykerScalaDialect",
+      source.concurrency -> "strykerConcurrency",
+      source.debugLogTestRunnerStdout -> "strykerDebugLogTestRunnerStdout",
+      source.debugDebugTestRunner -> "strykerDebugDebugTestRunner",
+      source.staticTmpDir -> "strykerStaticTmpDir",
+      source.cleanTmpDir -> "strykerCleanTmpDir",
+      source.openReport -> "strykerOpenReport"
+    )
+
+    cases.traverse_ { case (configValue, key) =>
+      configValue.attempt
+        .map(_.leftValue.messages.loneElement)
+        .assertEquals(s"Missing mill config $key")
+    }
   }
 
   test("fails on values not supported by the mill plugin") {

modules/mill/src/test/scala/stryker4s/mill/StrykerModuleTest.scala

@@ -7,6 +7,7 @@ import mill.util.TokenReaders.*
 import stryker4s.testkit.Stryker4sSuite
 
 import java.nio.file.Paths
+import scala.meta.dialects
 
 class Stryker4sModuleTest extends Stryker4sSuite {
 
@@ -50,6 +51,29 @@ class Stryker4sModuleTest extends Stryker4sSuite {
     lazy val millDiscover = Discover[this.type]
   }
 
+  object scala2Project extends TestRootModule {
+    // Scala 2.13 module with `-Xsource:3` enabled, plus an unrelated option to distinguish exists/forall
+    object source3 extends ScalaModule, Stryker4sModule {
+      def scalaVersion = "2.13.18"
+      override def scalacOptions = Seq("-Xsource:3", "-deprecation")
+
+      object test extends ScalaTests {
+        def testFramework = "munit.Framework"
+      }
+    }
+    // Scala 2.13 module without `-Xsource:3`
+    object plain extends ScalaModule, Stryker4sModule {
+      def scalaVersion = "2.13.18"
+      override def scalacOptions = Seq("-deprecation")
+
+      object test extends ScalaTests {
+        def testFramework = "munit.Framework"
+      }
+    }
+
+    lazy val millDiscover = Discover[this.type]
+  }
+
   test("strykerTestModule resolves the child test module") {
     UnitTester(unitTestProject, resourceProject("unit-test-project")).scoped { _ =>
       assertEquals(unitTestProject.foo.strykerTestModule, unitTestProject.foo.test)
@@ -81,4 +105,30 @@ class Stryker4sModuleTest extends Stryker4sSuite {
       )
     }
   }
+
+  test("strykerMutate and strykerFiles produce globs relative to the module dir") {
+    UnitTester(unitTestProject, resourceProject("unit-test-project")).scoped { eval =>
+      assertEquals(eval(unitTestProject.foo.strykerMutate).value.value, Some(Seq("src/**.scala")))
+      assertEquals(eval(unitTestProject.foo.strykerFiles).value.value, Some(Seq("src/**")))
+    }
+  }
+
+  test("strykerScalaDialect derives the scalameta dialect from the scala version") {
+    UnitTester(unitTestProject, resourceProject("unit-test-project")).scoped { eval =>
+      // scalaVersion is 3.3.8, so the dialect should be Scala 3.3
+      assertEquals(eval(unitTestProject.foo.strykerScalaDialect).value.value, Some(dialects.Scala33))
+    }
+  }
+
+  test("strykerScalaDialect picks the source3 dialect for Scala 2 with -Xsource:3") {
+    UnitTester(scala2Project, resourceProject("unit-test-project")).scoped { eval =>
+      assertEquals(eval(scala2Project.source3.strykerScalaDialect).value.value, Some(dialects.Scala213Source3))
+    }
+  }
+
+  test("strykerScalaDialect picks the plain dialect for Scala 2 without -Xsource:3") {
+    UnitTester(scala2Project, resourceProject("unit-test-project")).scoped { eval =>
+      assertEquals(eval(scala2Project.plain.strykerScalaDialect).value.value, Some(dialects.Scala213))
+    }
+  }
 }

modules/mill/src/test/scala/stryker4s/mill/runner/MillTestDiscoveryTest.scala

@@ -19,16 +19,23 @@ class MillTestDiscoveryTest extends Stryker4sSuite with LogMatchers {
 
     val group = groups.loneElement
     assertEquals(group.frameworkClass, "munit.Framework")
-    assert(
-      group.taskDefs.exists(_.fullyQualifiedName == "stryker4s.mill.runner.MillTestDiscoveryTest"),
-      s"This test suite should be discovered, but got: ${group.taskDefs.map(_.fullyQualifiedName)}"
-    )
+    val taskDef = group.taskDefs
+      .find(_.fullyQualifiedName == "stryker4s.mill.runner.MillTestDiscoveryTest")
+      .getOrElse(
+        fail(s"This test suite should be discovered, but got: ${group.taskDefs.map(_.fullyQualifiedName)}")
+      )
+    // Discovered tests are never explicitly specified on the command line
+    assertEquals(taskDef.explicitlySpecified, false)
+    assertNotLoggedWarn("No tests found for test framework")
   }
 
   test("warns when no tests are found for the framework") {
     val groups = MillTestDiscovery.discover("munit.Framework", Seq.empty, fullClasspath)
 
     assertEquals(groups.loneElement.taskDefs, Seq.empty)
-    assertLoggedWarn("No tests found for test framework munit.Framework")
+    assertLoggedWarn(
+      "No tests found for test framework munit.Framework. " +
+        "Will likely result in no tests being run and a NoCoverage result for all mutants."
+    )
   }
 }