From 7679ae17b8da7c512d03bf9a3a76db0677403d3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Koz=C5=82owski?= Date: Sun, 20 Nov 2022 01:18:50 +0100 Subject: [PATCH] Ignore ignored tests in pure suites --- modules/core/src/weaver/suites.scala | 126 ++++++++++-------- .../test/src-jvm/junit/JUnitRunnerTests.scala | 20 +++ .../cats/test/src-jvm/junit/Meta.scala | 16 +++ 3 files changed, 110 insertions(+), 52 deletions(-) diff --git a/modules/core/src/weaver/suites.scala b/modules/core/src/weaver/suites.scala index 34abc48f..b3a118f8 100644 --- a/modules/core/src/weaver/suites.scala +++ b/modules/core/src/weaver/suites.scala @@ -67,8 +67,56 @@ abstract class RunnableSuite[F[_]] extends EffectSuite[F] { def plan : List[TestName] private[weaver] def runUnsafe(args: List[String])(report: TestOutcome => Unit) : Unit = effectCompat.unsafeRunSync(run(args)(outcome => effectCompat.effect.delay(report(outcome)))) + + def isCI: Boolean = System.getenv("CI") == "true" + + private[weaver] def analyze[Res, F1[_]](testSeq: Seq[(TestName, Res => F1[TestOutcome])], args: List[String]): TagAnalysisResult[Res, F1] = { + val testsNotIgnored: Seq[(TestName, Res => F1[TestOutcome])] = + testSeq.filterNot(_._1.tags(TestName.Tags.ignore)) + + val testsTaggedOnly: Seq[(TestName, Res => F1[TestOutcome])] = + testSeq.filter(_._1.tags(TestName.Tags.only)) + + val onlyTestsNotIgnored = + testsTaggedOnly.filter(taggedOnly => testsNotIgnored.contains(taggedOnly)) + + val filteredTests = if (onlyTestsNotIgnored.isEmpty) { + val argsFilter = Filters.filterTests(this.name)(args) + testsNotIgnored.collect { + case (name, test) if argsFilter(name) => test + } + } else onlyTestsNotIgnored.map(_._2) + + if (testsTaggedOnly.nonEmpty && isCI) { + val failureOutcomes = testsTaggedOnly.map(_._1).map(onlyNotOnCiFailure) + TagAnalysisResult.Outcomes(failureOutcomes) + } else TagAnalysisResult.FilteredTests(filteredTests) + } + + + private[this] def onlyNotOnCiFailure(test: TestName): TestOutcome = { + val result = Result.Failure( + msg = "'Only' tag is not allowed when `isCI=true`", + source = None, + location = List(test.location) + ) + TestOutcome( + name = test.name, + duration = FiniteDuration(0, "ns"), + result = result, + log = Chain.empty + ) + } + +} + +private[weaver] sealed trait TagAnalysisResult[Res, F[_]] +object TagAnalysisResult { + case class Outcomes[Res, F[_]](outcomes: Seq[TestOutcome]) extends TagAnalysisResult[Res, F] + case class FilteredTests[Res, F[_]](tests: Seq[Res => F[TestOutcome]]) extends TagAnalysisResult[Res, F] } + abstract class MutableFSuite[F[_]] extends RunnableSuite[F] { type Res @@ -95,53 +143,28 @@ abstract class MutableFSuite[F[_]] extends RunnableSuite[F] { def usingRes(run : Res => F[Expectations]) : Unit = apply(run) } - def isCI: Boolean = "true" == System.getenv("CI") - - override def spec(args: List[String]) : Stream[F, TestOutcome] = + override def spec(args: List[String]): Stream[F, TestOutcome] = synchronized { if (!isInitialized) isInitialized = true - val testsNotIgnored: Seq[(TestName, Res => F[TestOutcome])] = testSeq.filterNot(_._1.tags(TestName.Tags.ignore)) - val testsTaggedOnly: Seq[(TestName, Res => F[TestOutcome])] = testSeq.filter(_._1.tags(TestName.Tags.only)) - val onlyTestsNotIgnored = testsTaggedOnly.filter(taggedOnly => testsNotIgnored.contains(taggedOnly)) - val filteredTests = if (onlyTestsNotIgnored.isEmpty) { - val argsFilter = Filters.filterTests(this.name)(args) - testsNotIgnored.collect { - case (name, test) if argsFilter(name) => test - } - } else onlyTestsNotIgnored.map(_._2) - val parallism = math.max(1, maxParallelism) - - if (testsTaggedOnly.nonEmpty && isCI) { - val failureOutcomes = testsTaggedOnly - .map(_._1) - .map(onlyNotOnCiFailure) - Stream.emits(failureOutcomes).lift[F](effectCompat.effect) + val parallelism = math.max(1, maxParallelism) + + analyze(testSeq, args) match { + case TagAnalysisResult.Outcomes(outcomes) => fs2.Stream.emits(outcomes) + case TagAnalysisResult.FilteredTests(filteredTests) + if filteredTests.isEmpty => + Stream.empty // no need to allocate resources + case TagAnalysisResult.FilteredTests(filteredTests) => for { + resource <- Stream.resource(sharedResource) + tests = filteredTests.map(_.apply(resource)) + testStream = Stream.emits(tests).covary[F] + result <- if (parallelism > 1) + testStream.parEvalMap(parallelism)(identity)(effectCompat.effect) + else testStream.evalMap(identity) + } yield result } - else if (filteredTests.isEmpty) Stream.empty // no need to allocate resources - else for { - resource <- Stream.resource(sharedResource) - tests = filteredTests.map(_.apply(resource)) - testStream = Stream.emits(tests).lift[F](effectCompat.effect) - result <- if (parallism > 1 ) testStream.parEvalMap(parallism)(identity)(effectCompat.effect) - else testStream.evalMap(identity) - } yield result } - private[this] def onlyNotOnCiFailure(test: TestName): TestOutcome = { - val result = Result.Failure( - msg = "'Only' tag is not allowed when `isCI=true`", - source = None, - location = List(test.location) - ) - TestOutcome( - name = test.name, - duration = FiniteDuration(0, "ns"), - result = result, - log = Chain.empty - ) - } - - private[this] var testSeq = Seq.empty[(TestName, Res => F[TestOutcome])] + private[this] var testSeq: Seq[(TestName, Res => F[TestOutcome])] = Seq.empty def plan: List[TestName] = testSeq.map(_._1).toList @@ -161,19 +184,18 @@ trait FunSuiteAux { abstract class FunSuiteF[F[_]] extends RunnableSuite[F] with FunSuiteAux { self => override def test(name: TestName)(run: => Expectations): Unit = synchronized { if(isInitialized) throw initError - testSeq = testSeq :+ (name -> (() => Test.pure(name.name)(() => run))) + testSeq = testSeq :+ (name -> ((_: Unit) => Test.pure(name.name)(() => run))) } override def name : String = self.getClass.getName.replace("$", "") - private def pureSpec(args: List[String]) = synchronized { + + private def pureSpec(args: List[String]): fs2.Stream[fs2.Pure, TestOutcome] = synchronized { if(!isInitialized) isInitialized = true - val argsFilter = Filters.filterTests(this.name)(args) - val filteredTests = if (testSeq.exists(_._1.tags(TestName.Tags.only))){ - testSeq.filter(_._1.tags(TestName.Tags.only)).map { case (_, test) => test} - } else testSeq.collect { - case (name, test) if argsFilter(name) => test - } - fs2.Stream.emits(filteredTests.map(execute => execute())) + analyze[Unit, cats.Id](testSeq, args) match { + case TagAnalysisResult.Outcomes(outcomes) => fs2.Stream.emits(outcomes) + case TagAnalysisResult.FilteredTests(filteredTests) => + fs2.Stream.emits(filteredTests.map(execute => execute(()))) + } } override def spec(args: List[String]) = pureSpec(args).covary[F] @@ -182,7 +204,7 @@ abstract class FunSuiteF[F[_]] extends RunnableSuite[F] with FunSuiteAux { self pureSpec(args).compile.toVector.foreach(report) - private[this] var testSeq = Seq.empty[(TestName, () => TestOutcome)] + private[this] var testSeq = Seq.empty[(TestName, Unit => TestOutcome)] def plan: List[TestName] = testSeq.map(_._1).toList private[this] var isInitialized = false diff --git a/modules/framework/cats/test/src-jvm/junit/JUnitRunnerTests.scala b/modules/framework/cats/test/src-jvm/junit/JUnitRunnerTests.scala index eda9d450..59b93113 100644 --- a/modules/framework/cats/test/src-jvm/junit/JUnitRunnerTests.scala +++ b/modules/framework/cats/test/src-jvm/junit/JUnitRunnerTests.scala @@ -114,6 +114,21 @@ object JUnitRunnerTests extends IOSuite { } } + test("Tests tagged with ignore are ignored (FunSuite)") { blocker => + runPure(blocker, Meta.IgnorePure).map { notifications => + val expected = List( + TestSuiteStarted("weaver.junit.Meta$IgnorePure$"), + TestIgnored("is ignored(weaver.junit.Meta$IgnorePure$)"), + TestStarted("not ignored 1(weaver.junit.Meta$IgnorePure$)"), + TestFinished("not ignored 1(weaver.junit.Meta$IgnorePure$)"), + TestStarted("not ignored 2(weaver.junit.Meta$IgnorePure$)"), + TestFinished("not ignored 2(weaver.junit.Meta$IgnorePure$)"), + TestSuiteFinished("weaver.junit.Meta$IgnorePure$") + ) + expect.same(notifications, expected) + } + } + test( "Even if all tests are ignored, will fail if a test is tagged with only") { blocker => @@ -172,6 +187,11 @@ object JUnitRunnerTests extends IOSuite { suite: SimpleIOSuite): IO[List[Notification]] = run(blocker, suite.getClass()) + def runPure( + blocker: BlockerCompat[IO], + suite: FunSuite): IO[List[Notification]] = + run(blocker, suite.getClass()) + sealed trait Notification case class TestSuiteStarted(name: String) extends Notification case class TestAssumptionFailure(failure: Failure) extends Notification diff --git a/modules/framework/cats/test/src-jvm/junit/Meta.scala b/modules/framework/cats/test/src-jvm/junit/Meta.scala index 1e888c4d..792b0ef5 100644 --- a/modules/framework/cats/test/src-jvm/junit/Meta.scala +++ b/modules/framework/cats/test/src-jvm/junit/Meta.scala @@ -113,6 +113,22 @@ object Meta { } + object IgnorePure extends FunSuite { + + test("not ignored 1") { + success + } + + test("not ignored 2") { + success + } + + test("is ignored".ignore) { + failure("foo") + } + + } + class Sharing(global: GlobalRead) extends IOSuite { type Res = Unit