Merge pull request #83 from JuliaTesting/mm/pass-info

mmiller-max · web-flow · commit d120625993e2 · 2022-03-30T08:13:38.000+01:00
Use Pass info in v1.7 and later
diff --git a/Project.toml b/Project.toml
@@ -1,6 +1,6 @@
 name = "TestReports"
 uuid = "dcd651b4-b50a-5b6b-8f22-87e9f253a252"
-version = "0.6.3"
+version = "0.7.0"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
diff --git a/README.md b/README.md
@@ -49,7 +49,7 @@ to report an test failure in [DataDepsGenerators.jl](https://github.com/oxinabox
 
 ![Screenshot of GoCD web-interface showing failing tests](docs/src/assets/FailingTests.PNG)
 
-The corresponding `testlog.xml` file (after pretty printing) is below.
+The corresponding `testlog.xml` file (produced with an earlier version of `TestReports`, and therefore missing some of the new features, and after pretty printing) is below.
 
 ```xml
 <?xml version="1.0" encoding="UTF-8"?>
@@ -77,8 +77,4 @@ The corresponding `testlog.xml` file (after pretty printing) is below.
     <testcase name="pass (info lost)" id="_testcase_id_"/>
   </testsuite>
 </testsuites>
-```
-### Using directly
-I actually don't recommend using this directly in your tests.
-It is more flexible to just keep using the default testset type,
-and then use something like the Runner script to generate a wrapper of your tests with this testset around it.
+```
diff --git a/docs/src/index.md b/docs/src/index.md
@@ -51,7 +51,7 @@ to report an test failure in [DataDepsGenerators.jl](https://github.com/oxinabox
 
 ![Screenshot of GoCD web-interface showing failing tests](assets/FailingTests.PNG)
 
-The corresponding `testlog.xml` file (after pretty printing) is below.
+The corresponding `testlog.xml` file (produced with an earlier version of `TestReports`, and therefore missing some of the new features, and after pretty printing) is below.
 
 ```xml
 <?xml version="1.0" encoding="UTF-8"?>
diff --git a/docs/src/manual.md b/docs/src/manual.md
@@ -146,17 +146,6 @@ This script creates a file `testlog.xml` in the current directory.
 
 ## Known Limitations
 
-### `Pass` Result Information
-
-When a `Test` passes, the information about the test is not retained in the `Result`.
-This means that it cannot be reported, and the testcase for each passing test will be:
-
-```xml
-<testcase name="pass (info lost)" id="_testcase_id_"/>
-```
-
-This will hopefully be improved in the future - [see Julia#25483](https://github.com/JuliaLang/julia/issues/25483).
-
 ### Single Nesting
 
 Julia `TestSet`s can be nested to an arbitary degree, but this is not allowed
@@ -185,16 +174,16 @@ Will generate the following XML (when pretty printed):
 
 ```xml
 <?xml version="1.0" encoding="UTF-8"?>
-<testsuites name="" id="_id_" tests="4" failures="0" errors="0">
-    <testsuite name="TopLevel/Middle1" id="_id_" tests="2" failures="0" errors="0">
-        <testcase name="pass (info lost)" id="_testcase_id_"/>
-        <testcase name="pass (info lost)" id="_testcase_id_"/>
+<testsuites tests="4" failures="0" errors="0">
+    <testsuite name="TopLevel/Middle1" tests="2" failures="0" errors="0" time="0.156" timestamp="2022-03-30T07:55:12.173" hostname="hostname" id="0">
+        <testcase name="1 == 1" id="1" classname="TopLevel/Middle1" time="0.028"/>
+        <testcase name="2 == 2" id="2" classname="TopLevel/Middle1" time="0.000"/>
     </testsuite>
-    <testsuite name="TopLevel/Middle2/Inner1" id="_id_" tests="1" failures="0" errors="0">
-        <testcase name="pass (info lost)" id="_testcase_id_"/>
+    <testsuite name="TopLevel/Middle2/Inner1" tests="1" failures="0" errors="0" time="0.000" timestamp="2022-03-30T07:55:12.329" hostname="hostname" id="1">
+        <testcase name="1 == 1" id="1" classname="TopLevel/Middle2/Inner1" time="0.000"/>
     </testsuite>
-    <testsuite name="TopLevel" id="_id_" tests="1" failures="0" errors="0">
-        <testcase name="pass (info lost)" id="_testcase_id_"/>
+    <testsuite name="TopLevel" tests="1" failures="0" errors="0" time="0.156" timestamp="2022-03-30T07:55:12.173" hostname="hostname" id="2">
+        <testcase name="1 == 1" id="1" classname="TopLevel" time="0.000"/>
     </testsuite>
 </testsuites>
 ```
diff --git a/src/testsets.jl b/src/testsets.jl
@@ -27,6 +27,16 @@ For a `Result`, return Dates.Millisecond(0).
 time_taken(result::ReportingResult) = result.time_taken
 time_taken(result::Result) = Dates.Millisecond(0)
 
+"""
+    ispass(result)
+
+Return `true` if `result` is a `Pass` or a `ReportingResult{Pass}`, otherwise
+return `false`.
+"""
+ispass(::Pass) = true
+ispass(result::ReportingResult{Pass}) = ispass(result.result)
+ispass(result) = false
+
 """
     ReportingTestSet
 
diff --git a/src/to_xml.jl b/src/to_xml.jl
@@ -199,7 +199,8 @@ function to_xml(ts::AbstractTestSet)
         set_attribute!(x_testcase, "classname", ts.description)
         set_attribute!(x_testcase, "time", time_taken(result)::Millisecond)
         # Set attributes which require variables in this scope
-        (result isa Pass || result isa ReportingResult{Pass}) && set_attribute!(x_testcase, "name", x_testcase["name"] * " (Test $total_ntests)")
+        ntests > 0 && set_attribute!(x_testcase, "id", total_ntests)  # Ignore both testsuites and errors outside of tests
+        ispass(result) && VERSION < v"1.7.0" && set_attribute!(x_testcase, "name", x_testcase["name"] * " (Test $total_ntests)")
         x_testcase
     end
 
@@ -218,9 +219,16 @@ end
 Create a testcase node from the result and return it along with
 information on number of tests.
 """
-function to_xml(res::Pass)
-    x_testcase = testcase_xml("pass (info lost)", "_testcase_id_", [])
-    x_testcase, 1, 0, 0  # Increment number of tests by 1
+@static if VERSION >= v"1.7.0"
+    function to_xml(res::Pass)
+        x_testcase = testcase_xml(res.orig_expr, "_testcase_id_", [])
+        x_testcase, 1, 0, 0  # Increment number of tests by 1
+    end
+else
+    function to_xml(res::Pass)
+        x_testcase = testcase_xml("pass (info lost)", "_testcase_id_", [])
+        x_testcase, 1, 0, 0  # Increment number of tests by 1
+    end
 end
 
 function to_xml(v::Fail)
@@ -231,7 +239,7 @@ end
 
 function to_xml(v::Broken)
     x_testcase = testcase_xml(v, [skip_xml()]) # it is not actually skipped
-    x_testcase, 0, 0, 0
+    x_testcase, 1, 0, 0
 end
 
 function to_xml(v::Error)
diff --git a/test/recordproperty.jl b/test/recordproperty.jl
@@ -64,7 +64,8 @@
             TestReports.test(pkg)
         end
         logfile = joinpath(@__DIR__, "testlog.xml")
-        @test_reference "references/test_with_properties.txt" open(f->read(f, String), logfile) |> clean_output
+        test_file = VERSION >= v"1.7.0" ? "references/test_with_properties.txt" : "references/test_with_properties_pre_1_7.txt"
+        @test_reference test_file open(f->read(f, String), logfile) |> clean_output
     end
 
     # Test for warning when ID set twice
diff --git a/test/references/complexexample.txt b/test/references/complexexample.txt
@@ -1,19 +1,19 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<testsuites tests="18" failures="6" errors="4"><testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math/Multiplication" time="0.0"/><testcase name="1 * 2 == 5" id="_testcase_id_" classname="Math/Multiplication" time="0.0"><failure message="2 == 5" type="test">Test Failed
+<testsuites tests="19" failures="6" errors="4"><testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="1 * 3 == 3" id="1" classname="Math/Multiplication" time="0.0"/><testcase name="1 * 2 == 5" id="2" classname="Math/Multiplication" time="0.0"><failure message="2 == 5" type="test">Test Failed
   Expression: 1 * 2 == 5
-   Evaluated: 2 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="_testcase_id_" classname="Math/Multiplication" time="0.0"/></testsuite><testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math/addition/negative addition" time="0.0"/><testcase name="1 + -2 == 1" id="_testcase_id_" classname="Math/addition/negative addition" time="0.0"><failure message="-1 == 1" type="test">Test Failed
+   Evaluated: 2 == 5</failure></testcase><testcase name="1 * 4 == 4" id="3" classname="Math/Multiplication" time="0.0"/></testsuite><testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="1 + -1 == 0" id="1" classname="Math/addition/negative addition" time="0.0"/><testcase name="1 + -2 == 1" id="2" classname="Math/addition/negative addition" time="0.0"><failure message="-1 == 1" type="test">Test Failed
   Expression: 1 + -2 == 1
-   Evaluated: -1 == 1</failure></testcase><testcase name="pass (info lost) (Test 3)" id="_testcase_id_" classname="Math/addition/negative addition" time="0.0"/></testsuite><testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math/addition" time="0.0"/><testcase name="1 + 2 == 5" id="_testcase_id_" classname="Math/addition" time="0.0"><failure message="3 == 5" type="test">Test Failed
+   Evaluated: -1 == 1</failure></testcase><testcase name="10 + -5 == 5" id="3" classname="Math/addition/negative addition" time="0.0"/></testsuite><testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="1 + 1 == 2" id="1" classname="Math/addition" time="0.0"/><testcase name="1 + 2 == 5" id="2" classname="Math/addition" time="0.0"><failure message="3 == 5" type="test">Test Failed
   Expression: 1 + 2 == 5
-   Evaluated: 3 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="_testcase_id_" classname="Math/addition" time="0.0"/></testsuite><testsuite name="Math/other" tests="3" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="sqrt(-1)" id="_testcase_id_" classname="Math/other" time="0.0"><skip/></testcase><error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean" classname="Math/other" time="0.0"></error><error message="Nooo" type="ErrorException" classname="Math/other" time="0.0">Nooo
-</error><error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass" classname="Math/other" time="0.0"></error></testsuite><testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="4"><error message="Got exception outside of a @test" type="ErrorException" classname="Math/Error outside of tests" time="0.0">Outside of tests
-</error></testsuite><testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="5"><testcase name="throw(ArgumentError(&quot;1&quot;))" id="_testcase_id_" classname="Math/Different failures" time="0.0"><failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
+   Evaluated: 3 == 5</failure></testcase><testcase name="1 + 4 == 5" id="3" classname="Math/addition" time="0.0"/></testsuite><testsuite name="Math/other" tests="4" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="sqrt(-1)" id="1" classname="Math/other" time="0.0"><skip/></testcase><error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean" classname="Math/other" time="0.0" id="2"></error><error message="Nooo" type="ErrorException" classname="Math/other" time="0.0" id="3">Nooo
+</error><error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass" classname="Math/other" time="0.0" id="4"></error></testsuite><testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="4"><error message="Got exception outside of a @test" type="ErrorException" classname="Math/Error outside of tests" time="0.0">Outside of tests
+</error></testsuite><testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="5"><testcase name="throw(ArgumentError(&quot;1&quot;))" id="1" classname="Math/Different failures" time="0.0"><failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
   Expression: throw(ArgumentError("1"))
     Expected: DimensionMismatch
-      Thrown: ArgumentError</failure></testcase><testcase name="true" id="_testcase_id_" classname="Math/Different failures" time="0.0"><failure message="No exception thrown" type="test_throws_nothing">Test Failed
+      Thrown: ArgumentError</failure></testcase><testcase name="true" id="2" classname="Math/Different failures" time="0.0"><failure message="No exception thrown" type="test_throws_nothing">Test Failed
   Expression: true
     Expected: DimensionMismatch
-  No exception thrown</failure></testcase></testsuite><testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math/using function from a module" time="0.0"/></testsuite><testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="Math" time="0.0"/><testcase name="sqrt(20) == 5" id="_testcase_id_" classname="Math" time="0.0"><failure message="4.47213595499958 == 5" type="test">Test Failed
+  No exception thrown</failure></testcase></testsuite><testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6"><testcase name="nthreads() &gt; 0" id="1" classname="Math/using function from a module" time="0.0"/></testsuite><testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7"><testcase name="4 % 2 == 0" id="1" classname="Math" time="0.0"/><testcase name="sqrt(20) == 5" id="2" classname="Math" time="0.0"><failure message="4.47213595499958 == 5" type="test">Test Failed
   Expression: sqrt(20) == 5
-   Evaluated: 4.47213595499958 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="_testcase_id_" classname="Math" time="0.0"/></testsuite></testsuites>
+   Evaluated: 4.47213595499958 == 5</failure></testcase><testcase name="16 == 16" id="3" classname="Math" time="0.0"/></testsuite></testsuites>
 
diff --git a/test/references/complexexample_pre_1_7.txt b/test/references/complexexample_pre_1_7.txt
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="19" failures="6" errors="4"><testsuite name="Math/Multiplication" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/Multiplication" time="0.0"/><testcase name="1 * 2 == 5" id="2" classname="Math/Multiplication" time="0.0"><failure message="2 == 5" type="test">Test Failed
+  Expression: 1 * 2 == 5
+   Evaluated: 2 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/Multiplication" time="0.0"/></testsuite><testsuite name="Math/addition/negative addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="1"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/addition/negative addition" time="0.0"/><testcase name="1 + -2 == 1" id="2" classname="Math/addition/negative addition" time="0.0"><failure message="-1 == 1" type="test">Test Failed
+  Expression: 1 + -2 == 1
+   Evaluated: -1 == 1</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/addition/negative addition" time="0.0"/></testsuite><testsuite name="Math/addition" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="2"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/addition" time="0.0"/><testcase name="1 + 2 == 5" id="2" classname="Math/addition" time="0.0"><failure message="3 == 5" type="test">Test Failed
+  Expression: 1 + 2 == 5
+   Evaluated: 3 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math/addition" time="0.0"/></testsuite><testsuite name="Math/other" tests="4" failures="0" errors="3" time="0.0" timestamp="0" hostname="localhost" id="3"><testcase name="sqrt(-1)" id="1" classname="Math/other" time="0.0"><skip/></testcase><error message="Expression evaluated to non-Boolean" type="Expression evaluated to non-Boolean" classname="Math/other" time="0.0" id="2"></error><error message="Nooo" type="ErrorException" classname="Math/other" time="0.0" id="3">Nooo
+</error><error message="Got correct result, please change to @test if no longer broken." type="Unexpected Pass" classname="Math/other" time="0.0" id="4"></error></testsuite><testsuite name="Math/Error outside of tests" tests="0" failures="0" errors="1" time="0.0" timestamp="0" hostname="localhost" id="4"><error message="Got exception outside of a @test" type="ErrorException" classname="Math/Error outside of tests" time="0.0">Outside of tests
+</error></testsuite><testsuite name="Math/Different failures" tests="2" failures="2" errors="0" time="0.0" timestamp="0" hostname="localhost" id="5"><testcase name="throw(ArgumentError(&quot;1&quot;))" id="1" classname="Math/Different failures" time="0.0"><failure message="Wrong exception type thrown" type="test_throws_wrong">Test Failed
+  Expression: throw(ArgumentError("1"))
+    Expected: DimensionMismatch
+      Thrown: ArgumentError</failure></testcase><testcase name="true" id="2" classname="Math/Different failures" time="0.0"><failure message="No exception thrown" type="test_throws_nothing">Test Failed
+  Expression: true
+    Expected: DimensionMismatch
+  No exception thrown</failure></testcase></testsuite><testsuite name="Math/using function from a module" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="6"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math/using function from a module" time="0.0"/></testsuite><testsuite name="Math" tests="3" failures="1" errors="0" time="0.0" timestamp="0" hostname="localhost" id="7"><testcase name="pass (info lost) (Test 1)" id="1" classname="Math" time="0.0"/><testcase name="sqrt(20) == 5" id="2" classname="Math" time="0.0"><failure message="4.47213595499958 == 5" type="test">Test Failed
+  Expression: sqrt(20) == 5
+   Evaluated: 4.47213595499958 == 5</failure></testcase><testcase name="pass (info lost) (Test 3)" id="3" classname="Math" time="0.0"/></testsuite></testsuites>
+
diff --git a/test/references/singlenest.txt b/test/references/singlenest.txt
@@ -1,2 +1,2 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<testsuites tests="1" failures="0" errors="0"><testsuite name="a" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="a" time="0.0"/></testsuite></testsuites>
+<testsuites tests="1" failures="0" errors="0"><testsuite name="a" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="1 == 1" id="1" classname="a" time="0.0"/></testsuite></testsuites>
diff --git a/test/references/singlenest_pre_1_7.txt b/test/references/singlenest_pre_1_7.txt
@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="1" failures="0" errors="0"><testsuite name="a" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="1" classname="a" time="0.0"/></testsuite></testsuites>
diff --git a/test/references/test_with_properties.txt b/test/references/test_with_properties.txt
diff --git a/test/references/test_with_properties_pre_1_7.txt b/test/references/test_with_properties_pre_1_7.txt
diff --git a/test/reportgeneration.jl b/test/reportgeneration.jl
diff --git a/test/testsets.jl b/test/testsets.jl

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`<?xml version="1.0" encoding="UTF-8"?>`
`2`		`-<testsuites tests="1" failures="0" errors="0"><testsuite name="a" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="_testcase_id_" classname="a" time="0.0"/></testsuite></testsuites>`
	`2`	`+<testsuites tests="1" failures="0" errors="0"><testsuite name="a" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="1 == 1" id="1" classname="a" time="0.0"/></testsuite></testsuites>`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+<?xml version="1.0" encoding="UTF-8"?>`
	`2`	`+<testsuites tests="1" failures="0" errors="0"><testsuite name="a" tests="1" failures="0" errors="0" time="0.0" timestamp="0" hostname="localhost" id="0"><testcase name="pass (info lost) (Test 1)" id="1" classname="a" time="0.0"/></testsuite></testsuites>`