Skip to content

feat： show expected_failure and unexpected_success in HTMLreport #74

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

feat： show expected_failure and unexpected_success in HTMLreport #74

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b34039d
feat： show `expected_failure` and `unexpected_success` in HTMLreport
dongfangtianyu Sep 17, 2020
c7e4293
fix: expectedFailures and unexpectedSuccesses result input error list
dongfangtianyu Sep 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 25 additions & 7 deletions HtmlTestRunner/result.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
from __future__ import print_function

import copy
import os
import sys
import time
import copy
import traceback
from unittest import TestResult, TextTestResult
from unittest.result import failfast

from jinja2 import Template


DEFAULT_TEMPLATE = os.path.join(os.path.dirname(__file__), "template", "report_template.html")


Expand Down Expand Up @@ -71,7 +70,7 @@ def strip_module_names(testcase_names):
class _TestInfo(object):
"""" Keeps information about the execution of a test method. """

(SUCCESS, FAILURE, ERROR, SKIP) = range(4)
(SUCCESS, FAILURE, ERROR, SKIP, XFAIL, XPASS,) = range(6)

def __init__(self, test_result, test_method, outcome=SUCCESS,
err=None, subTest=None):
Expand All @@ -86,7 +85,7 @@ def __init__(self, test_result, test_method, outcome=SUCCESS,

self.test_description = self.test_result.getDescription(test_method)
self.test_exception_info = (
'' if outcome in (self.SUCCESS, self.SKIP)
'' if outcome in (self.SUCCESS, self.SKIP, self.XPASS)
else self.test_result._exc_info_to_string(
self.err, test_method))

Expand Down Expand Up @@ -235,6 +234,19 @@ def addSkip(self, test, reason):
testinfo = self.infoclass(self, test, self.infoclass.SKIP, reason)
self._prepare_callback(testinfo, self.skipped, "SKIP", "S")

def addExpectedFailure(self, test, err):
"""Called when an expected failure/error occurred."""
self._save_output_data()
testinfo = self.infoclass(self, test, self.infoclass.XFAIL, err)
self._prepare_callback(testinfo, self.expectedFailures, "expected failure", "X")

@failfast
def addUnexpectedSuccess(self, test):
"""Called when a test was expected to fail, but succeed."""
self._save_output_data()
self._prepare_callback(self.infoclass(self, test, self.infoclass.XPASS), self.unexpectedSuccesses, "unexpected success",
"U")

def printErrorList(self, flavour, errors):
"""
Writes information about the FAIL or ERROR to the stream.
Expand All @@ -261,7 +273,7 @@ def _get_info_by_testcase(self):
tests_by_testcase[testcase_name] = []
tests_by_testcase[testcase_name].append(subtest_info)

for tests in (self.successes, self.failures, self.errors, self.skipped):
for tests in (self.successes, self.failures, self.errors, self.skipped, self.expectedFailures, self.unexpectedSuccesses):
for test_info in tests:
# subtests will be contained by _SubTestInfos objects but there is also the
# case where all subtests pass and the method is added as a success as well
Expand Down Expand Up @@ -293,7 +305,7 @@ def _format_duration(elapsed_time):
def get_results_summary(self, tests):
"""Create a summary of the outcomes of all given tests."""

failures = errors = skips = successes = 0
failures = errors = skips = successes = expected_failure = unexpected_success = 0
for test in tests:
outcome = test.outcome
if outcome == test.ERROR:
Expand All @@ -304,6 +316,10 @@ def get_results_summary(self, tests):
skips += 1
elif outcome == test.SUCCESS:
successes += 1
elif outcome == test.XFAIL:
expected_failure += 1
elif outcome == test.XPASS:
unexpected_success += 1

elapsed_time = 0
for testinfo in tests:
Expand All @@ -319,6 +335,8 @@ def get_results_summary(self, tests):
"failure": failures,
"skip": skips,
"success": successes,
"expected_failure": expected_failure,
"unexpected_success": unexpected_success,
"duration": self._format_duration(elapsed_time)
}

Expand All @@ -343,7 +361,7 @@ def _get_report_summaries(self, all_results, testRunner):

def generate_reports(self, testRunner):
""" Generate report(s) for all given test cases that have been run. """
status_tags = ('success', 'danger', 'warning', 'info')
status_tags = ('success', 'danger', 'warning', 'info', 'info', 'warning')
all_results = self._get_info_by_testcase()
summaries = self._get_report_summaries(all_results, testRunner)

Expand Down
29 changes: 24 additions & 5 deletions HtmlTestRunner/template/report_template.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,14 @@
<h2 class="text-capitalize">{{ title }}</h2>
<p class='attribute'><strong>Start Time: </strong>{{ header_info.start_time.strftime("%Y-%m-%d %H:%M:%S") }}</p>
<p class='attribute'><strong>Duration: </strong>{{ header_info.status.duration }}</p>
<p class='attribute'><strong>Summary: </strong>Total: {{ header_info.status.total }}, Pass: {{ header_info.status.success }}{% if header_info.status.failure %}, Fail: {{ header_info.status.failure }}{% endif %}{% if header_info.status.error %}, Error: {{ header_info.status.error }}{% endif %}{% if header_info.status.skip %}, Skip: {{ header_info.status.skip }}{% endif %}</p>
<p class='attribute'><strong>Summary: </strong>Total: {{ header_info.status.total }}, Pass: {{ header_info.status.success }}
{% if header_info.status.failure %}, Fail: {{ header_info.status.failure }}{% endif %}
{% if header_info.status.error %}, Error: {{ header_info.status.error }}{% endif %}
{% if header_info.status.skip %}, Skip: {{ header_info.status.skip }}{% endif %}
{% if header_info.status.expected_failure %}, XFail: {{ header_info.status.expected_failure }}{% endif %}
{% if header_info.status.unexpected_success %}, XPass: {{ header_info.status.unexpected_success }}{% endif %}
</p>
<p class='attribute'><strong>Tester: </strong> {{ tester }}</p>
</div>
</div>
{%- for test_case_name, tests_results in all_results.items() %}
Expand All @@ -34,15 +41,21 @@ <h2 class="text-capitalize">{{ title }}</h2>
<tr class='{{ status_tags[test_case.outcome] }}'>
<td class="col-xs-10">{{ test_case.test_id.split(".")[-1] }}</td>
<td class="col-xs-1">
<span class="label label-{{ status_tags[test_case.outcome] }}" style="display:block;width:40px;">
<span class="label label-{{ status_tags[test_case.outcome] }}" style="display:block;width:45px;">
{%- if test_case.outcome == test_case.SUCCESS -%}
Pass
{%- elif test_case.outcome == test_case.SKIP -%}
Skip
{%- elif test_case.outcome == test_case.FAILURE -%}
Fail
{%- else -%}
{%- elif test_case.outcome == test_case.ERROR -%}
Error
{%- elif test_case.outcome == test_case.XFAIL -%}
XFail
{%- elif test_case.outcome == test_case.XPASS -%}
XPass
{%- else -%}
Other
{%- endif -%}
</span>
</td>
Expand Down Expand Up @@ -129,7 +142,13 @@ <h2 class="text-capitalize">{{ title }}</h2>
{%- endfor %}
<tr>
<td colspan="3">
Total: {{ summaries[test_case_name].total }}, Pass: {{ summaries[test_case_name].success }}{% if summaries[test_case_name].failure %}, Fail: {{ summaries[test_case_name].failure }}{% endif %}{% if summaries[test_case_name].error %}, Error: {{ summaries[test_case_name].error }}{% endif %}{% if summaries[test_case_name].skip %}, Skip: {{ summaries[test_case_name].skip }}{% endif %} -- Duration: {{ summaries[test_case_name].duration }}
Total: {{ summaries[test_case_name].total }}, Pass: {{ summaries[test_case_name].success }}
{% if summaries[test_case_name].failure %}, Fail: {{ summaries[test_case_name].failure }}{% endif %}
{% if summaries[test_case_name].error %}, Error: {{ summaries[test_case_name].error }}{% endif %}
{% if summaries[test_case_name].skip %}, Skip: {{ summaries[test_case_name].skip }}{% endif %}
{% if summaries[test_case_name].expected_failure %}, XFail: {{ summaries[test_case_name].expected_failure }}{% endif %}
{% if summaries[test_case_name].unexpected_success %}, XPass: {{ header_info.status.unexpected_success }}{% endif %}
-- Duration: {{ summaries[test_case_name].duration }}
</td>
</tr>
</tbody>
Expand Down Expand Up @@ -159,4 +178,4 @@ <h2 class="text-capitalize">{{ title }}</h2>
});
</script>
</body>
</html
</html>