Skip to content

Commit 5bbab3c

Browse files
jkujku
committed
Add summary report publishing:
* The action now publishes json results as GitHub artifact * A workflow is added that looks for those results in projects known to use tuf-conformance * A report is generated and published on GH pages Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>
1 parent b35c27f commit 5bbab3c

File tree

5 files changed

+227
-3
lines changed

5 files changed

+227
-3
lines changed

‎.github/scripts/generate_client_report.py‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# Generate an html summary from client conformance results
2+
3+
import argparse
4+
import json
5+
from dataclasses import dataclass
6+
from datetime import UTC, datetime
7+
from pathlib import Path
8+
9+
10+
@dataclass
11+
class Result:
12+
name: str
13+
results_found: bool = False
14+
total: int = -1
15+
passed: int = -1
16+
failed: int = -1
17+
xfailed: int = -1
18+
skipped: int = -1
19+
20+
def __init__(self, report_path: Path) -> None:
21+
with report_path.open() as f:
22+
data = json.load(f)
23+
24+
self.name = report_path.name.replace(".json", "")
25+
26+
if data == {}:
27+
return # no results found
28+
self.results_found = True
29+
30+
summary = data["summary"]
31+
self.total = summary["total"]
32+
self.passed = summary.get("passed", 0) + summary.get("subtests passed", 0)
33+
self.failed = summary.get("failed", 0) + summary.get("subtests failed", 0)
34+
self.xfailed = summary.get("xfailed", 0) + summary.get("subtests xfailed", 0)
35+
self.skipped = summary.get("skipped", 0) + summary.get("subtests skipped", 0)
36+
37+
# TODO: parse data["tests"] and add feature checks like
38+
# self.supports_delegated_targets
39+
40+
41+
def _generate_html(results: list[Result]) -> str:
42+
html = f"""
43+
<html>
44+
<head>
45+
<title>TUF Client Conformance Results</title>
46+
<style>
47+
body {{ font-family: sans-serif; margin: 2em; }}
48+
table {{ border-collapse: collapse; }}
49+
th, td {{ border: 1px solid #ccc; padding: 8px 12px; }}
50+
th {{ background-color: #f4f4f4; }}
51+
.failed {{ background-color: #ffe0e0; }}
52+
.passed {{ background-color: #e0ffe0; }}
53+
.not-found {{ background-color: #eeeeee; }}
54+
</style>
55+
</head>
56+
<body>
57+
<h1>TUF Client Conformance Results</h1>
58+
<p>Last updated: {datetime.now(UTC).isoformat(timespec="minutes")}Z</p>
59+
<table>
60+
<thead>
61+
<tr>
62+
<th>Client</th>
63+
<th>Pass Rate</th>
64+
<th>Passed</th>
65+
<th>Failed</th>
66+
<th>Skipped</th>
67+
<th>Xfailed</th>
68+
</tr>
69+
</thead>
70+
<tbody>
71+
"""
72+
for res in results:
73+
if not res.results_found:
74+
status_class = "not-found"
75+
elif res.failed == 0:
76+
status_class = "passed"
77+
else:
78+
status_class = "failed"
79+
passrate = round(100 * res.passed / res.total) if res.total > 0 else 0
80+
html += f"""
81+
<tr class="{status_class}">
82+
<td><strong>{res.name}</strong></td>
83+
<td>{f"{passrate}%" if res.results_found else ""}</td>
84+
<td>{res.passed if res.results_found else ""}</td>
85+
<td>{res.failed if res.results_found else ""}</td>
86+
<td>{res.skipped if res.results_found else ""}</td>
87+
<td>{res.xfailed if res.results_found else ""}</td>
88+
</tr>
89+
"""
90+
html += """
91+
</tbody>
92+
</table>
93+
</body>
94+
</html>
95+
"""
96+
return html
97+
98+
99+
if __name__ == "__main__":
100+
parser = argparse.ArgumentParser()
101+
parser.add_argument("--reports-dir", required=True)
102+
parser.add_argument("--output", required=True)
103+
args = parser.parse_args()
104+
105+
# Read all client results
106+
results: list[Result] = []
107+
for report_path in Path(args.reports_dir).glob("**/*.json"):
108+
results.append(Result(report_path))
109+
results.sort(key=lambda result: result.name)
110+
111+
# Write summary HTML
112+
output_file = Path(args.output)
113+
output_file.parent.mkdir(parents=True, exist_ok=True)
114+
with output_file.open("w") as f:
115+
f.write(_generate_html(results))

‎.github/workflows/publish-report.yml‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
name: Publish client conformance report
2+
3+
on:
4+
schedule:
5+
- cron: '0 2 * * *'
6+
workflow_dispatch:
7+
8+
jobs:
9+
fetch-results:
10+
name: Fetch ${{ matrix.name }} results
11+
runs-on: ubuntu-latest
12+
strategy:
13+
fail-fast: false # Don't stop if one client results download fails
14+
matrix:
15+
include:
16+
- name: python-tuf
17+
repo: theupdateframework/python-tuf
18+
workflow: conformance.yml
19+
- name: tuf-js
20+
repo: theupdateframework/tuf-js
21+
workflow: conformance.yml
22+
- name: sigstore-java
23+
repo: sigstore/sigstore-java
24+
workflow: tuf-conformance.yml
25+
- name: sigstore-ruby
26+
repo: sigstore/sigstore-ruby
27+
workflow: ci.yml
28+
29+
30+
steps:
31+
- name: Download artifact
32+
uses: dawidd6/action-download-artifact@ac66b43f0e6a346234dd65d4d0c8fbb31cb316e5 # v11
33+
with:
34+
# A token is required to read other repos' artifacts. This token is a fine grained token
35+
# with read-only "public repositories" access only. It is set to expire in 1 year
36+
# (maximum accepted by OpenSSF)
37+
github_token: ${{ secrets.READ_ONLY_TOKEN }}
38+
repo: ${{ matrix.repo }}
39+
workflow: ${{ matrix.workflow }}
40+
branch: main
41+
name: tuf-conformance-results
42+
workflow_conclusion: success
43+
path: ./results/
44+
continue-on-error: true
45+
46+
- name: Rename artifact for aggregation
47+
run: |
48+
if [ -f "./results/tuf-conformance-report.json" ]; then
49+
mv ./results/tuf-conformance-report.json "./results/${{ matrix.name }}.json"
50+
else
51+
# create empty file so report generator can still include the client as "not found"
52+
mkdir ./results
53+
echo "{}" > "./results/${{ matrix.name }}.json"
54+
fi
55+
56+
- name: Upload ${{ matrix.name }} results
57+
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
58+
with:
59+
name: ${{ matrix.name }}-result
60+
path: ./results/${{ matrix.name }}.json
61+
62+
build-report:
63+
name: Build client conformance report
64+
runs-on: ubuntu-latest
65+
needs: [fetch-results]
66+
if: always()
67+
68+
steps:
69+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
70+
71+
- name: Download all individual results
72+
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
73+
with:
74+
pattern: '*-result'
75+
path: ./results
76+
77+
- name: Generate report
78+
run: |
79+
python .github/scripts/generate_client_report.py \
80+
--reports-dir ./results \
81+
--output results/index.html
82+
83+
- name: Upload report for Pages
84+
uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0
85+
with:
86+
path: results/
87+
88+
deploy-pages:
89+
permissions:
90+
pages: write
91+
id-token: write
92+
environment:
93+
name: github-pages
94+
url: ${{ steps.deployment.outputs.page_url }}
95+
runs-on: ubuntu-latest
96+
needs: build-report
97+
steps:
98+
- name: Deploy to GitHub Pages
99+
id: deployment
100+
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ dev: faketime env/pyvenv.cfg
3333
.PHONY: test-all
3434
test-all: test-python-tuf test-go-tuf
3535

36-
lint_dirs = tuf_conformance clients/python-tuf
36+
lint_dirs = tuf_conformance clients/python-tuf .github/scripts
3737
lint: env/pyvenv.cfg
3838
./env/bin/ruff format --diff $(lint_dirs)
3939
./env/bin/ruff check $(lint_dirs)

‎action.yml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ runs:
4242
fi
4343
4444
# run test suite
45-
pytest -v "$TEST_LOCATION" \
45+
pytest -v --json-report --json-report-file=tuf-conformance-report.json "$TEST_LOCATION" \
4646
--entrypoint "$ENTRYPOINT" \
4747
--repository-dump-dir ./test-repositories \
4848
shell: bash
@@ -53,3 +53,11 @@ runs:
5353
with:
5454
name: ${{ steps.tuf-conformance.outputs.NAME }}
5555
path: test-repositories
56+
57+
- name: Upload conformance result
58+
uses: actions/upload-artifact@v4
59+
with:
60+
name: tuf-conformance-results
61+
overwrite: true
62+
path: ./tuf-conformance-report.json
63+
retention-days: 7

‎pyproject.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ name = "tuf-conformance"
77
dependencies = [
88
"securesystemslib[crypto]==1.3.1",
99
"tuf==6.0.0",
10-
"pytest==8.4.2"
10+
"pytest==8.4.2",
11+
"pytest-json-report==1.5.0"
1112
]
1213
dynamic = ["version"]
1314
requires-python = ">= 3.10"

0 commit comments

Comments
 (0)