Skip to content

Commit 68e81e9

Browse files
jkujku
authored
Publish a conformance report (#322)
* Bump python version in action & workflows Signed-off-by: Jussi Kukkonen <jkukkonen@google.com> * Add summary report publishing: * The action now publishes json results as GitHub artifact * A workflow is added that looks for those results in projects known to use tuf-conformance * A report is generated and published on GH pages Signed-off-by: Jussi Kukkonen <jkukkonen@google.com> * publish-report workflow: make client branch a variable Signed-off-by: Jussi Kukkonen <jkukkonen@google.com> --------- Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>
1 parent daf5ad1 commit 68e81e9

File tree

6 files changed

+233
-5
lines changed

6 files changed

+233
-5
lines changed

‎.github/scripts/generate_client_report.py‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# Generate an html summary from client conformance results
2+
3+
import argparse
4+
import json
5+
from dataclasses import dataclass
6+
from datetime import UTC, datetime
7+
from pathlib import Path
8+
9+
10+
@dataclass
11+
class Result:
12+
name: str
13+
results_found: bool = False
14+
total: int = -1
15+
passed: int = -1
16+
failed: int = -1
17+
xfailed: int = -1
18+
skipped: int = -1
19+
20+
def __init__(self, report_path: Path) -> None:
21+
with report_path.open() as f:
22+
data = json.load(f)
23+
24+
self.name = report_path.name.replace(".json", "")
25+
26+
if data == {}:
27+
return # no results found
28+
self.results_found = True
29+
30+
summary = data["summary"]
31+
self.total = summary["total"]
32+
self.passed = summary.get("passed", 0) + summary.get("subtests passed", 0)
33+
self.failed = summary.get("failed", 0) + summary.get("subtests failed", 0)
34+
self.xfailed = summary.get("xfailed", 0) + summary.get("subtests xfailed", 0)
35+
self.skipped = summary.get("skipped", 0) + summary.get("subtests skipped", 0)
36+
37+
# TODO: parse data["tests"] and add feature checks like
38+
# self.supports_delegated_targets
39+
40+
41+
def _generate_html(results: list[Result]) -> str:
42+
html = f"""
43+
<html>
44+
<head>
45+
<title>TUF Client Conformance Results</title>
46+
<style>
47+
body {{ font-family: sans-serif; margin: 2em; }}
48+
table {{ border-collapse: collapse; }}
49+
th, td {{ border: 1px solid #ccc; padding: 8px 12px; }}
50+
th {{ background-color: #f4f4f4; }}
51+
.failed {{ background-color: #ffe0e0; }}
52+
.passed {{ background-color: #e0ffe0; }}
53+
.not-found {{ background-color: #eeeeee; }}
54+
</style>
55+
</head>
56+
<body>
57+
<h1>TUF Client Conformance Results</h1>
58+
<p>Last updated: {datetime.now(UTC).isoformat(timespec="minutes")}Z</p>
59+
<table>
60+
<thead>
61+
<tr>
62+
<th>Client</th>
63+
<th>Pass Rate</th>
64+
<th>Passed</th>
65+
<th>Failed</th>
66+
<th>Skipped</th>
67+
<th>Xfailed</th>
68+
</tr>
69+
</thead>
70+
<tbody>
71+
"""
72+
for res in results:
73+
if not res.results_found:
74+
status_class = "not-found"
75+
elif res.failed == 0:
76+
status_class = "passed"
77+
else:
78+
status_class = "failed"
79+
passrate = round(100 * res.passed / res.total) if res.total > 0 else 0
80+
html += f"""
81+
<tr class="{status_class}">
82+
<td><strong>{res.name}</strong></td>
83+
<td>{f"{passrate}%" if res.results_found else ""}</td>
84+
<td>{res.passed if res.results_found else ""}</td>
85+
<td>{res.failed if res.results_found else ""}</td>
86+
<td>{res.skipped if res.results_found else ""}</td>
87+
<td>{res.xfailed if res.results_found else ""}</td>
88+
</tr>
89+
"""
90+
html += """
91+
</tbody>
92+
</table>
93+
</body>
94+
</html>
95+
"""
96+
return html
97+
98+
99+
if __name__ == "__main__":
100+
parser = argparse.ArgumentParser()
101+
parser.add_argument("--reports-dir", required=True)
102+
parser.add_argument("--output", required=True)
103+
args = parser.parse_args()
104+
105+
# Read all client results
106+
results: list[Result] = []
107+
for report_path in Path(args.reports_dir).glob("**/*.json"):
108+
results.append(Result(report_path))
109+
results.sort(key=lambda result: result.name)
110+
111+
# Write summary HTML
112+
output_file = Path(args.output)
113+
output_file.parent.mkdir(parents=True, exist_ok=True)
114+
with output_file.open("w") as f:
115+
f.write(_generate_html(results))

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ jobs:
2323
- name: Set up Python
2424
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
2525
with:
26-
python-version: "3.11"
26+
python-version: "3.14"
2727
cache: "pip"
2828

2929
- name: Install lint dependencies

‎.github/workflows/publish-report.yml‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
name: Publish client conformance report
2+
3+
on:
4+
schedule:
5+
- cron: '0 2 * * *'
6+
workflow_dispatch:
7+
8+
jobs:
9+
fetch-results:
10+
name: Fetch ${{ matrix.name }} results
11+
runs-on: ubuntu-latest
12+
strategy:
13+
fail-fast: false # Don't stop if one client results download fails
14+
matrix:
15+
include:
16+
- name: python-tuf
17+
repo: theupdateframework/python-tuf
18+
workflow: conformance.yml
19+
branch: develop
20+
- name: tuf-js
21+
repo: theupdateframework/tuf-js
22+
workflow: conformance.yml
23+
branch: main
24+
- name: sigstore-java
25+
repo: sigstore/sigstore-java
26+
workflow: tuf-conformance.yml
27+
branch: main
28+
- name: sigstore-ruby
29+
repo: sigstore/sigstore-ruby
30+
workflow: ci.yml
31+
branch: main
32+
33+
34+
steps:
35+
- name: Download artifact
36+
uses: dawidd6/action-download-artifact@ac66b43f0e6a346234dd65d4d0c8fbb31cb316e5 # v11
37+
with:
38+
# A token is required to read other repos' artifacts. This token is a fine grained token
39+
# with read-only "public repositories" access only. It is set to expire in 1 year
40+
# (maximum accepted by OpenSSF)
41+
github_token: ${{ secrets.READ_ONLY_TOKEN }}
42+
repo: ${{ matrix.repo }}
43+
workflow: ${{ matrix.workflow }}
44+
branch: ${{ matrix.branch }}
45+
name: tuf-conformance-results
46+
workflow_conclusion: success
47+
path: ./results/
48+
continue-on-error: true
49+
50+
- name: Rename artifact for aggregation
51+
run: |
52+
if [ -f "./results/tuf-conformance-report.json" ]; then
53+
mv ./results/tuf-conformance-report.json "./results/${{ matrix.name }}.json"
54+
else
55+
# create empty file so report generator can still include the client as "not found"
56+
mkdir ./results
57+
echo "{}" > "./results/${{ matrix.name }}.json"
58+
fi
59+
60+
- name: Upload ${{ matrix.name }} results
61+
uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
62+
with:
63+
name: ${{ matrix.name }}-result
64+
path: ./results/${{ matrix.name }}.json
65+
66+
build-report:
67+
name: Build client conformance report
68+
runs-on: ubuntu-latest
69+
needs: [fetch-results]
70+
if: always()
71+
72+
steps:
73+
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
74+
75+
- name: Download all individual results
76+
uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
77+
with:
78+
pattern: '*-result'
79+
path: ./results
80+
81+
- name: Generate report
82+
run: |
83+
python .github/scripts/generate_client_report.py \
84+
--reports-dir ./results \
85+
--output results/index.html
86+
87+
- name: Upload report for Pages
88+
uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0
89+
with:
90+
path: results/
91+
92+
deploy-pages:
93+
permissions:
94+
pages: write
95+
id-token: write
96+
environment:
97+
name: github-pages
98+
url: ${{ steps.deployment.outputs.page_url }}
99+
runs-on: ubuntu-latest
100+
needs: build-report
101+
steps:
102+
- name: Deploy to GitHub Pages
103+
id: deployment
104+
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ dev: faketime env/pyvenv.cfg
3333
.PHONY: test-all
3434
test-all: test-python-tuf test-go-tuf
3535

36-
lint_dirs = tuf_conformance clients/python-tuf
36+
lint_dirs = tuf_conformance clients/python-tuf .github/scripts
3737
lint: env/pyvenv.cfg
3838
./env/bin/ruff format --diff $(lint_dirs)
3939
./env/bin/ruff check $(lint_dirs)

‎action.yml‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ runs:
1717
- name: Set up Python
1818
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
1919
with:
20-
python-version: "3.11"
20+
python-version: "3.14"
2121

2222
- name: Install test
2323
run: |
@@ -42,7 +42,7 @@ runs:
4242
fi
4343
4444
# run test suite
45-
pytest -v "$TEST_LOCATION" \
45+
pytest -v --json-report --json-report-file=tuf-conformance-report.json "$TEST_LOCATION" \
4646
--entrypoint "$ENTRYPOINT" \
4747
--repository-dump-dir ./test-repositories \
4848
shell: bash
@@ -53,3 +53,11 @@ runs:
5353
with:
5454
name: ${{ steps.tuf-conformance.outputs.NAME }}
5555
path: test-repositories
56+
57+
- name: Upload conformance result
58+
uses: actions/upload-artifact@v4
59+
with:
60+
name: tuf-conformance-results
61+
overwrite: true
62+
path: ./tuf-conformance-report.json
63+
retention-days: 7

‎pyproject.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ name = "tuf-conformance"
77
dependencies = [
88
"securesystemslib[crypto]==1.3.1",
99
"tuf==6.0.0",
10-
"pytest==8.4.2"
10+
"pytest==8.4.2",
11+
"pytest-json-report==1.5.0"
1112
]
1213
dynamic = ["version"]
1314
requires-python = ">= 3.10"

0 commit comments

Comments
 (0)