Skip to content

Commit 3c84e8f

Browse files
dimartirodimartiro
authored
Merge master into forking (#464)
1 parent d43a539 commit 3c84e8f

File tree

93 files changed

+6064
-2365
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

93 files changed

+6064
-2365
lines changed

‎.github/scripts/check-test-regression.py‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
#!/usr/bin/env python3
2+
"""Parse forge test output and compare with baseline."""
3+
4+
import sys
5+
import re
6+
from pathlib import Path
7+
from collections import defaultdict
8+
9+
10+
def parse_forge_output(log_file):
11+
"""Parse forge test output and extract test results."""
12+
results = {}
13+
14+
if not log_file.exists():
15+
print(f"Error: Forge output log not found: {log_file}")
16+
sys.exit(1)
17+
18+
print(f"Parsing test results from {log_file}...")
19+
20+
with open(log_file, 'r', encoding='utf-8', errors='ignore') as f:
21+
for line in f:
22+
line = line.strip()
23+
24+
# Match [PASS] lines
25+
if line.startswith('[PASS]'):
26+
match = re.search(r'^\[PASS\]\s+([^\s(]+)', line)
27+
if match:
28+
results[match.group(1)] = 'PASS'
29+
30+
# Match [FAIL lines
31+
elif line.startswith('[FAIL'):
32+
match = re.search(r'^\[FAIL[^\]]*\]\s+([^\s(]+)', line)
33+
if match:
34+
results[match.group(1)] = 'FAIL'
35+
36+
return results
37+
38+
39+
def save_results(results, output_file):
40+
"""Save results to file."""
41+
with open(output_file, 'w') as f:
42+
for test, status in sorted(results.items()):
43+
f.write(f"{test}:{status}\n")
44+
45+
46+
def load_results(file_path):
47+
"""Load results from file."""
48+
results = {}
49+
if file_path.exists():
50+
with open(file_path, 'r') as f:
51+
for line in f:
52+
line = line.strip()
53+
if ':' in line:
54+
test, status = line.split(':', 1)
55+
results[test] = status
56+
return results
57+
58+
59+
def print_summary(project_name, current_results, baseline_results=None):
60+
"""Print test results summary."""
61+
passing = [t for t, s in current_results.items() if s == 'PASS']
62+
failing = [t for t, s in current_results.items() if s == 'FAIL']
63+
64+
print("━" * 60)
65+
print(f"Test Results for {project_name}")
66+
print("━" * 60)
67+
print(f"Total tests: {len(current_results)}")
68+
print(f" ✓ Passing: {len(passing)}")
69+
print(f" ✗ Failing: {len(failing)}")
70+
print()
71+
72+
if failing:
73+
print("Failed tests:")
74+
for test in sorted(failing):
75+
print(f" - {test}")
76+
print()
77+
78+
print(f"Results saved to: test-results-{project_name}.txt")
79+
print()
80+
81+
if baseline_results is None:
82+
print("No baseline file specified (first run or master branch)")
83+
84+
print("━" * 60)
85+
86+
87+
def compare_with_baseline(project_name, current_results, baseline_results):
88+
"""Compare current results with baseline."""
89+
baseline_passing = {t for t, s in baseline_results.items() if s == 'PASS'}
90+
baseline_failing = {t for t, s in baseline_results.items() if s == 'FAIL'}
91+
92+
current_passing = {t for t, s in current_results.items() if s == 'PASS'}
93+
current_failing = {t for t, s in current_results.items() if s == 'FAIL'}
94+
95+
# Find regressions (passed before, failing now)
96+
regressions = baseline_passing & current_failing
97+
98+
# Find improvements (failed before, passing now)
99+
improvements = baseline_failing & current_passing
100+
101+
# Find new tests
102+
all_baseline = set(baseline_results.keys())
103+
all_current = set(current_results.keys())
104+
new_tests = all_current - all_baseline
105+
106+
print("Comparing test results for", project_name)
107+
print("━" * 60)
108+
print("Test Statistics:")
109+
print(f" Baseline: {len(baseline_passing)} passing, {len(baseline_failing)} failing")
110+
print(f" Current: {len(current_passing)} passing, {len(current_failing)} failing")
111+
print()
112+
113+
if improvements:
114+
print(f"Improvements: {len(improvements)} test(s) now passing")
115+
for test in sorted(improvements):
116+
print(f" - {test}")
117+
print()
118+
119+
if new_tests:
120+
print(f"New tests detected: {len(new_tests)}")
121+
for test in sorted(new_tests):
122+
status = current_results[test]
123+
print(f" - {test} ({status})")
124+
print()
125+
126+
if regressions:
127+
print(f"ERROR: REGRESSIONS DETECTED - {len(regressions)} test(s) now failing")
128+
print()
129+
print("The following tests passed in the baseline but are now failing:")
130+
for test in sorted(regressions):
131+
print(f" - {test}")
132+
print()
133+
print("━" * 60)
134+
print("Regression check FAILED")
135+
return False
136+
137+
print("No regressions detected")
138+
print("━" * 60)
139+
return True
140+
141+
142+
def main():
143+
if len(sys.argv) < 3:
144+
print("Usage: check-test-regression.py PROJECT_NAME FORGE_OUTPUT_LOG [BASELINE_FILE]")
145+
sys.exit(1)
146+
147+
project_name = sys.argv[1]
148+
log_file = Path(sys.argv[2])
149+
baseline_file = Path(sys.argv[3]) if len(sys.argv) > 3 else None
150+
151+
# Parse current results
152+
current_results = parse_forge_output(log_file)
153+
154+
# Check if any results were found
155+
if not current_results:
156+
print(f"WARNING: No test results found in {log_file}")
157+
print("This usually means tests failed to compile or run.")
158+
159+
# Save current results
160+
output_file = f"test-results-{project_name}.txt"
161+
save_results(current_results, output_file)
162+
163+
# If no baseline, just print summary and exit
164+
if not baseline_file or not baseline_file.exists():
165+
print_summary(project_name, current_results)
166+
if baseline_file and not baseline_file.exists():
167+
print(f"WARNING: No baseline file found at {baseline_file}")
168+
print("This is the first PR run - baseline will be created on master merge")
169+
print("━" * 60)
170+
sys.exit(0)
171+
172+
# Load baseline and compare
173+
baseline_results = load_results(baseline_file)
174+
success = compare_with_baseline(project_name, current_results, baseline_results)
175+
176+
sys.exit(0 if success else 1)
177+
178+
179+
if __name__ == '__main__':
180+
main()

‎.github/scripts/matrices.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ def __init__(
7070
t_linux_x86 = Target("parity-large-new", "x86_64-unknown-linux-gnu", "linux-amd64")
7171
# TODO: Figure out how to make this work
7272
# t_linux_arm = Target("ubuntu-latest", "aarch64-unknown-linux-gnu", "linux-aarch64")
73-
t_macos = Target("parity-macos", "aarch64-apple-darwin", "macosx-aarch64")
73+
t_macos = Target("macos-latest", "aarch64-apple-darwin", "macosx-aarch64")
7474
t_windows = Target("windows-latest", "x86_64-pc-windows-msvc", "windows-amd64")
7575
targets = [t_linux_x86, t_macos, t_windows] if is_pr else [t_linux_x86, t_macos, t_windows]
7676

@@ -99,12 +99,13 @@ def __init__(
9999
n_partitions=2,
100100
pr_cross_platform=False,
101101
),
102-
Case(
103-
name="integration / polkadot_localnode",
104-
filter="(package(=cast) | package(=forge)) & test(/polkadot_localnode/)",
105-
n_partitions=1,
106-
pr_cross_platform=False,
107-
),
102+
# TODO: run the local node tests on polkadot-anvil
103+
# Case(
104+
# name="integration / polkadot_localnode",
105+
# filter="(package(=cast) | package(=forge)) & test(/polkadot_localnode/)",
106+
# n_partitions=1,
107+
# pr_cross_platform=False,
108+
# ),
108109
]
109110

110111

‎.github/scripts/test-external-projects.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/usr/bin/env bash
2+
set -o pipefail
3+
4+
# Test external projects
5+
# Usage: test-external-projects.sh
6+
7+
# Parse projects from PROJECTS environment variable
8+
echo "$PROJECTS" | jq -c '.[]' | while read -r project; do
9+
PROJECT_NAME=$(echo "$project" | jq -r '.name')
10+
REPO=$(echo "$project" | jq -r '.repo')
11+
WORKING_DIR=$(echo "$project" | jq -r '.working_dir // ""')
12+
WORKING_DIRS=$(echo "$project" | jq -r '.working_dirs // ""')
13+
SETUP=$(echo "$project" | jq -r '.setup // ""')
14+
15+
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
16+
echo "Testing: $PROJECT_NAME"
17+
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
18+
19+
# Clone project
20+
git clone --depth 1 --recursive "https://github.com/$REPO" "test-projects/$PROJECT_NAME"
21+
cd "test-projects/$PROJECT_NAME"
22+
23+
# Run setup if provided
24+
if [ -n "$SETUP" ]; then
25+
eval "$SETUP"
26+
fi
27+
28+
# Run tests
29+
if [ -n "$WORKING_DIRS" ]; then
30+
# Multiple directories
31+
IFS=',' read -ra DIRS <<< "$WORKING_DIRS"
32+
for dir in "${DIRS[@]}"; do
33+
echo "Testing in: $dir"
34+
cd "$dir"
35+
forge test --polkadot 2>&1 | tee -a "${GITHUB_WORKSPACE}/test-output-${PROJECT_NAME}.log"
36+
cd - > /dev/null
37+
done
38+
else
39+
# Single directory
40+
if [ -n "$WORKING_DIR" ]; then
41+
cd "$WORKING_DIR"
42+
fi
43+
forge test --polkadot 2>&1 | tee "${GITHUB_WORKSPACE}/test-output-${PROJECT_NAME}.log"
44+
fi
45+
46+
cd "$GITHUB_WORKSPACE"
47+
done

‎.github/workflows/benchmarks.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ jobs:
4141
sudo apt-get install -y build-essential pkg-config
4242
4343
- name: Setup Rust toolchain
44-
uses: dtolnay/rust-toolchain@1.88.0
44+
uses: dtolnay/rust-toolchain@1.89.0
4545

4646
- name: Cache Rust dependencies
4747
uses: Swatinem/rust-cache@v2

‎.github/workflows/docker-publish.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525
timeout-minutes: 120
2626
steps:
2727
- uses: actions/checkout@v4
28-
- uses: dtolnay/rust-toolchain@1.88.0
28+
- uses: dtolnay/rust-toolchain@1.89.0
2929
- uses: Swatinem/rust-cache@v2
3030
with:
3131
cache-on-failure: true

0 commit comments

Comments
 (0)