From af981958e90fa024b6ba62f822df9d8cf3a4bffc Mon Sep 17 00:00:00 2001 From: Spencer Taylor-Brown Date: Wed, 30 Jul 2025 13:04:02 +0000 Subject: [PATCH 1/5] chore(cli): improve compare fixtures script. --- src/cli/compare_fixtures.py | 73 +++++++++++++++++++++++++------------ 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/src/cli/compare_fixtures.py b/src/cli/compare_fixtures.py index 42d0a7ecd81..6bc41209bd0 100644 --- a/src/cli/compare_fixtures.py +++ b/src/cli/compare_fixtures.py @@ -9,8 +9,9 @@ import json import shutil import sys +from collections import defaultdict from pathlib import Path -from typing import Set +from typing import List, Set import click @@ -49,12 +50,19 @@ def find_duplicates(base_hashes: Set[HexNumber], patch_hashes: Set[HexNumber]) - return base_hashes & patch_hashes -def pop_by_hash(index: IndexFile, fixture_hash: HexNumber) -> TestCaseIndexFile: - """Pops a single test case from an index file by its hash.""" - for i in range(len(index.test_cases)): - if index.test_cases[i].fixture_hash == fixture_hash: - return index.test_cases.pop(i) - raise Exception(f"Hash {fixture_hash} not found in index.") +def pop_all_by_hash(index: IndexFile, fixture_hash: HexNumber) -> List[TestCaseIndexFile]: + """Pops all test cases from an index file by their hash.""" + test_cases = [] + remaining_cases = [] + for test_case in index.test_cases: + if test_case.fixture_hash == fixture_hash: + test_cases.append(test_case) + else: + remaining_cases.append(test_case) + if not test_cases: + raise Exception(f"Hash {fixture_hash} not found in index.") + index.test_cases = remaining_cases + return test_cases def remove_fixture_from_file(file: Path, test_case_id: str): @@ -70,19 +78,16 @@ def remove_fixture_from_file(file: Path, test_case_id: str): raise KeyError(f"Test case {test_case_id} not found in {file}") from None -def remove_fixture( - folder: Path, - index: IndexFile, - fixture_hash: HexNumber, - dry_run: bool, -): - """Remove a single fixture from a folder that matches the given hash.""" - test_case = pop_by_hash(index, fixture_hash) - test_case_file = folder / test_case.json_path - if dry_run: - print(f"Remove {test_case.id} from {test_case_file}") - else: - remove_fixture_from_file(test_case_file, test_case.id) +def batch_remove_fixtures_from_files(removals_by_file): + """Batch process file removals to minimize I/O.""" + for file_path, test_case_ids in removals_by_file.items(): + try: + full_file = json.loads(file_path.read_text()) + for test_case_id in test_case_ids: + full_file.pop(test_case_id, None) + file_path.write_text(json.dumps(full_file, indent=2)) + except Exception as e: + print(f"Error processing {file_path}: {e}") def rewrite_index(folder: Path, index: IndexFile, dry_run: bool): @@ -144,10 +149,32 @@ def main( click.echo("Patch folder would be empty after fixture removal.") sys.exit(0) + # Collect removals by file for batching + base_removals_by_file = defaultdict(list) + patch_removals_by_file = defaultdict(list) + for duplicate_hash in duplicate_hashes: - # Remove from both folders - remove_fixture(base, base_index, duplicate_hash, dry_run) - remove_fixture(patch, patch_index, duplicate_hash, dry_run) + base_test_cases = pop_all_by_hash(base_index, duplicate_hash) + patch_test_cases = pop_all_by_hash(patch_index, duplicate_hash) + + for base_test_case in base_test_cases: + base_file = base / base_test_case.json_path + if dry_run: + print(f"Remove {base_test_case.id} from {base_file}") + else: + base_removals_by_file[base_file].append(base_test_case.id) + + for patch_test_case in patch_test_cases: + patch_file = patch / patch_test_case.json_path + if dry_run: + print(f"Remove {patch_test_case.id} from {patch_file}") + else: + patch_removals_by_file[patch_file].append(patch_test_case.id) + + # Batch process file operations + if not dry_run: + batch_remove_fixtures_from_files(base_removals_by_file) + batch_remove_fixtures_from_files(patch_removals_by_file) # Rewrite indices if necessary rewrite_index(base, base_index, dry_run) From c9decb2c499ef0fe3ee9051a377ee487123e26e6 Mon Sep 17 00:00:00 2001 From: Spencer Taylor-Brown Date: Wed, 30 Jul 2025 13:04:48 +0000 Subject: [PATCH 2/5] chore(ci): add better logging for coverage ci step. --- .github/workflows/coverage.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 6cb7a2c285f..cb4bb34f576 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -141,14 +141,14 @@ jobs: run: | ./.github/scripts/fill_prepatched_tests.sh "$MODIFIED_TEST_FILES $DELETED_TEST_FILES" "${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS" "${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS" "${{ env.BLOCK_GAS_LIMIT }}" "${{ env.FILL_UNTIL }}" - - name: Print tests that will be covered + - name: Print Unique Test IDs that will be covered if: ${{ steps.pre-patch-fill.outputs.any_modified_fixtures == 'true' || steps.ported-from.outputs.any_ported == 'true' }} run: | - echo "Original BASE tests:" - ls ${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS + echo "=== Original BASE (main) test IDs ===" + python -c "import json; data=json.load(open('${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS/.meta/index.json')); [print(tc['id']) for tc in data['test_cases']]" echo "--------------------" - echo "Ported PATCH tests:" - ls ${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS + echo "=== Ported PATCH test IDs ===" + python -c "import json; data=json.load(open('${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS/.meta/index.json')); [print(tc['id']) for tc in data['test_cases']]" - name: Run coverage of the BASE tests uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 From d6fcee3208e1a95adc1bced8773850d7ae28abc4 Mon Sep 17 00:00:00 2001 From: Spencer Taylor-Brown Date: Wed, 30 Jul 2025 13:14:34 +0000 Subject: [PATCH 3/5] chore(ci): add prague back for coverage ci. --- .github/workflows/coverage.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index cb4bb34f576..c3e28cf00c3 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -4,7 +4,6 @@ on: pull_request: paths: - "tests/**" # This triggers the workflow for any changes in the tests folder - - "!tests/prague/**" # exclude changes in 'tests/prague' - "!tests/osaka/**" # exclude changes in 'tests/osaka' - "!tests/unscheduled/**" # exclude changes in 'tests/unscheduled' From 6b3bfb46a5628a3c5fa31b4bf5db857413533bd8 Mon Sep 17 00:00:00 2001 From: Spencer Taylor-Brown Date: Wed, 30 Jul 2025 13:21:50 +0000 Subject: [PATCH 4/5] chore(ci): add summary test diff to coverage ci, use uv for python. --- .github/workflows/coverage.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index c3e28cf00c3..7236b8a215d 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -144,10 +144,11 @@ jobs: if: ${{ steps.pre-patch-fill.outputs.any_modified_fixtures == 'true' || steps.ported-from.outputs.any_ported == 'true' }} run: | echo "=== Original BASE (main) test IDs ===" - python -c "import json; data=json.load(open('${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS/.meta/index.json')); [print(tc['id']) for tc in data['test_cases']]" - echo "--------------------" + uv run python -c "import json; data=json.load(open('${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS/.meta/index.json')); [print(tc['id']) for tc in data['test_cases']]" echo "=== Ported PATCH test IDs ===" - python -c "import json; data=json.load(open('${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS/.meta/index.json')); [print(tc['id']) for tc in data['test_cases']]" + uv run python -c "import json; data=json.load(open('${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS/.meta/index.json')); [print(tc['id']) for tc in data['test_cases']]" + echo "=== SUMMARY ===" + uv run python -c "import json; base=json.load(open('${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS/.meta/index.json')); patch=json.load(open('${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS/.meta/index.json')); print(f'BASE: {len(base[\"test_cases\"])} tests, PATCH: {len(patch[\"test_cases\"])} tests, Difference: {len(patch[\"test_cases\"]) - len(base[\"test_cases\"])}')" - name: Run coverage of the BASE tests uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 From 22d1a024fa8f4805598ef81a621924a5854f0e15 Mon Sep 17 00:00:00 2001 From: Mario Vega Date: Thu, 31 Jul 2025 02:20:42 +0200 Subject: [PATCH 5/5] Update src/cli/compare_fixtures.py --- src/cli/compare_fixtures.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cli/compare_fixtures.py b/src/cli/compare_fixtures.py index 6bc41209bd0..787e0f9f077 100644 --- a/src/cli/compare_fixtures.py +++ b/src/cli/compare_fixtures.py @@ -85,7 +85,10 @@ def batch_remove_fixtures_from_files(removals_by_file): full_file = json.loads(file_path.read_text()) for test_case_id in test_case_ids: full_file.pop(test_case_id, None) - file_path.write_text(json.dumps(full_file, indent=2)) + if len(full_file) > 0: + file_path.write_text(json.dumps(full_file, indent=2)) + else: + file_path.unlink() except Exception as e: print(f"Error processing {file_path}: {e}")