Merge branch 'dev' into eip-7495

Author: etan-status

.circleci/config.yml

@@ -35,14 +35,14 @@ commands:
     description: "Restore the cache with pyspec keys"
     steps:
       - restore_cached_venv:
-          venv_name: v30-pyspec
-          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }}
+          venv_name: v33-pyspec
+          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "pyproject.toml" }}-{{ python3 --version }}
   save_pyspec_cached_venv:
     description: Save a venv into a cache with pyspec keys"
     steps:
       - save_cached_venv:
-          venv_name: v30-pyspec
-          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }}
+          venv_name: v33-pyspec
+          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "pyproject.toml" }}-{{ python3 --version }}
           venv_path: ./venv
 jobs:
   checkout_specs:
@@ -75,7 +75,7 @@ jobs:
       - restore_pyspec_cached_venv
       - run:
           name: Install pyspec requirements
-          command: make eth2spec
+          command: make pyspec
       - save_pyspec_cached_venv
   test-phase0:
     docker:

.github/workflows/generate_vectors.yml

@@ -17,6 +17,7 @@ on:
 
 jobs:
   generate-tests:
+    timeout-minutes: 720  # 12 hours
     runs-on: [self-hosted-ghr-custom, size-xl-x64, profile-consensusSpecs]
     steps:
       - name: Checkout repository
@@ -25,12 +26,6 @@ jobs:
           repository: 'ethereum/consensus-specs'
           path: 'consensus-specs'
           ref: ${{ inputs.ref || 'dev' }}
-      - name: Checkout consensus-spec-tests repository
-        uses: actions/checkout@v4
-        with:
-          repository: 'ethereum/consensus-spec-tests'
-          path: 'consensus-spec-tests'
-          fetch-depth: 1
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
@@ -39,7 +34,7 @@ jobs:
       - name: Generate tests
         run: |
           cd consensus-specs
-          make -j 16 gen_all 2>&1 | tee ../consensustestgen.log
+          make -j$(nproc) gen_all 2>&1 | tee ../consensustestgen.log
           cp -r presets/ ../consensus-spec-tests/presets
           cp -r configs/ ../consensus-spec-tests/configs
           find . -type d -empty -delete
@@ -78,4 +73,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: consensustestgen.log
-          path: consensustestgen.log
+          path: consensustestgen.log

Makefile

@@ -16,11 +16,9 @@ ALL_EXECUTABLE_SPEC_NAMES = \
 
 # A list of fake targets.
 .PHONY: \
-	check_toc     \
 	clean         \
 	coverage      \
 	detect_errors \
-	eth2spec      \
 	gen_all       \
 	gen_list      \
 	help          \
@@ -39,11 +37,9 @@ NORM = $(shell tput sgr0)
 
 # Print target descriptions.
 help:
-	@echo "make $(BOLD)check_toc$(NORM)     -- check table of contents"
 	@echo "make $(BOLD)clean$(NORM)         -- delete all untracked files"
 	@echo "make $(BOLD)coverage$(NORM)      -- run pyspec tests with coverage"
 	@echo "make $(BOLD)detect_errors$(NORM) -- detect generator errors"
-	@echo "make $(BOLD)eth2spec$(NORM)      -- force rebuild eth2spec package"
 	@echo "make $(BOLD)gen_<gen>$(NORM)     -- run a single generator"
 	@echo "make $(BOLD)gen_all$(NORM)       -- run all generators"
 	@echo "make $(BOLD)gen_list$(NORM)      -- list all generator targets"
@@ -62,33 +58,23 @@ PYTHON_VENV = $(VENV)/bin/python3
 PIP_VENV = $(VENV)/bin/pip3
 CODESPELL_VENV = $(VENV)/bin/codespell
 
-# Make a virtual environment will all of the necessary dependencies.
-$(VENV): requirements_preinstallation.txt
+# Make a virtual environment.
+$(VENV):
 	@echo "Creating virtual environment"
 	@python3 -m venv $(VENV)
-	@$(PIP_VENV) install -r requirements_preinstallation.txt
+	@$(PIP_VENV) install --quiet uv==0.5.24
 
 ###############################################################################
 # Specification
 ###############################################################################
 
 TEST_LIBS_DIR = $(CURDIR)/tests/core
 PYSPEC_DIR = $(TEST_LIBS_DIR)/pyspec
-SITE_PACKAGES := $(wildcard $(VENV)/lib/python*/site-packages)
-ETH2SPEC := $(SITE_PACKAGES)/eth2spec
-
-# Install the eth2spec package.
-# The pipe indicates that venv is an order-only prerequisite.
-# When restoring venv cache, its timestamp is newer than eth2spec.
-$(ETH2SPEC): setup.py | $(VENV)
-	@$(PIP_VENV) install .[docs,lint,test,generator]
-
-# Force rebuild/install the eth2spec package.
-eth2spec:
-	$(MAKE) --always-make $(ETH2SPEC)
 
 # Create the pyspec for all phases.
-pyspec: $(VENV) setup.py
+pyspec: $(VENV) setup.py pyproject.toml
+	@echo "Building eth2spec"
+	@$(PYTHON_VENV) -m uv pip install --reinstall-package=eth2spec .[docs,lint,test,generator]
 	@echo "Building all pyspecs"
 	@$(PYTHON_VENV) setup.py pyspecdev
 
@@ -99,6 +85,8 @@ pyspec: $(VENV) setup.py
 TEST_REPORT_DIR = $(PYSPEC_DIR)/test-reports
 
 # Run pyspec tests.
+# Note: for debugging output to show, print to stderr.
+#
 # To run a specific test, append k=<test>, eg:
 #   make test k=test_verify_kzg_proof
 # To run tests for a specific fork, append fork=<fork>, eg:
@@ -113,10 +101,11 @@ test: MAYBE_TEST := $(if $(k),-k=$(k))
 test: MAYBE_FORK := $(if $(fork),--fork=$(fork))
 test: PRESET := --preset=$(if $(preset),$(preset),minimal)
 test: BLS := --bls-type=$(if $(bls),$(bls),fastest)
-test: $(ETH2SPEC) pyspec
+test: pyspec
 	@mkdir -p $(TEST_REPORT_DIR)
 	@$(PYTHON_VENV) -m pytest \
 		-n auto \
+		--capture=no \
 		$(MAYBE_TEST) \
 		$(MAYBE_FORK) \
 		$(PRESET) \
@@ -136,7 +125,7 @@ COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth2spec.$S.$(
 # Run pytest with coverage tracking
 _test_with_coverage: MAYBE_TEST := $(if $(k),-k=$(k))
 _test_with_coverage: MAYBE_FORK := $(if $(fork),--fork=$(fork))
-_test_with_coverage: $(ETH2SPEC) pyspec
+_test_with_coverage: pyspec
 	@$(PYTHON_VENV) -m pytest \
 		-n auto \
 		$(MAYBE_TEST) \
@@ -193,10 +182,6 @@ MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/*/*.md) \
                  $(wildcard $(SPEC_DIR)/_features/*/*/*.md) \
                  $(wildcard $(SSZ_DIR)/*.md)
 
-# Check all files and error if any ToC were modified.
-check_toc: $(MARKDOWN_FILES:=.toc)
-	@[ "$$(find . -name '*.md.tmp' -print -quit)" ] && exit 1 || exit 0
-
 # Generate ToC sections & save copy of original if modified.
 %.toc:
 	@cp $* $*.tmp; \
@@ -209,8 +194,12 @@ check_toc: $(MARKDOWN_FILES:=.toc)
 		echo "\033[1;34m See $*.tmp\033[0m"; \
 	fi
 
+# Check all files and error if any ToC were modified.
+_check_toc: $(MARKDOWN_FILES:=.toc)
+	@[ "$$(find . -name '*.md.tmp' -print -quit)" ] && exit 1 || exit 0
+
 # Check for mistakes.
-lint: $(ETH2SPEC) pyspec check_toc
+lint: pyspec _check_toc
 	@$(CODESPELL_VENV) . --skip "./.git,$(VENV),$(PYSPEC_DIR)/.mypy_cache" -I .codespell-whitelist
 	@$(PYTHON_VENV) -m flake8 --config $(FLAKE8_CONFIG) $(PYSPEC_DIR)/eth2spec
 	@$(PYTHON_VENV) -m flake8 --config $(FLAKE8_CONFIG) $(TEST_GENERATORS_DIR)
@@ -235,17 +224,19 @@ gen_list:
 	done
 
 # Run one generator.
+# This will forcibly rebuild pyspec just in case.
 # To check modules for a generator, append modcheck=true, eg:
 #   make gen_genesis modcheck=true
 gen_%: MAYBE_MODCHECK := $(if $(filter true,$(modcheck)),--modcheck)
-gen_%: $(ETH2SPEC) pyspec
+gen_%: pyspec
 	@mkdir -p $(TEST_VECTOR_DIR)
 	@$(PYTHON_VENV) $(GENERATOR_DIR)/$*/main.py \
 		--output $(TEST_VECTOR_DIR) \
 		$(MAYBE_MODCHECK)
 
 # Run all generators then check for errors.
-gen_all: $(GENERATOR_TARGETS) detect_errors
+gen_all: $(GENERATOR_TARGETS)
+	@$(MAKE) detect_errors
 
 # Detect errors in generators.
 detect_errors: $(TEST_VECTOR_DIR)
@@ -257,7 +248,7 @@ detect_errors: $(TEST_VECTOR_DIR)
 	fi
 
 # Generate KZG trusted setups for testing.
-kzg_setups: $(ETH2SPEC)
+kzg_setups: pyspec
 	@for preset in minimal mainnet; do \
 		$(PYTHON_VENV) $(SCRIPTS_DIR)/gen_kzg_trusted_setups.py \
 			--secret=1337 \

README.md

@@ -6,11 +6,11 @@ To learn more about proof-of-stake and sharding, see the [PoS documentation](htt
 
 This repository hosts the current Ethereum proof-of-stake specifications. Discussions about design rationale and proposed changes can be brought up and discussed as issues. Solidified, agreed-upon changes to the spec can be made through pull requests.
 
-## Specs
+## Specifications
 
 [![GitHub release](https://img.shields.io/github/v/release/ethereum/consensus-specs)](https://github.com/ethereum/consensus-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec) [![testgen](https://github.com/ethereum/consensus-specs/actions/workflows/generate_vectors.yml/badge.svg?branch=dev&event=schedule)](https://github.com/ethereum/consensus-specs/actions/workflows/generate_vectors.yml)
 
-Core specifications for Ethereum proof-of-stake clients can be found in [specs](./specs). These are divided into features.
+Core specifications for Ethereum proof-of-stake clients can be found in [specs](specs). These are divided into features.
 Features are researched and developed in parallel, and then consolidated into sequential upgrades when ready.
 
 ### Stable Specifications
@@ -30,14 +30,6 @@ Features are researched and developed in parallel, and then consolidated into se
 | 5 | **Electra** | TBD | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/electra/beacon-chain.md)</li><li>[Electra fork](specs/electra/fork.md)</li></ul><li>Additions</li><ul><li>[Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [networking](specs/electra/light-client/p2p-interface.md))</li><li>[Honest validator guide changes](specs/electra/validator.md)</li><li>[P2P networking](specs/electra/p2p-interface.md)</li></ul></ul> |
 | 6 | **Fulu** | TBD | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/fulu/beacon-chain.md)</li><li>[Fulu fork](specs/fulu/fork.md)</li><li>[Data availability sampling core](specs/fulu/das-core.md)</li><li>[Polynomial commitments sampling](specs/fulu/polynomial-commitments-sampling.md)</li><li>[Fork choice changes](specs/fulu/fork-choice.md)</li></ul><li>Additions</li><ul><li>[P2P networking](specs/fulu/p2p-interface.md)</li><li>[Peer sampling](specs/fulu/peer-sampling.md)</li></ul></ul> |
 
-### Outdated Specifications
-
-| Code Name or Topic | Specs | Notes |
-| - | - | - |
-| Sharding | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/_features/sharding/beacon-chain.md)</li></ul><li>Additions</li><ul><li>[P2P networking](specs/_features/sharding/p2p-interface.md)</li></ul></ul> |
-| Custody Game | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/_features/custody_game/beacon-chain.md)</li></ul><li>Additions</li><ul><li>[Honest validator guide changes](specs/_features/custody_game/validator.md)</li></ul></ul> | Dependent on sharding |
-| Data Availability Sampling | <ul><li>Core</li><ul><li>[Core types and functions](specs/_features/das/das-core.md)</li><li>[Fork choice changes](specs/_features/das/fork-choice.md)</li></ul><li>Additions</li><ul><li>[P2P Networking](specs/_features/das/p2p-interface.md)</li><li>[Sampling process](specs/_features/das/sampling.md)</li></ul></ul> | <ul><li> Dependent on sharding</li><li>[Technical explainer](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD)</li></ul> |
-
 ### Accompanying documents can be found in [specs](specs) and include:
 
 * [SimpleSerialize (SSZ) spec](ssz/simple-serialize.md)
@@ -50,7 +42,7 @@ Additional specifications and standards outside of requisite client functionalit
 
 * [Beacon APIs](https://github.com/ethereum/beacon-apis)
 * [Engine APIs](https://github.com/ethereum/execution-apis/tree/main/src/engine)
-* [Beacon Metrics](https://github.com/ethereum/beacon-metrics/)
+* [Beacon Metrics](https://github.com/ethereum/beacon-metrics)
 
 ## Design goals
 
@@ -81,8 +73,8 @@ Documentation on the different components used during spec writing can be found
 
 Conformance tests built from the executable python spec are available in the [Ethereum Proof-of-Stake Consensus Spec Tests](https://github.com/ethereum/consensus-spec-tests) repo. Compressed tarballs are available in [releases](https://github.com/ethereum/consensus-spec-tests/releases).
 
+## Installation and usage
 
-## Installation and Usage
 The consensus-specs repo can be used by running the tests locally or inside a docker container.
 
 To run the tests locally:

configs/mainnet.yaml

@@ -115,15 +115,13 @@ DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa
 # Networking
 # ---------------------------------------------------------------
 # `10 * 2**20` (= 10485760, 10 MiB)
-GOSSIP_MAX_SIZE: 10485760
+MAX_PAYLOAD_SIZE: 10485760
 # `2**10` (= 1024)
 MAX_REQUEST_BLOCKS: 1024
 # `2**8` (= 256)
 EPOCHS_PER_SUBNET_SUBSCRIPTION: 256
 # `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months)
 MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024
-# `10 * 2**20` (=10485760, 10 MiB)
-MAX_CHUNK_SIZE: 10485760
 # 5s
 TTFB_TIMEOUT: 5
 # 10s
@@ -172,9 +170,10 @@ DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128
 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
 SAMPLES_PER_SLOT: 8
 CUSTODY_REQUIREMENT: 4
+VALIDATOR_CUSTODY_REQUIREMENT: 8
+BALANCE_PER_ADDITIONAL_CUSTODY_GROUP: 32000000000
 MAX_BLOBS_PER_BLOCK_FULU: 12
-# `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_FULU`
-MAX_REQUEST_BLOB_SIDECARS_FULU: 1536
+MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096
 
 # Whisk
 # `Epoch(2**8)`

configs/minimal.yaml

@@ -116,15 +116,13 @@ DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890
 # Networking
 # ---------------------------------------------------------------
 # `10 * 2**20` (= 10485760, 10 MiB)
-GOSSIP_MAX_SIZE: 10485760
+MAX_PAYLOAD_SIZE: 10485760
 # `2**10` (= 1024)
 MAX_REQUEST_BLOCKS: 1024
 # `2**8` (= 256)
 EPOCHS_PER_SUBNET_SUBSCRIPTION: 256
 # [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272)
 MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272
-# `10 * 2**20` (=10485760, 10 MiB)
-MAX_CHUNK_SIZE: 10485760
 # 5s
 TTFB_TIMEOUT: 5
 # 10s
@@ -173,9 +171,10 @@ DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128
 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
 SAMPLES_PER_SLOT: 8
 CUSTODY_REQUIREMENT: 4
+VALIDATOR_CUSTODY_REQUIREMENT: 8
+BALANCE_PER_ADDITIONAL_CUSTODY_GROUP: 32000000000
 MAX_BLOBS_PER_BLOCK_FULU: 12
-# `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_FULU`
-MAX_REQUEST_BLOB_SIDECARS_FULU: 1536
+MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096
 
 # Whisk
 WHISK_EPOCHS_PER_SHUFFLING_PHASE: 4

docker/README.md

@@ -1,18 +1,20 @@
 ## Docker related information
 
 This dockerfile sets up the dependencies required to run consensus-spec tests. The docker image can be locally built with:
-- `docker build ./ -t $IMAGE_NAME -f ./docker/Dockerfile`
 
+- `docker build ./ -t $IMAGE_NAME -f ./docker/Dockerfile`
 
 Handy commands:
+
 - `docker run -it $IMAGE_NAME /bin/sh` will give you a shell inside the docker container to manually run any tests
 - `docker run $IMAGE_NAME make test` will run the make test command inside the docker container
 
 Ideally manual running of docker containers is for advanced users, we recommend the script based approach described below for most users.
 
-The `scripts/build_run_docker_tests.sh` script will cover most usecases. The script allows the user to configure the fork(altair/bellatrix/capella..), `$IMAGE_NAME` (specifies the container to use), preset type (mainnet/minimal), and test all forks flags. Ideally, this is the main way that users interact with the spec tests instead of running it locally with varying versions of dependencies.
+The `scripts/build_run_docker_tests.sh` script will cover most use cases. The script allows the user to configure the fork(altair/bellatrix/capella..), `$IMAGE_NAME` (specifies the container to use), preset type (mainnet/minimal), and test all forks flags. Ideally, this is the main way that users interact with the spec tests instead of running it locally with varying versions of dependencies.
 
 E.g:
+
 - `./build_run_docker_tests.sh --p mainnet` will run the mainnet preset tests
 - `./build_run_docker_tests.sh --a` will run all the tests across all the forks
 - `./build_run_docker_tests.sh --f deneb` will only run deneb tests