feat: remove l2-node in e2e test and make it to run in local stack op… (#1001)

* feat: remove l2-node in e2e test and make it to run in local stack optionally

* fix: l2 spec transaction data size limit e2e test

* feat: revise blockscout configs

* feat: revised comment in Makefile

* feat: update transaction-exclusion-api and postman image

* feat: remove expected-traces-api-version-v2 from coordinator local overrides toml

* feat: remove shomei-frontend from ci and add disabled option in Type2StateProofProviderConfig

* feat: add endpoint for type2-state-proof-provider

* feat: revise coordinator local-dev override configs

* feat: remove white space

Author: jonesho

Makefile

@@ -23,6 +23,7 @@ start-env: CLEAN_PREVIOUS_ENV:=true
 start-env: COMPOSE_FILE:=docker/compose-tracing-v2.yml
 start-env: L1_CONTRACT_VERSION:=6
 start-env: SKIP_CONTRACTS_DEPLOYMENT:=false
+start-env: SKIP_L1_L2_NODE_HEALTH_CHECK:=false
 start-env: LINEA_PROTOCOL_CONTRACTS_ONLY:=false
 start-env:
 	@if [ "$(CLEAN_PREVIOUS_ENV)" = "true" ]; then \
@@ -32,8 +33,9 @@ start-env:
 	fi; \
 	mkdir -p tmp/local; \
 	COMPOSE_PROFILES=$(COMPOSE_PROFILES) docker compose -f $(COMPOSE_FILE) up -d; \
-	while [ "$$(docker compose -f $(COMPOSE_FILE) ps -q l1-el-node | xargs docker inspect -f '{{.State.Health.Status}}')" != "healthy" ] || \
-  			[ "$$(docker compose -f $(COMPOSE_FILE) ps -q sequencer | xargs docker inspect -f '{{.State.Health.Status}}')" != "healthy" ]; do \
+	while [ "$(SKIP_L1_L2_NODE_HEALTH_CHECK)" = "false" ] && \
+			{ [ "$$(docker compose -f $(COMPOSE_FILE) ps -q l1-el-node | xargs docker inspect -f '{{.State.Health.Status}}')" != "healthy" ] || \
+  			[ "$$(docker compose -f $(COMPOSE_FILE) ps -q sequencer | xargs docker inspect -f '{{.State.Health.Status}}')" != "healthy" ]; }; do \
   			sleep 2; \
   			echo "Checking health status of l1-el-node and sequencer..."; \
   	done
@@ -44,17 +46,17 @@ start-env:
 	fi
 
 start-l1:
-	command start-env COMPOSE_PROFILES:=l1 COMPOSE_FILE:=docker/compose-tracing-v2.yml SKIP_CONTRACTS_DEPLOYMENT:=true
+	make start-env COMPOSE_PROFILES:=l1 COMPOSE_FILE:=docker/compose-tracing-v2.yml SKIP_CONTRACTS_DEPLOYMENT:=true SKIP_L1_L2_NODE_HEALTH_CHECK:=true
 
-start-l2:
-	command start-env COMPOSE_PROFILES:=l2 COMPOSE_FILE:=docker/compose-tracing-v2.yml SKIP_CONTRACTS_DEPLOYMENT:=true
+start-l1-l2:
+	make start-env COMPOSE_PROFILES:=l1,l2 COMPOSE_FILE:=docker/compose-tracing-v2.yml SKIP_CONTRACTS_DEPLOYMENT:=true SKIP_L1_L2_NODE_HEALTH_CHECK:=true
 
 start-l2-blockchain-only:
-	command start-env COMPOSE_PROFILES:=l2-bc COMPOSE_FILE:=docker/compose-tracing-v2.yml SKIP_CONTRACTS_DEPLOYMENT:=true
+	make start-env COMPOSE_PROFILES:=l2-bc COMPOSE_FILE:=docker/compose-tracing-v2.yml SKIP_CONTRACTS_DEPLOYMENT:=true SKIP_L1_L2_NODE_HEALTH_CHECK:=true
 
 fresh-start-l2-blockchain-only:
-		make clean-environment
-		make start-l2-blockchain-only
+	make clean-environment
+	make start-l2-blockchain-only
 
 ##
 ## Creating new targets to avoid conflicts with existing targets
@@ -63,8 +65,12 @@ fresh-start-l2-blockchain-only:
 start-env-with-tracing-v2:
 	make start-env COMPOSE_FILE=docker/compose-tracing-v2.yml LINEA_PROTOCOL_CONTRACTS_ONLY=true
 
+## Enable L2 geth node
+start-env-with-tracing-v2-extra:
+	make start-env COMPOSE_PROFILES:=l1,l2 COMPOSE_FILE:=docker/compose-tracing-v2-extra-extension.yml LINEA_PROTOCOL_CONTRACTS_ONLY=true DISABLE_JSON_RPC_PRICING_PROPAGATION=false DISABLE_TYPE2_STATE_PROOF_PROVIDER=false
+
 start-env-with-tracing-v2-ci:
-	make start-env COMPOSE_FILE=docker/compose-tracing-v2-ci-extension.yml DISABLE_JSON_RPC_PRICING_PROPAGATION=false
+	make start-env COMPOSE_FILE=docker/compose-tracing-v2-ci-extension.yml
 
 start-env-with-staterecovery: COMPOSE_PROFILES:=l1,l2,staterecovery
 start-env-with-staterecovery: L1_CONTRACT_VERSION:=6

build.gradle

@@ -85,7 +85,7 @@ allprojects {
     systemProperty("L1_RPC_URL", "http://localhost:8445")
     systemProperty("L2_RPC_URL", "http://localhost:8545")
     systemProperty("L1_GENESIS", "docker/config/l1-node/el/genesis.json")
-    systemProperty("L2_GENESIS", "docker/config/linea-local-dev-genesis-PoA-geth.json")
+    systemProperty("L2_GENESIS", "docker/config/linea-local-dev-genesis-PoA-besu.json")
 
     systemProperties["junit.jupiter.execution.timeout.default"] = "5 m" // 5 minutes
     systemProperties["junit.jupiter.execution.parallel.enabled"] = true

config/blockscout/l1-blockscout.env

@@ -18,8 +18,8 @@ BLOCKSCOUT_PROTOCOL=http #use https for prod
 # SECRET_KEY_BASE=
 # CHECK_ORIGIN=
 PORT=4000
-COIN=GOERLI-ETH
-COIN_NAME="Test Eth"
+COIN="ethereum"
+COIN_NAME="ETH"
 # METADATA_CONTRACT=
 # VALIDATORS_CONTRACT=
 # KEYS_MANAGER_CONTRACT=
@@ -88,7 +88,7 @@ TXS_STATS_DAYS_TO_COMPILE_AT_INIT=10
 COIN_BALANCE_HISTORY_DAYS=90
 APPS_MENU=false
 # GAS_PRICE=
-CHAIN_ID=19940131
+CHAIN_ID=31648428
 MAX_SIZE_UNLESS_HIDE_ARRAY=50
 HIDE_BLOCK_MINER=false
 DISPLAY_TOKEN_ICONS=false

config/blockscout/l2-blockscout.env

@@ -1,6 +1,6 @@
 # DOC of ENV https://docs.blockscout.com/for-developers/information-and-settings/env-variables
 # DOCKER_TAG=
-ETHEREUM_JSONRPC_VARIANT=geth
+ETHEREUM_JSONRPC_VARIANT=besu
 NETWORK=Ethereum
 SUBNETWORK="Local Linea"
 LOGO=/images/blockscout_logo.svg
@@ -14,8 +14,8 @@ API_PATH=/
 SOCKET_ROOT=/
 BLOCKSCOUT_PROTOCOL=http #use https for prod
 PORT=4000
-COIN=GOERLI-ETH
-COIN_NAME="Goerli Eth"
+COIN="ethereum"
+COIN_NAME="ETH"
 CHAIN_SPEC_PATH=/app/genesis.json
 POOL_SIZE=20
 POOL_SIZE_API=5
@@ -62,7 +62,7 @@ TXS_HISTORIAN_INIT_LAG=0
 TXS_STATS_DAYS_TO_COMPILE_AT_INIT=10
 COIN_BALANCE_HISTORY_DAYS=90
 APPS_MENU=false
-CHAIN_ID=59140
+CHAIN_ID=1337
 MAX_SIZE_UNLESS_HIDE_ARRAY=50
 HIDE_BLOCK_MINER=false
 DISPLAY_TOKEN_ICONS=false

config/coordinator/coordinator-local-dev.config-traces-v2.overrides.toml

@@ -9,11 +9,6 @@ fs-responses-directory = "tmp/local/prover/v3/compression/responses"
 fs-requests-directory = "tmp/local/prover/v3/aggregation/requests"
 fs-responses-directory = "tmp/local/prover/v3/aggregation/responses"
 
-[dynamic-gas-price-service]
-geth-gas-price-update-recipients=[
-  "http://127.0.0.1:8845"
-]
-
 [l2]
 rpc-endpoint="http://127.0.0.1:8745"
 blocks-to-finalization=0
@@ -30,4 +25,4 @@ endpoints=[]
 [l2-network-gas-pricing.json-rpc-pricing-propagation]
 disabled=true
 geth-gas-price-update-recipients=[]
-besu-gas-price-update-recipients=["http://127.0.0.1:9045"]
+besu-gas-price-update-recipients=[]

config/coordinator/coordinator-local-dev.config.overrides.toml

@@ -12,19 +12,18 @@ endpoint="http://127.0.0.1:9000"
 
 [prover]
 [prover.execution]
-fs-requests-directory = "tmp/local/prover/v3/execution/requests"
-fs-responses-directory = "tmp/local/prover/v3/execution/responses"
+fs-requests-directory="tmp/local/prover/v3/execution/requests"
+fs-responses-directory="tmp/local/prover/v3/execution/responses"
 [prover.blob-compression]
-fs-requests-directory = "tmp/local/prover/v3/compression/requests"
-fs-responses-directory = "tmp/local/prover/v3/compression/responses"
+fs-requests-directory="tmp/local/prover/v3/compression/requests"
+fs-responses-directory="tmp/local/prover/v3/compression/responses"
 [prover.proof-aggregation]
-fs-requests-directory = "tmp/local/prover/v2/aggregation/requests"
-fs-responses-directory = "tmp/local/prover/v2/aggregation/responses"
+fs-requests-directory="tmp/local/prover/v3/aggregation/requests"
+fs-responses-directory="tmp/local/prover/v3/aggregation/responses"
 
 # Config of Traces API Facade endpoint
 [traces]
 blob-compressor-version="V1_2"
-expected-traces-api-version-v2="beta-v2.1-rc14"
 [traces.counters-v2]
 endpoints=["http://127.0.0.1:8745/"]
 [traces.conflation-v2]
@@ -33,12 +32,17 @@ endpoints=["http://127.0.0.1:8745/"]
 [state-manager]
 endpoints=["http://127.0.0.1:8998/"]
 
+[type2-state-proof-provider]
+disabled=true
+endpoints=["http://127.0.0.1:8889/"]
+
 [l2-network-gas-pricing.extra-data-pricing-propagation]
-extra-data-update-recipient = "http://127.0.0.1:8545/"
+extra-data-update-recipient="http://127.0.0.1:8545/"
 
 [l2-network-gas-pricing.json-rpc-pricing-propagation]
-geth-gas-price-update-recipients = ["http://127.0.0.1:8845"]
-besu-gas-price-update-recipients = []
+disabled=true
+geth-gas-price-update-recipients=["http://127.0.0.1:8845/"]
+besu-gas-price-update-recipients=[]
 
 [l1]
 rpc-endpoint="http://127.0.0.1:8445"
@@ -52,20 +56,11 @@ finalized-block-tag="finalized"
 earliestBlock=0
 
 [l2]
-rpc-endpoint="http://127.0.0.1:8845"
+rpc-endpoint="http://127.0.0.1:9045"
 blocks-to-finalization=0
 
 [database]
 host="localhost"
 
 [api]
 observability_port=9546
-
-[type2-state-proof-provider]
-endpoints = ["http://127.0.0.1:8889/"]
-
-[conflation]
-#switch-block-number=20
-
-[type2-state-proof-provider]
-endpoints=[]

coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/L1DependentApp.kt

@@ -1077,7 +1077,10 @@ class L1DependentApp(
       l2Web3jClient: Web3j,
       vertx: Vertx
     ): LongRunningService {
-      if (type2StateProofProviderConfig == null || type2StateProofProviderConfig.endpoints.isEmpty()) {
+      if (type2StateProofProviderConfig == null ||
+        type2StateProofProviderConfig.disabled ||
+        type2StateProofProviderConfig.endpoints.isEmpty()
+      ) {
         return DisabledLongRunningService
       }
 

coordinator/app/src/main/kotlin/net/consensys/zkevm/coordinator/app/config/CoordinatorConfig.kt

@@ -479,11 +479,12 @@ data class SmartContractErrorCodesConfig(val smartContractErrors: SmartContractE
 data class GasPriceCapTimeOfDayMultipliersConfig(val gasPriceCapTimeOfDayMultipliers: TimeOfDayMultipliers)
 
 data class Type2StateProofProviderConfig(
+  override var disabled: Boolean = false,
   val endpoints: List<URL>,
   val l1QueryBlockTag: BlockParameter.Tag = BlockParameter.Tag.LATEST,
   val l1PollingInterval: Duration = Duration.ofSeconds(12),
   override val requestRetry: RequestRetryConfigTomlFriendly
-) : RequestRetryConfigurable
+) : FeatureToggleable, RequestRetryConfigurable
 
 data class TracesLimitsV2ConfigFile(val tracesLimits: Map<TracingModuleV2, UInt>)
 

coordinator/ethereum/test-utils/src/main/kotlin/net/consensys/zkevm/ethereum/AccountManager.kt

@@ -229,7 +229,7 @@ object L1AccountManager : AccountManager by WhaleBasedAccountManager(
 
 object L2AccountManager : AccountManager by WhaleBasedAccountManager(
   web3jClient = Web3jClientManager.l2Client,
-  genesisFile = getPathTo(System.getProperty("L2_GENESIS", "docker/config/linea-local-dev-genesis-PoA-geth.json")),
+  genesisFile = getPathTo(System.getProperty("L2_GENESIS", "docker/config/linea-local-dev-genesis-PoA-besu.json")),
   log = LogManager.getLogger(L2AccountManager::class.java)
 )
 

docker/compose-spec-extra-observability.yml

@@ -36,22 +36,22 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
-      l2-node:
+      l2-node-besu:
         condition: service_started
     # - smart-contract-verifier not supported for v0.1
     command: bash -c "bin/blockscout eval \"Elixir.Explorer.ReleaseTasks.create_and_migrate()\" && bin/blockscout start"
     env_file:
       - ../config/blockscout/l2-blockscout.env
     environment:
-      ETHEREUM_JSONRPC_HTTP_URL: http://l2-node:8545/
-      ETHEREUM_JSONRPC_TRACE_URL: http://l2-node:8545/
-      ETHEREUM_JSONRPC_WS_URL: ws://l2-node:8546/
+      ETHEREUM_JSONRPC_HTTP_URL: http://l2-node-besu:8545/
+      ETHEREUM_JSONRPC_TRACE_URL: http://l2-node-besu:8545/
+      ETHEREUM_JSONRPC_WS_URL: ws://l2-node-besu:8546/
       DATABASE_URL: postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@postgres:5432/l2_blockscout_db?ssl=false
       ECTO_USE_SSL: 'false'
     ports:
       - "4000:4000"
     volumes:
-      - ./config/linea-local-dev-genesis-PoA-geth.json:/app/genesis.json:ro
+      - ./config/linea-local-dev-genesis-PoA-besu.json:/app/genesis.json:ro
     networks:
       - linea
 

docker/compose-spec-l2-services.yml

@@ -48,7 +48,7 @@ services:
     hostname: l2-node
     image: consensys/linea-geth:${ZKGETH_TAG:-0588665}
     platform: linux/amd64
-    profiles: [ "l2", "l2-bc", "debug", "external-to-monorepo" ]
+    profiles: [ "l2", "debug" ]
     depends_on:
       sequencer:
         condition: service_healthy
@@ -181,7 +181,7 @@ services:
   postman:
     container_name: postman
     hostname: postman
-    image: consensys/linea-postman:${POSTMAN_TAG:-c332ec7}
+    image: consensys/linea-postman:${POSTMAN_TAG:-8a0bcc8}
     profiles: [ "l2", "debug" ]
     platform: linux/amd64
     restart: on-failure
@@ -221,7 +221,7 @@ services:
     restart: on-failure
     environment:
       config__override__l2-network-gas-pricing__json-rpc-pricing-propagation__disabled: ${DISABLE_JSON_RPC_PRICING_PROPAGATION:-true}
-      config__override__type2-state-proof-provider__endpoints: # leave empty to disable
+      config__override__type2-state-proof-provider__disabled: ${DISABLE_TYPE2_STATE_PROOF_PROVIDER:-true}
     command: [ 'java', '-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005','-Dvertx.configurationFile=/var/lib/coordinator/vertx-options.json', '-Dlog4j2.configurationFile=/var/lib/coordinator/log4j2-dev.xml', '-jar', 'libs/coordinator.jar', '--traces-limits-v2', 'config/traces-limits-v2.toml', '--smart-contract-errors', 'config/smart-contract-errors.toml', '--gas-price-cap-time-of-day-multipliers', 'config/gas-price-cap-time-of-day-multipliers.toml', 'config/coordinator-docker.config.toml', 'config/coordinator-docker-traces-v2-override.config.toml' ]
     volumes:
       - ../config/coordinator/coordinator-docker.config.toml:/opt/consensys/linea/coordinator/config/coordinator-docker.config.toml:ro
@@ -369,7 +369,7 @@ services:
     image: consensys/linea-shomei:2.3.0
     hostname: shomei-frontend
     container_name: shomei-frontend
-    profiles: [ "l2", "l2-bc", "external-to-monorepo" ]
+    profiles: [ "l2", "l2-bc" ]
     depends_on:
       zkbesu-shomei:
         condition: service_started
@@ -410,7 +410,7 @@ services:
   transaction-exclusion-api:
     hostname: transaction-exclusion-api
     container_name: transaction-exclusion-api
-    image: consensys/linea-transaction-exclusion-api:${TRANSACTION_EXCLUSION_API_TAG:-c332ec7}
+    image: consensys/linea-transaction-exclusion-api:${TRANSACTION_EXCLUSION_API_TAG:-8a0bcc8}
     profiles: [ "l2", "debug" ]
     restart: on-failure
     depends_on:

docker/compose-tracing-v2-ci-extension.yml

@@ -2,21 +2,11 @@ include:
   - compose-tracing-v2.yml
 
 services:
-  l2-node:
-    extends:
-      file: compose-spec-l2-services.yml
-      service: l2-node
-
   l2-node-besu:
     extends:
       file: compose-spec-l2-services.yml
       service: l2-node-besu
 
-  shomei-frontend:
-    extends:
-      file: compose-spec-l2-services.yml
-      service: shomei-frontend
-
   postman:
     extends:
       file: compose-spec-l2-services.yml

docker/compose-tracing-v2-extra-extension.yml

@@ -0,0 +1,29 @@
+include:
+  - compose-tracing-v2.yml
+
+services:
+  shomei-frontend:
+    extends:
+      file: compose-spec-l2-services.yml
+      service: shomei-frontend
+
+  l2-node:
+    extends:
+      file: compose-spec-l2-services.yml
+      service: l2-node
+
+  l2-node-besu:
+    extends:
+      file: compose-spec-l2-services.yml
+      service: l2-node-besu
+
+  postman:
+    extends:
+      file: compose-spec-l2-services.yml
+      service: postman
+
+  transaction-exclusion-api:
+    extends:
+      file: compose-spec-l2-services.yml
+      service: transaction-exclusion-api
+

docker/config/l2-node-besu/l2-node-besu-config.toml

@@ -46,6 +46,7 @@ plugin-linea-tx-pool-profitability-check-api-enabled=true
 plugin-linea-tx-pool-simulation-check-api-enabled=true
 plugin-linea-tx-pool-simulation-check-p2p-enabled=true
 plugin-linea-extra-data-pricing-enabled=true
+plugin-linea-max-tx-calldata-size=30000 # lower this to 30000 (default 60000) for the transaction data limit e2e test 
 plugin-linea-tx-pool-min-margin="0.8"
 plugin-linea-min-margin="1.0"
 plugin-linea-fixed-gas-cost-wei=30000000