feat: implement pectra (#1359)

* wip: add pectra support

* xxx

* xxx

* update spec parsing

* updates

* updates

* updates

* updates

* update migrations

* work

* fix(migrations): pectra tables

* fix(migrations): dont fail if consensus_layer_events table exists

* ops: add staging-3 to gh-workflows

* fix: added pectra configs

(cherry picked from commit a17f8f8)

* refactor(api): validator addition

* fix(api): correct mainnet limits, returning dashboard eb sum

(cherry picked from commit cb05f80)

* feat: added gnosis premium perks

(cherry picked from commit fa6bcd9)

* feat: added pectra-devnet-6 configs

(cherry picked from commit 300f299)

* feat: basic apr calc

(cherry picked from commit 8a33529)

* feat: withdrawal credential handling

(cherry picked from commit c90fe0c)

* feat: deposits total

* feat: more spec config

* fix: backwards compatible deposits

* refactor: remove hotfixes

* fix(frontend, network): outdated devnet entries

* fix(db): do not fatal if goose fails to retrieve version

* BEDS-140: feat(exporter): clickhouse cl dashboard data exporter

* fix(network): support httpClient = nil

* fix(migrations): fix pectra-tables migrations

* fix(migrations): fix network data types

* fix(migrations): change order

* add(migrations): add eth1indexer el requests migration

* fix(slot_exporter): add workaround for holesky issues

* fix(slot_exporter): make cl events being present mandatory again

* add(slot_exporter): transform RemovedExcessBalance events

* feat(exporter): improve validator status update query

* fix(db): TransformRemovedExcessBalanceEvents

* fix: make linter happy about frontend-types

* fix(slot_exporter): disable late-slot-proposal only for holesky

* ops: add deploy-v2-staging-sepolia

* fix(slot_expotrer): TransformRemovedExcessBalanceEvents syntax

* feat: adapt pricing page to charge by `effective balance`

See: BEDS-1257
- these changes are caused by the Pectra hardfork
- refactor `PremiumAddonBox`

* chore(eslint): apply `sort-modules` rule

* chore(git-blame-ignore): add `styling` commit

* chore(deps): update `typescript & vue-tsc`

Conditional props did not fail for `ariaLabel`

See: 7474faa

* refactor: make usage of `locale` typesafe

* fix: added pectra configs

(cherry picked from commit a17f8f8)
(cherry picked from commit 2450ba3)

* refactor: clarified variable names & logic flow

* feat: eb-based vdb limits

* feat: clarified validator addition logic

* refactor: split user dashboards function

* refactor: move validators eb info to own file

* refactor: moved validator slice eb checks to handler

* refactor: removed dead code

* feat: eb-based vdb limits

* feat: stat calculation based on EB

* refactor: improved code style

* fix: handle `effective balance` values as big numbers

See: BEDS-1257

* feat: add banner if `max stake limit` exceeded

See: BEDS-1258

* refactor: cleanup code

* refactor: split validator addition logic

* fix: handle empty dashboard eb calculation

* refactor: calculating EBs from validator list

* feat: show `effective balance` as limit in `DashboardValidatorManagementModal.vue`

See: BEDS-1271

* refactor(dashboard): use `overview` to get `balance` for `banner`

---------

Co-authored-by: peter <1674920+peterbitfly@users.noreply.github.com>
Co-authored-by: invis-bitfly <162128378+invis-bitfly@users.noreply.github.com>
Co-authored-by: remoterami <142154971+remoterami@users.noreply.github.com>
Co-authored-by: Patrick Pfeiffer <306324+guybrush@users.noreply.github.com>
Co-authored-by: enzo-bitfly <194075446+enzo-bitfly@users.noreply.github.com>
Co-authored-by: marcel-bitfly <174338434+marcel-bitfly@users.noreply.github.com>

Author: 6 people

.git-blame-ignore-revs

@@ -4,6 +4,8 @@
 # in order to work execute once:
 # git config blame.ignoreRevsFile .git-blame-ignore-revs
 
+# chore(eslint): apply `sort-modules` rule
+7816ff9215db01151d0f498b454319949f063e7e
 #chore(eslint): add `newline` rule for `destructured object properties`
 00f3ddc552d796c72860a4892f5084aa54dc396d
 # chore(eslint): add rule to enforce indentation of 4 spaces for json

.github/workflows/backend-publish-docker-deploy.yml

@@ -10,6 +10,7 @@ on:
       - deploy-v2-pectra-devnet-5
       - deploy-v2-pectra-devnet-6
       - deploy-v2-staging-holesky
+      - deploy-v2-staging-sepolia
 
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:

.github/workflows/frontend-publish-docker-deploy.yml

@@ -10,6 +10,7 @@ on:
       - deploy-v2-pectra-devnet-5
       - deploy-v2-pectra-devnet-6
       - deploy-v2-staging-holesky
+      - deploy-v2-staging-sepolia
 
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:

backend/cmd/exporter/main.go

@@ -54,12 +54,12 @@ func Run() {
 	}
 
 	wg := &sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		db.WriterDb, db.ReaderDb = db.MustInitDB(&cfg.WriterDatabase, &cfg.ReaderDatabase, "pgx", "postgres")
+	}()
 	if !cfg.JustV2 {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			db.WriterDb, db.ReaderDb = db.MustInitDB(&cfg.WriterDatabase, &cfg.ReaderDatabase, "pgx", "postgres")
-		}()
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
@@ -152,12 +152,12 @@ func Run() {
 	monitoring.Start()
 
 	if !cfg.JustV2 {
-		defer db.ReaderDb.Close()
-		defer db.WriterDb.Close()
 		defer db.AlloyReader.Close()
 		defer db.AlloyWriter.Close()
 		defer db.BigtableClient.Close()
 	}
+	defer db.ReaderDb.Close() // we need it to get the pectra workaround events
+	defer db.WriterDb.Close()
 	defer db.ClickHouseReader.Close()
 	defer db.ClickHouseWriter.Close()
 	defer db.ClickHouseNativeWriter.Close()

backend/cmd/misc/main.go

@@ -142,12 +142,13 @@ func Run() {
 	requires, ok := REQUIRES_LIST[opts.Command]
 	if !ok {
 		requires = misctypes.Requires{
-			Bigtable:   true,
-			Redis:      true,
-			ClNode:     true,
-			ElNode:     true,
-			UserDBs:    true,
-			NetworkDBs: true,
+			Bigtable:      true,
+			Redis:         true,
+			ClNode:        true,
+			ElNode:        true,
+			UserDBs:       true,
+			NetworkDBs:    true,
+			ClickhouseDBs: true,
 		}
 	}
 
@@ -199,9 +200,11 @@ func Run() {
 	}
 
 	// clickhouse
-	db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&cfg.ClickHouse.WriterDatabase, &cfg.ClickHouse.ReaderDatabase, "clickhouse", "clickhouse")
-	defer db.ClickHouseReader.Close()
-	defer db.ClickHouseWriter.Close()
+	if requires.ClickhouseDBs {
+		db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&cfg.ClickHouse.WriterDatabase, &cfg.ClickHouse.ReaderDatabase, "clickhouse", "clickhouse")
+		defer db.ClickHouseReader.Close()
+		defer db.ClickHouseWriter.Close()
+	}
 
 	// Initialize the persistent redis client
 	if requires.Redis {
@@ -1146,7 +1149,7 @@ func debugBlocks(clClient *rpc.LighthouseClient) error {
 		} else if clBlock.ExecutionPayload.BlockNumber != i {
 			log.Warnf("clBlock.ExecutionPayload.BlockNumber != i: %v != %v", clBlock.ExecutionPayload.BlockNumber, i)
 		} else {
-			logFields["cl.txs"] = len(clBlock.ExecutionPayload.Transactions)
+			logFields["cl.txs"] = clBlock.ExecutionPayload.TransactionsCount
 		}
 
 		log.InfoWithFields(logFields, "debug block")

backend/cmd/misc/misctypes/requires.go

@@ -1,10 +1,11 @@
 package misctypes
 
 type Requires struct {
-	Bigtable   bool
-	Redis      bool
-	ClNode     bool
-	ElNode     bool
-	NetworkDBs bool
-	UserDBs    bool
+	Bigtable      bool
+	Redis         bool
+	ClNode        bool
+	ElNode        bool
+	NetworkDBs    bool
+	UserDBs       bool
+	ClickhouseDBs bool
 }

backend/go.mod

@@ -6,7 +6,7 @@ require (
 	cloud.google.com/go/bigtable v1.31.0
 	cloud.google.com/go/secretmanager v1.14.1
 	firebase.google.com/go/v4 v4.14.1
-	github.com/ClickHouse/clickhouse-go/v2 v2.30.3
+	github.com/ClickHouse/clickhouse-go/v2 v2.30.0
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21
 	github.com/Tangui-Bitfly/ethsimtracer v0.0.0-20241031103622-e76546c3d9c1

backend/go.sum

@@ -32,8 +32,8 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/ClickHouse/ch-go v0.64.1 h1:FWpP+QU4KchgzpEekuv8YoI/fUc4H2r6Bwc5WwrzvcI=
 github.com/ClickHouse/ch-go v0.64.1/go.mod h1:RBUynvczWwVzhS6Up9lPKlH1mrk4UAmle6uzCiW4Pkc=
-github.com/ClickHouse/clickhouse-go/v2 v2.30.3 h1:m0VZqUNCJ7lOmZfmOE3HZUMixZHftKmZLqcrz2+UVHk=
-github.com/ClickHouse/clickhouse-go/v2 v2.30.3/go.mod h1:V1aZaG0ctMbd8KVi+D4loXi97duWYtHiQHMCgipKJcI=
+github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo=
+github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo=
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
 github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
@@ -222,8 +222,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg=
 github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/docker v27.4.1+incompatible h1:ZJvcY7gfwHn1JF48PfbyXg7Jyt9ZCWDW+GGXOIxEwp4=
-github.com/docker/docker v27.4.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v27.3.0+incompatible h1:BNb1QY6o4JdKpqwi9IB+HUYcRRrVN4aGFUTvDmWYK1A=
+github.com/docker/docker v27.3.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
 github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=

backend/pkg/api/data_access/data_access.go

@@ -18,6 +18,7 @@ import (
 
 type DataAccessor interface {
 	ValidatorDashboardRepository
+	ValidatorRepository
 	SearchRepository
 	NetworkRepository
 	ClientRepository

backend/pkg/api/data_access/dummy.go

@@ -213,6 +213,10 @@ func (d *DummyService) GetValidatorsFromSlices(ctx context.Context, indices []ui
 	return getDummyData[[]t.VDBValidator](ctx)
 }
 
+func (d *DummyService) GetValidatorsEffectiveBalanceTotal(ctx context.Context, indices []uint64) (uint64, error) {
+	return getDummyData[uint64](ctx)
+}
+
 func (d *DummyService) GetUserDashboards(ctx context.Context, userId uint64) (*t.UserDashboardsData, error) {
 	return getDummyStruct[t.UserDashboardsData](ctx)
 }
@@ -269,16 +273,16 @@ func (d *DummyService) AddValidatorDashboardValidators(ctx context.Context, dash
 	return getDummyData[[]t.VDBPostValidatorsData](ctx)
 }
 
-func (d *DummyService) AddValidatorDashboardValidatorsByDepositAddress(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, address string, limit uint64) ([]t.VDBPostValidatorsData, error) {
-	return getDummyData[[]t.VDBPostValidatorsData](ctx)
+func (d *DummyService) GetValidatorsByDepositAddress(ctx context.Context, depositAddress string) ([]t.VDBValidator, error) {
+	return getDummyData[[]t.VDBValidator](ctx)
 }
 
-func (d *DummyService) AddValidatorDashboardValidatorsByWithdrawalCredential(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, address string, limit uint64) ([]t.VDBPostValidatorsData, error) {
-	return getDummyData[[]t.VDBPostValidatorsData](ctx)
+func (d *DummyService) GetValidatorsByWithdrawalCredentials(ctx context.Context, withdrawalCredentials string) ([]t.VDBValidator, error) {
+	return getDummyData[[]t.VDBValidator](ctx)
 }
 
-func (d *DummyService) AddValidatorDashboardValidatorsByGraffiti(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, graffiti string, limit uint64) ([]t.VDBPostValidatorsData, error) {
-	return getDummyData[[]t.VDBPostValidatorsData](ctx)
+func (d *DummyService) GetValidatorsByGraffiti(ctx context.Context, graffiti string) ([]t.VDBValidator, error) {
+	return getDummyData[[]t.VDBValidator](ctx)
 }
 
 func (d *DummyService) GetValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, cursor string, colSort t.Sort[enums.VDBManageValidatorsColumn], search string, limit uint64) ([]t.VDBManageValidatorsTableRow, *t.Paging, error) {
@@ -541,10 +545,14 @@ func (d *DummyService) GetValidatorDashboardGroupCount(ctx context.Context, dash
 	return getDummyData[uint64](ctx)
 }
 
-func (d *DummyService) GetValidatorDashboardValidatorsCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error) {
+func (d *DummyService) GetValidatorDashboardEffectiveBalanceTotal(ctx context.Context, dashboardId t.VDBId, onlyActive bool) (uint64, error) {
 	return getDummyData[uint64](ctx)
 }
 
+func (d *DummyService) GetValidatorsEffectiveBalances(ctx context.Context, validators []t.VDBValidator, onlyActive bool) (map[t.VDBValidator]uint64, error) {
+	return map[t.VDBValidator]uint64{}, nil
+}
+
 func (d *DummyService) GetValidatorDashboardPublicIdCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error) {
 	return getDummyData[uint64](ctx)
 }