Skip to content

Commit 0264b74

Browse files
invis-bitflyinvis-bitfly
committed
feat(migrations): add backfill for botched epoch_start, balance_start, balance_min
1 parent 62c7841 commit 0264b74

10 files changed

+1422
-39
lines changed

‎backend/pkg/commons/db/migrations/clickhouse/20250723104036_add_aggregate_backfill_table.sql‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
-- +goose Up
2+
-- +goose StatementBegin
3+
CREATE TABLE IF NOT EXISTS _exporter_aggregate_backfill_metadata
4+
(
5+
`backfill_name` String COMMENT 'name of the backfill',
6+
`aggregation` String COMMENT 'aggregation type (hourly, daily, weekly, monthly)',
7+
`t` DateTime COMMENT 'timestamp of the aggregation',
8+
`max_epoch_timestamp` DateTime COMMENT 'maximum epoch timestamp to be used for the aggregation during backfill',
9+
`did_check` Nullable(DateTime) COMMENT 'timestamp when the check was performed',
10+
`backfill_id` Nullable(UUID) COMMENT 'id of the backfill',
11+
`successful_backfill` Nullable(DateTime) COMMENT 'if the backfill was successful',
12+
-- check that backfill_id is set only if did_check is not null
13+
CONSTRAINT check_backfill_id CHECK (backfill_id IS NULL OR did_check IS NOT NULL),
14+
-- check that successful_backfill is set only if backfill_id is not null
15+
CONSTRAINT check_successful_backfill CHECK (successful_backfill IS NULL OR backfill_id IS NOT NULL)
16+
)
17+
ENGINE = ReplacingMergeTree
18+
ORDER BY (backfill_name, aggregation, t)
19+
SETTINGS non_replicated_deduplication_window = 2048, replicated_deduplication_window = 2048;
20+
-- +goose StatementEnd
21+
-- +goose StatementBegin
22+
INSERT INTO _exporter_aggregate_backfill_metadata (backfill_name, aggregation, t, max_epoch_timestamp)
23+
select 'botched_epoch_start', '_final_validator_dashboard_data_hourly', t, (select max(epoch_timestamp) from _final_validator_dashboard_data_epoch) from _final_validator_dashboard_data_hourly PREWHERE validator_index = 0 limit 1 by t
24+
UNION ALL
25+
select 'botched_epoch_start', '_final_validator_dashboard_data_daily', t, (select max(epoch_timestamp) from _final_validator_dashboard_data_epoch) from _final_validator_dashboard_data_daily PREWHERE validator_index = 0 limit 1 by t
26+
UNION ALL
27+
select 'botched_epoch_start', '_final_validator_dashboard_data_weekly', t, (select max(epoch_timestamp) from _final_validator_dashboard_data_epoch) from _final_validator_dashboard_data_weekly PREWHERE validator_index = 0 limit 1 by t
28+
UNION ALL
29+
select 'botched_epoch_start', '_final_validator_dashboard_data_monthly', t, (select max(epoch_timestamp) from _final_validator_dashboard_data_epoch) from _final_validator_dashboard_data_monthly PREWHERE validator_index = 0 limit 1 by t;
30+
-- +goose StatementEnd
31+
-- latest entry in _exporter_aggregate_backfill_metadata gets marked as dirty from the get go - since we are doing the switch to the new revision it needs to be aggregated
32+
-- +goose StatementBegin
33+
INSERT INTO _exporter_aggregate_backfill_metadata (backfill_name, aggregation, t, max_epoch_timestamp, did_check, backfill_id)
34+
SELECT
35+
'botched_epoch_start',
36+
aggregation,
37+
t,
38+
max_epoch_timestamp,
39+
now(),
40+
generateUUIDv4()
41+
from _exporter_aggregate_backfill_metadata
42+
WHERE
43+
backfill_name = 'botched_epoch_start'
44+
ORDER BY t desc
45+
LIMIT 1 BY aggregation;
46+
-- +goose StatementEnd
47+
48+
49+
-- +goose Down
50+
-- +goose StatementBegin
51+
DROP TABLE IF EXISTS _exporter_aggregate_backfill_metadata;
52+
-- +goose StatementEnd

‎backend/pkg/commons/db/migrations/clickhouse/20250812170142_add_botched_epoch_start_backfill_columns.sql‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
-- +goose Up
2+
3+
-- +goose StatementBegin
4+
ALTER TABLE _final_validator_dashboard_data_hourly
5+
COMMENT COLUMN epoch_start 'legacy column, used by ingestion. consumers of data should use the aliased variant instead',
6+
COMMENT COLUMN balance_start 'legacy column, used by ingestion. consumers of data should use the aliased variant instead',
7+
COMMENT COLUMN balance_min 'legacy column, used by ingestion. consumers of data should use the aliased variant instead',
8+
ADD COLUMN IF NOT EXISTS _is_backfill SimpleAggregateFunction(max, bool) DEFAULT 0 COMMENT 'if the row is part of a backfill insert',
9+
ADD COLUMN IF NOT EXISTS _botched_epoch_start_backfill_revision SimpleAggregateFunction(max, Int64) DEFAULT 0 COMMENT 'revision of the "botched epoch start" backfill. incrementing this will cause _staging_{epoch_start|balance_start|balance_min} to be replaced with whatever the _legacy columns are set to in the same row.',
10+
ADD COLUMN IF NOT EXISTS _staging_epoch_start AggregateFunction(minSimpleStateArgMax, Int64, Int64) Materialized initializeAggregation('minSimpleStateArgMaxState', epoch_start::Int64, _botched_epoch_start_backfill_revision::Int64),
11+
ADD COLUMN IF NOT EXISTS _staging_balance_start AggregateFunction(argMinMergeStateArgMax, AggregateFunction(argMin, Int64, Int64), SimpleAggregateFunction(max, Int64)) Materialized initializeAggregation('argMinMergeStateArgMaxState', balance_start, _botched_epoch_start_backfill_revision::Int64),
12+
ADD COLUMN IF NOT EXISTS _staging_balance_min AggregateFunction(minSimpleStateArgMax, Int64, Int64) Materialized initializeAggregation('minSimpleStateArgMaxState', balance_min::Int64, _botched_epoch_start_backfill_revision::Int64),
13+
-- the existing columns will be swapped out with the following aliases once the backfill is complete
14+
ADD COLUMN IF NOT EXISTS _public_epoch_start SimpleAggregateFunction(min, Int64) ALIAS finalizeAggregation(_staging_epoch_start),
15+
ADD COLUMN IF NOT EXISTS _public_balance_start AggregateFunction(argMin, Int64, Int64) ALIAS finalizeAggregation(_staging_balance_start),
16+
ADD COLUMN IF NOT EXISTS _public_balance_min SimpleAggregateFunction(min, Int64) ALIAS finalizeAggregation(_staging_balance_min)
17+
settings mutations_sync=2, alter_sync=1;
18+
-- +goose StatementEnd
19+
-- +goose StatementBegin
20+
ALTER TABLE _final_validator_dashboard_data_daily
21+
COMMENT COLUMN epoch_start 'legacy column, used by ingestion. consumers of data should use the aliased variant instead',
22+
COMMENT COLUMN balance_start 'legacy column, used by ingestion. consumers of data should use the aliased variant instead',
23+
COMMENT COLUMN balance_min 'legacy column, used by ingestion. consumers of data should use the aliased variant instead',
24+
ADD COLUMN IF NOT EXISTS _is_backfill SimpleAggregateFunction(max, bool) DEFAULT 0 COMMENT 'if the row is part of a backfill insert',
25+
ADD COLUMN IF NOT EXISTS _botched_epoch_start_backfill_revision SimpleAggregateFunction(max, Int64) DEFAULT 0 COMMENT 'revision of the "botched epoch start" backfill. incrementing this will cause _staging_{epoch_start|balance_start|balance_min} to be replaced with whatever the _legacy columns are set to in the same row.',
26+
ADD COLUMN IF NOT EXISTS _staging_epoch_start AggregateFunction(minSimpleStateArgMax, Int64, Int64) Materialized initializeAggregation('minSimpleStateArgMaxState', epoch_start::Int64, _botched_epoch_start_backfill_revision::Int64),
27+
ADD COLUMN IF NOT EXISTS _staging_balance_start AggregateFunction(argMinMergeStateArgMax, AggregateFunction(argMin, Int64, Int64), SimpleAggregateFunction(max, Int64)) Materialized initializeAggregation('argMinMergeStateArgMaxState', balance_start, _botched_epoch_start_backfill_revision::Int64),
28+
ADD COLUMN IF NOT EXISTS _staging_balance_min AggregateFunction(minSimpleStateArgMax, Int64, Int64) Materialized initializeAggregation('minSimpleStateArgMaxState', balance_min::Int64, _botched_epoch_start_backfill_revision::Int64),
29+
-- the existing columns will be swapped out with the following aliases once the backfill is complete
30+
ADD COLUMN IF NOT EXISTS _public_epoch_start SimpleAggregateFunction(min, Int64) ALIAS finalizeAggregation(_staging_epoch_start),
31+
ADD COLUMN IF NOT EXISTS _public_balance_start AggregateFunction(argMin, Int64, Int64) ALIAS finalizeAggregation(_staging_balance_start),
32+
ADD COLUMN IF NOT EXISTS _public_balance_min SimpleAggregateFunction(min, Int64) ALIAS finalizeAggregation(_staging_balance_min)
33+
settings mutations_sync=2, alter_sync=1;
34+
-- +goose StatementEnd
35+
-- +goose StatementBegin
36+
ALTER TABLE _final_validator_dashboard_data_weekly
37+
COMMENT COLUMN epoch_start 'legacy column, used by ingestion. consumers of data should use the aliased variant instead',
38+
COMMENT COLUMN balance_start 'legacy column, used by ingestion. consumers of data should use the aliased variant instead',
39+
COMMENT COLUMN balance_min 'legacy column, used by ingestion. consumers of data should use the aliased variant instead',
40+
ADD COLUMN IF NOT EXISTS _is_backfill SimpleAggregateFunction(max, bool) DEFAULT 0 COMMENT 'if the row is part of a backfill insert',
41+
ADD COLUMN IF NOT EXISTS _botched_epoch_start_backfill_revision SimpleAggregateFunction(max, Int64) DEFAULT 0 COMMENT 'revision of the "botched epoch start" backfill. incrementing this will cause _staging_{epoch_start|balance_start|balance_min} to be replaced with whatever the _legacy columns are set to in the same row.',
42+
ADD COLUMN IF NOT EXISTS _staging_epoch_start AggregateFunction(minSimpleStateArgMax, Int64, Int64) Materialized initializeAggregation('minSimpleStateArgMaxState', epoch_start::Int64, _botched_epoch_start_backfill_revision::Int64),
43+
ADD COLUMN IF NOT EXISTS _staging_balance_start AggregateFunction(argMinMergeStateArgMax, AggregateFunction(argMin, Int64, Int64), SimpleAggregateFunction(max, Int64)) Materialized initializeAggregation('argMinMergeStateArgMaxState', balance_start, _botched_epoch_start_backfill_revision::Int64),
44+
ADD COLUMN IF NOT EXISTS _staging_balance_min AggregateFunction(minSimpleStateArgMax, Int64, Int64) Materialized initializeAggregation('minSimpleStateArgMaxState', balance_min::Int64, _botched_epoch_start_backfill_revision::Int64),
45+
-- the existing columns will be swapped out with the following aliases once the backfill is complete
46+
ADD COLUMN IF NOT EXISTS _public_epoch_start SimpleAggregateFunction(min, Int64) ALIAS finalizeAggregation(_staging_epoch_start),
47+
ADD COLUMN IF NOT EXISTS _public_balance_start AggregateFunction(argMin, Int64, Int64) ALIAS finalizeAggregation(_staging_balance_start),
48+
ADD COLUMN IF NOT EXISTS _public_balance_min SimpleAggregateFunction(min, Int64) ALIAS finalizeAggregation(_staging_balance_min)
49+
settings mutations_sync=2, alter_sync=1;
50+
-- +goose StatementEnd
51+
-- +goose StatementBegin
52+
ALTER TABLE _final_validator_dashboard_data_monthly
53+
COMMENT COLUMN epoch_start 'legacy column, used by ingestion. consumers of data should use the aliased variant instead',
54+
COMMENT COLUMN balance_start 'legacy column, used by ingestion. consumers of data should use the aliased variant instead',
55+
COMMENT COLUMN balance_min 'legacy column, used by ingestion. consumers of data should use the aliased variant instead',
56+
ADD COLUMN IF NOT EXISTS _is_backfill SimpleAggregateFunction(max, bool) DEFAULT 0 COMMENT 'if the row is part of a backfill insert',
57+
ADD COLUMN IF NOT EXISTS _botched_epoch_start_backfill_revision SimpleAggregateFunction(max, Int64) DEFAULT 0 COMMENT 'revision of the "botched epoch start" backfill. incrementing this will cause _staging_{epoch_start|balance_start|balance_min} to be replaced with whatever the _legacy columns are set to in the same row.',
58+
ADD COLUMN IF NOT EXISTS _staging_epoch_start AggregateFunction(minSimpleStateArgMax, Int64, Int64) Materialized initializeAggregation('minSimpleStateArgMaxState', epoch_start::Int64, _botched_epoch_start_backfill_revision::Int64),
59+
ADD COLUMN IF NOT EXISTS _staging_balance_start AggregateFunction(argMinMergeStateArgMax, AggregateFunction(argMin, Int64, Int64), SimpleAggregateFunction(max, Int64)) Materialized initializeAggregation('argMinMergeStateArgMaxState', balance_start, _botched_epoch_start_backfill_revision::Int64),
60+
ADD COLUMN IF NOT EXISTS _staging_balance_min AggregateFunction(minSimpleStateArgMax, Int64, Int64) Materialized initializeAggregation('minSimpleStateArgMaxState', balance_min::Int64, _botched_epoch_start_backfill_revision::Int64),
61+
-- the existing columns will be swapped out with the following aliases once the backfill is complete
62+
ADD COLUMN IF NOT EXISTS _public_epoch_start SimpleAggregateFunction(min, Int64) ALIAS finalizeAggregation(_staging_epoch_start),
63+
ADD COLUMN IF NOT EXISTS _public_balance_start AggregateFunction(argMin, Int64, Int64) ALIAS finalizeAggregation(_staging_balance_start),
64+
ADD COLUMN IF NOT EXISTS _public_balance_min SimpleAggregateFunction(min, Int64) ALIAS finalizeAggregation(_staging_balance_min)
65+
settings mutations_sync=2, alter_sync=1;
66+
-- +goose StatementEnd
67+
68+
-- +goose Down
69+
-- +goose StatementBegin
70+
ALTER TABLE _final_validator_dashboard_data_hourly
71+
DROP COLUMN IF EXISTS _is_backfill,
72+
DROP COLUMN IF EXISTS _botched_epoch_start_backfill_revision,
73+
DROP COLUMN IF EXISTS _staging_epoch_start,
74+
DROP COLUMN IF EXISTS _staging_balance_start,
75+
DROP COLUMN IF EXISTS _staging_balance_min,
76+
DROP COLUMN IF EXISTS _public_epoch_start,
77+
DROP COLUMN IF EXISTS _public_balance_start,
78+
DROP COLUMN IF EXISTS _public_balance_min
79+
settings mutations_sync=2, alter_sync=1;
80+
-- +goose StatementEnd
81+
-- +goose StatementBegin
82+
ALTER TABLE _final_validator_dashboard_data_daily
83+
DROP COLUMN IF EXISTS _is_backfill,
84+
DROP COLUMN IF EXISTS _botched_epoch_start_backfill_revision,
85+
DROP COLUMN IF EXISTS _staging_epoch_start,
86+
DROP COLUMN IF EXISTS _staging_balance_start,
87+
DROP COLUMN IF EXISTS _staging_balance_min,
88+
DROP COLUMN IF EXISTS _public_epoch_start,
89+
DROP COLUMN IF EXISTS _public_balance_start,
90+
DROP COLUMN IF EXISTS _public_balance_min
91+
settings mutations_sync=2, alter_sync=1;
92+
-- +goose StatementEnd
93+
-- +goose StatementBegin
94+
ALTER TABLE _final_validator_dashboard_data_weekly
95+
DROP COLUMN IF EXISTS _is_backfill,
96+
DROP COLUMN IF EXISTS _botched_epoch_start_backfill_revision,
97+
DROP COLUMN IF EXISTS _staging_epoch_start,
98+
DROP COLUMN IF EXISTS _staging_balance_start,
99+
DROP COLUMN IF EXISTS _staging_balance_min,
100+
DROP COLUMN IF EXISTS _public_epoch_start,
101+
DROP COLUMN IF EXISTS _public_balance_start,
102+
DROP COLUMN IF EXISTS _public_balance_min
103+
settings mutations_sync=2, alter_sync=1;
104+
-- +goose StatementEnd
105+
-- +goose StatementBegin
106+
ALTER TABLE _final_validator_dashboard_data_monthly
107+
DROP COLUMN IF EXISTS _is_backfill,
108+
DROP COLUMN IF EXISTS _botched_epoch_start_backfill_revision,
109+
DROP COLUMN IF EXISTS _staging_epoch_start,
110+
DROP COLUMN IF EXISTS _staging_balance_start,
111+
DROP COLUMN IF EXISTS _staging_balance_min,
112+
DROP COLUMN IF EXISTS _public_epoch_start,
113+
DROP COLUMN IF EXISTS _public_balance_start,
114+
DROP COLUMN IF EXISTS _public_balance_min
115+
settings mutations_sync=2, alter_sync=1;
116+
-- +goose StatementEnd

‎backend/pkg/commons/db/migrations/clickhouse/20250812170146_swap_columns.sql‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
-- +goose Up
2+
3+
-- +goose StatementBegin
4+
ALTER TABLE _final_validator_dashboard_data_hourly
5+
RENAME COLUMN epoch_start TO _legacy_epoch_start,
6+
RENAME COLUMN balance_start TO _legacy_balance_start,
7+
RENAME COLUMN balance_min TO _legacy_balance_min,
8+
RENAME COLUMN _public_epoch_start TO epoch_start,
9+
RENAME COLUMN _public_balance_start TO balance_start,
10+
RENAME COLUMN _public_balance_min TO balance_min
11+
settings mutations_sync=2, alter_sync=1;
12+
-- +goose StatementEnd
13+
-- +goose StatementBegin
14+
ALTER TABLE _final_validator_dashboard_data_daily
15+
RENAME COLUMN epoch_start TO _legacy_epoch_start,
16+
RENAME COLUMN balance_start TO _legacy_balance_start,
17+
RENAME COLUMN balance_min TO _legacy_balance_min,
18+
RENAME COLUMN _public_epoch_start TO epoch_start,
19+
RENAME COLUMN _public_balance_start TO balance_start,
20+
RENAME COLUMN _public_balance_min TO balance_min
21+
settings mutations_sync=2, alter_sync=1;
22+
-- +goose StatementEnd
23+
-- +goose StatementBegin
24+
ALTER TABLE _final_validator_dashboard_data_weekly
25+
RENAME COLUMN epoch_start TO _legacy_epoch_start,
26+
RENAME COLUMN balance_start TO _legacy_balance_start,
27+
RENAME COLUMN balance_min TO _legacy_balance_min,
28+
RENAME COLUMN _public_epoch_start TO epoch_start,
29+
RENAME COLUMN _public_balance_start TO balance_start,
30+
RENAME COLUMN _public_balance_min TO balance_min
31+
settings mutations_sync=2, alter_sync=1;
32+
-- +goose StatementEnd
33+
-- +goose StatementBegin
34+
ALTER TABLE _final_validator_dashboard_data_monthly
35+
RENAME COLUMN epoch_start TO _legacy_epoch_start,
36+
RENAME COLUMN balance_start TO _legacy_balance_start,
37+
RENAME COLUMN balance_min TO _legacy_balance_min,
38+
RENAME COLUMN _public_epoch_start TO epoch_start,
39+
RENAME COLUMN _public_balance_start TO balance_start,
40+
RENAME COLUMN _public_balance_min TO balance_min
41+
settings mutations_sync=2, alter_sync=1;
42+
-- +goose StatementEnd
43+
44+
-- +goose Down
45+
-- +goose StatementBegin
46+
ALTER TABLE _final_validator_dashboard_data_hourly
47+
RENAME COLUMN _legacy_epoch_start TO epoch_start,
48+
RENAME COLUMN _legacy_balance_start TO balance_start,
49+
RENAME COLUMN _legacy_balance_min TO balance_min,
50+
RENAME COLUMN epoch_start TO _public_epoch_start,
51+
RENAME COLUMN balance_start TO _public_balance_start,
52+
RENAME COLUMN balance_min TO _public_balance_min
53+
settings mutations_sync=2, alter_sync=1;
54+
-- +goose StatementEnd
55+
-- +goose StatementBegin
56+
ALTER TABLE _final_validator_dashboard_data_daily
57+
RENAME COLUMN _legacy_epoch_start TO epoch_start,
58+
RENAME COLUMN _legacy_balance_start TO balance_start,
59+
RENAME COLUMN _legacy_balance_min TO balance_min,
60+
RENAME COLUMN epoch_start TO _public_epoch_start,
61+
RENAME COLUMN balance_start TO _public_balance_start,
62+
RENAME COLUMN balance_min TO _public_balance_min
63+
settings mutations_sync=2, alter_sync=1;
64+
-- +goose StatementEnd
65+
-- +goose StatementBegin
66+
ALTER TABLE _final_validator_dashboard_data_weekly
67+
RENAME COLUMN _legacy_epoch_start TO epoch_start,
68+
RENAME COLUMN _legacy_balance_start TO balance_start,
69+
RENAME COLUMN _legacy_balance_min TO balance_min,
70+
RENAME COLUMN epoch_start TO _public_epoch_start,
71+
RENAME COLUMN balance_start TO _public_balance_start,
72+
RENAME COLUMN balance_min TO _public_balance_min
73+
settings mutations_sync=2, alter_sync=1;
74+
-- +goose StatementEnd
75+
-- +goose StatementBegin
76+
ALTER TABLE _final_validator_dashboard_data_monthly
77+
RENAME COLUMN _legacy_epoch_start TO epoch_start,
78+
RENAME COLUMN _legacy_balance_start TO balance_start,
79+
RENAME COLUMN _legacy_balance_min TO balance_min,
80+
RENAME COLUMN epoch_start TO _public_epoch_start,
81+
RENAME COLUMN balance_start TO _public_balance_start,
82+
RENAME COLUMN balance_min TO _public_balance_min
83+
settings mutations_sync=2, alter_sync=1;
84+
-- +goose StatementEnd

0 commit comments

Comments
 (0)