Skip to content

Commit d1cc92f

Browse files
mscheadmschead
committed
fix: indexes create/remove process for backfill
1 parent 6f09f6b commit d1cc92f

File tree

2 files changed

+125
-105
lines changed

2 files changed

+125
-105
lines changed

‎backfill/recreate-indexes/recreate-indexes.go‎

Lines changed: 66 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -27,69 +27,91 @@ func main() {
2727

2828
log.Println("Connected to database")
2929

30+
tx, err := db.Begin()
31+
if err != nil {
32+
log.Fatalf("Failed to begin transaction: %v", err)
33+
}
34+
3035
// SQL commands grouped by table
3136
commands := []struct {
3237
description string
3338
query string
3439
}{
35-
// // CREATE INDEX commands for Balances
36-
// {"Create unique index on Balances", `CREATE UNIQUE INDEX "balances_unique_constraint" ON "Balances" (network, "chainId", account, qualname, "tokenId");`},
37-
// {"Create account index on Balances", `CREATE INDEX "balances_account_index" ON "Balances" (account);`},
38-
// {"Create tokenId index on Balances", `CREATE INDEX "balances_tokenid_index" ON "Balances" ("tokenId");`},
39-
// {"Create contractId index on Balances", `CREATE INDEX "balances_contractid_index" ON "Balances" ("contractId");`},
40-
// {"Create search index on Balances", `CREATE INDEX "balances_search_idx" ON "Balances" (LOWER(account));`},
41-
42-
// // CREATE INDEX commands for Blocks
43-
// {"Create height index on Blocks", `CREATE INDEX "blocks_height_idx" ON "Blocks" (height);`},
44-
// {"Create chainId height index on Blocks", `CREATE INDEX "blocks_chainid_height_idx" ON "Blocks" ("chainId", height);`},
45-
// {"Create chainId index on Blocks", `CREATE INDEX "blocks_chainid_idx" ON "Blocks" ("chainId");`},
46-
// {"Create canonical index on Blocks", `CREATE INDEX "blocks_canonical_idx" ON "Blocks" (canonical);`},
47-
// {"Create GIN trgm parent index on Blocks", `CREATE INDEX "blocks_trgm_parent_idx" ON "Blocks" USING gin (LOWER(parent) gin_trgm_ops);`},
48-
49-
// // CREATE INDEX commands for Contracts
50-
// {"Create unique index on Contracts", `CREATE UNIQUE INDEX "contract_unique_constraint" ON "Contracts" (network, "chainId", module, "tokenId");`},
51-
// {"Create search index on Contracts", `CREATE INDEX "contracts_search_idx" ON "Contracts" (LOWER(module));`},
52-
53-
// // CREATE INDEX commands for Events
54-
// {"Create transactionId index on Events", `CREATE INDEX "events_transactionid_idx" ON "Events" ("transactionId");`},
55-
56-
// CREATE INDEX commands for Signers
57-
{"Create publicKey transactionId index on Signers", `CREATE INDEX "signers_pubkey_transactionid_idx" ON "Signers" (pubkey, "transactionId");`},
58-
59-
// CREATE INDEX commands for Transactions
60-
{"Create requestkey index on Transactions", `CREATE INDEX "transactions_requestkey_idx" ON "Transactions" (requestkey);`},
61-
{"Create sender index on Transactions", `CREATE INDEX "transactions_sender_idx" ON "Transactions" (sender);`},
62-
{"Create chainId blockId index on Transactions", `CREATE INDEX "transactions_chainid_blockid_idx" ON "Transactions" ("chainId", "blockId");`},
63-
{"Create hash index on Transactions", `CREATE INDEX "transactions_hash_idx" ON "Transactions" (hash);`},
64-
{"Create canonical index on Transactions", `CREATE INDEX "transactions_canonical_idx" ON "Transactions" (canonical);`},
65-
{"Create GIN trgm requestkey index on Transactions", `CREATE INDEX "transactions_trgm_requestkey_idx" ON "Transactions" USING gin (LOWER(requestkey) gin_trgm_ops);`},
66-
{"Create GIN trgm hash index on Transactions", `CREATE INDEX "transactions_trgm_hash_idx" ON "Transactions" USING gin (LOWER(hash) gin_trgm_ops);`},
67-
{"Create GIN trgm txid index on Transactions", `CREATE INDEX "transactions_trgm_txid_idx" ON "Transactions" USING gin (LOWER(txid) gin_trgm_ops);`},
68-
{"Create GIN trgm pactid index on Transactions", `CREATE INDEX "transactions_trgm_pactid_idx" ON "Transactions" USING gin (LOWER(pactid) gin_trgm_ops);`},
69-
{"Create GIN trgm sender index on Transactions", `CREATE INDEX "transactions_trgm_sender_idx" ON "Transactions" USING gin (LOWER(sender) gin_trgm_ops);`},
70-
71-
// CREATE INDEX commands for Transfers
72-
{"Create contractId index on Transfers", `CREATE INDEX "transfers_contractid_idx" ON "Transfers" ("contractId");`},
73-
{"Create type index on Transfers", `CREATE INDEX "transfers_type_idx" ON "Transfers" (type);`},
74-
{"Create transactionId index on Transfers", `CREATE INDEX "transfers_transactionid_idx" ON "Transfers" ("transactionId");`},
75-
{"Create modulename index on Transfers", `CREATE INDEX "transfers_modulename_idx" ON "Transfers" (modulename);`},
76-
{"Create from_acct modulename index on Transfers", `CREATE INDEX "transfers_from_acct_modulename_idx" ON "Transfers" (from_acct, modulename);`},
40+
{"Create unique index on Balances", `CREATE UNIQUE INDEX "Balances_pkey" ON public."Balances" USING btree (id);`},
41+
{"Create index on account column of Balances", `CREATE INDEX balances_account_index ON public."Balances" USING btree (account);`},
42+
{"Create index on contractId column of Balances", `CREATE INDEX balances_contractid_index ON public."Balances" USING btree ("contractId");`},
43+
{"Create index on lower(account) column of Balances", `CREATE INDEX balances_search_idx ON public."Balances" USING btree (lower((account)::text));`},
44+
{"Create index on tokenId column of Balances", `CREATE INDEX balances_tokenid_index ON public."Balances" USING btree ("tokenId");`},
45+
{"Create unique constraint on Balances", `CREATE UNIQUE INDEX balances_unique_constraint ON public."Balances" USING btree ("chainId", account, module, "tokenId");`},
46+
{"Create unique index on Blocks", `CREATE UNIQUE INDEX "Blocks_pkey" ON public."Blocks" USING btree (id);`},
47+
{"Create index on canonical column of Blocks", `CREATE INDEX blocks_canonical_idx ON public."Blocks" USING btree (canonical);`},
48+
{"Create index on chainId, height columns of Blocks", `CREATE INDEX blocks_chainid_height_idx ON public."Blocks" USING btree ("chainId", height);`},
49+
{"Create index on chainId column of Blocks", `CREATE INDEX blocks_chainid_idx ON public."Blocks" USING btree ("chainId");`},
50+
{"Create unique index on Blocks chainwebVersion, chainId, hash", `CREATE UNIQUE INDEX "blocks_chainwebVersion_chainid_hash_unique_idx" ON public."Blocks" USING btree ("chainwebVersion", "chainId", hash);`},
51+
{"Create index on hash column of Blocks", `CREATE INDEX blocks_hash_idx ON public."Blocks" USING btree (hash);`},
52+
{"Create index on height, id columns of Blocks", `CREATE INDEX blocks_height_id_idx ON public."Blocks" USING btree (height, id);`},
53+
{"Create index on height column of Blocks", `CREATE INDEX blocks_height_idx ON public."Blocks" USING btree (height);`},
54+
{"Create GIN index on parent column of Blocks", `CREATE INDEX blocks_trgm_parent_idx ON public."Blocks" USING gin (lower((parent)::text));`},
55+
{"Create unique index on Contracts", `CREATE UNIQUE INDEX "Contracts_pkey" ON public."Contracts" USING btree (id);`},
56+
{"Create unique constraint on Contracts", `CREATE UNIQUE INDEX contract_unique_constraint ON public."Contracts" USING btree ("chainId", module, "tokenId");`},
57+
{"Create index on Contracts module", `CREATE INDEX contracts_search_idx ON public."Contracts" USING btree (lower((module)::text));`},
58+
{"Create unique index on Events", `CREATE UNIQUE INDEX "Events_pkey" ON public."Events" USING btree (id);`},
59+
{"Create index on Events module, name", `CREATE INDEX events_module_name_idx ON public."Events" USING btree (module, name);`},
60+
{"Create index on transactionId column of Events", `CREATE INDEX events_transactionid_idx ON public."Events" USING btree ("transactionId");`},
61+
{"Create unique index on Guards", `CREATE UNIQUE INDEX "Guards_pkey" ON public."Guards" USING btree (id);`},
62+
{"Create unique index on Guards publicKey, predicate, balanceId", `CREATE UNIQUE INDEX guards_publickey_predicate_balanceid_idx ON public."Guards" USING btree ("publicKey", predicate, "balanceId");`},
63+
{"Create unique index on Signers", `CREATE UNIQUE INDEX "Signers_pkey" ON public."Signers" USING btree (id);`},
64+
{"Create index on pubkey column of Signers", `CREATE INDEX signers_pubkey_idx ON public."Signers" USING btree (pubkey);`},
65+
{"Create index on pubkey, transactionId columns of Signers", `CREATE INDEX signers_pubkey_transactionid_idx ON public."Signers" USING btree (pubkey, "transactionId");`},
66+
{"Create index on transactionId column of Signers", `CREATE INDEX signers_transaction_id_idx ON public."Signers" USING btree ("transactionId");`},
67+
{"Create unique index on Transactions", `CREATE UNIQUE INDEX "Transactions_pkey" ON public."Transactions" USING btree (id);`},
68+
{"Create index on blockId column of Transactions", `CREATE INDEX "transactions_blockId_idx" ON public."Transactions" USING btree ("blockId");`},
69+
{"Create index on canonical column of Transactions", `CREATE INDEX transactions_canonical_idx ON public."Transactions" USING btree (canonical);`},
70+
{"Create index on chainId column of Transactions", `CREATE INDEX "transactions_chainId_idx" ON public."Transactions" USING btree ("chainId");`},
71+
{"Create index on chainId, blockId columns of Transactions", `CREATE INDEX transactions_chainid_blockid_idx ON public."Transactions" USING btree ("chainId", "blockId");`},
72+
{"Create index on hash column of Transactions", `CREATE INDEX transactions_hash_idx ON public."Transactions" USING btree (hash);`},
73+
{"Create index on requestkey column of Transactions", `CREATE INDEX transactions_requestkey_idx ON public."Transactions" USING btree (requestkey);`},
74+
{"Create index on sender, id columns of Transactions", `CREATE INDEX transactions_sender_id_idx ON public."Transactions" USING btree (sender, id);`},
75+
{"Create index on sender column of Transactions", `CREATE INDEX transactions_sender_idx ON public."Transactions" USING btree (sender);`},
76+
{"Create GIN index on hash column of Transactions", `CREATE INDEX transactions_trgm_hash_idx ON public."Transactions" USING gin (lower((hash)::text));`},
77+
{"Create GIN index on pactid column of Transactions", `CREATE INDEX transactions_trgm_pactid_idx ON public."Transactions" USING gin (lower((pactid)::text));`},
78+
{"Create GIN index on requestkey column of Transactions", `CREATE INDEX transactions_trgm_requestkey_idx ON public."Transactions" USING gin (lower((requestkey)::text));`},
79+
{"Create GIN index on sender column of Transactions", `CREATE INDEX transactions_trgm_sender_idx ON public."Transactions" USING gin (lower((sender)::text));`},
80+
{"Create GIN index on txid column of Transactions", `CREATE INDEX transactions_trgm_txid_idx ON public."Transactions" USING gin (lower((txid)::text));`},
81+
{"Create unique index on Transfers", `CREATE UNIQUE INDEX "Transfers_pkey" ON public."Transfers" USING btree (id);`},
82+
{"Create index on from_acct column of Transfers", `CREATE INDEX from_acct_idx ON public."Transfers" USING btree (from_acct);`},
83+
{"Create index on to_acct column of Transfers", `CREATE INDEX to_acct_idx ON public."Transfers" USING btree (to_acct);`},
84+
{"Create index on chainId, from_acct, modulename columns of Transfers", `CREATE INDEX transfers_chainid_from_acct_modulename_idx ON public."Transfers" USING btree ("chainId", from_acct, modulename);`},
85+
{"Create index on chainId, to_acct, modulename columns of Transfers", `CREATE INDEX transfers_chainid_to_acct_modulename_idx ON public."Transfers" USING btree ("chainId", to_acct, modulename);`},
86+
{"Create index on contractId column of Transfers", `CREATE INDEX transfers_contractid_idx ON public."Transfers" USING btree ("contractId");`},
87+
{"Create index on from_acct, modulename columns of Transfers", `CREATE INDEX transfers_from_acct_modulename_idx ON public."Transfers" USING btree (from_acct, modulename);`},
88+
{"Create index on hasTokenId column of Transfers", `CREATE INDEX "transfers_hasTokenId_idx" ON public."Transfers" USING btree ("hasTokenId");`},
89+
{"Create index on modulename column of Transfers", `CREATE INDEX transfers_modulename_idx ON public."Transfers" USING btree (modulename);`},
90+
{"Create index on transactionId column of Transfers", `CREATE INDEX transfers_transactionid_idx ON public."Transfers" USING btree ("transactionId");`},
91+
{"Create index on type column of Transfers", `CREATE INDEX transfers_type_idx ON public."Transfers" USING btree (type);`},
7792
}
7893

7994
for i, cmd := range commands {
8095
log.Printf("Executing (%d/%d): %s", i+1, len(commands), cmd.description)
8196

8297
start := time.Now()
83-
_, err := db.Exec(cmd.query)
98+
_, err := tx.Exec(cmd.query)
8499
elapsed := time.Since(start)
85100

86101
if err != nil {
87102
log.Printf("Error executing (%d): %s - %v", i+1, cmd.description, err)
88-
break // Optionally exit on error or log and continue
103+
tx.Rollback()
104+
log.Println("Transaction rolled back due to error")
105+
return
89106
} else {
90107
log.Printf("Successfully executed (%d): %s in %v", i+1, cmd.description, elapsed)
91108
}
92109
}
93110

111+
// Commit transaction
112+
if err := tx.Commit(); err != nil {
113+
log.Fatalf("Failed to commit transaction: %v", err)
114+
}
115+
94116
log.Println("All commands executed")
95117
}

‎backfill/sqls/before-backfill.sql‎

Lines changed: 59 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -1,73 +1,71 @@
1-
-- Drop indexes from the Balances table
2-
DROP INDEX IF EXISTS "balances_unique_constraint";
3-
DROP INDEX IF EXISTS "balances_account_index";
4-
DROP INDEX IF EXISTS "balances_tokenid_index";
5-
DROP INDEX IF EXISTS "balances_contractid_index";
6-
DROP INDEX IF EXISTS "balances_search_idx";
1+
BEGIN;
72

8-
-- Drop indexes from the Blocks table
9-
DROP INDEX IF EXISTS "blocks_chainwebVersion_chainid_hash_unique_idx";
10-
DROP INDEX IF EXISTS "blocks_height_idx";
11-
DROP INDEX IF EXISTS "blocks_chainid_height_idx";
12-
DROP INDEX IF EXISTS "blocks_chainid_idx";
13-
DROP INDEX IF EXISTS "blocks_canonical_idx";
14-
DROP INDEX IF EXISTS "blocks_trgm_parent_idx";
3+
ALTER TABLE "Guards" DROP CONSTRAINT IF EXISTS "Guards_pkey";
4+
DROP INDEX IF EXISTS "Guards_pkey";
5+
DROP INDEX IF EXISTS guards_publickey_predicate_balanceid_idx;
156

16-
-- Drop indexes from the Contracts table
17-
DROP INDEX IF EXISTS "contract_unique_constraint";
18-
DROP INDEX IF EXISTS "contracts_search_idx";
7+
ALTER TABLE "Balances" DROP CONSTRAINT IF EXISTS "Balances_pkey" CASCADE;
8+
DROP INDEX IF EXISTS "Balances_pkey";
9+
DROP INDEX IF EXISTS balances_account_index;
10+
DROP INDEX IF EXISTS balances_contractid_index;
11+
DROP INDEX IF EXISTS balances_search_idx;
12+
DROP INDEX IF EXISTS balances_tokenid_index;
13+
DROP INDEX IF EXISTS balances_unique_constraint;
1914

20-
-- Drop indexes from the Events table
21-
DROP INDEX IF EXISTS "events_transactionid_idx";
15+
ALTER TABLE "Contracts" DROP CONSTRAINT IF EXISTS "Contracts_pkey" CASCADE;
16+
DROP INDEX IF EXISTS "Contracts_pkey";
17+
DROP INDEX IF EXISTS contract_unique_constraint;
18+
DROP INDEX IF EXISTS contracts_search_idx;
2219

23-
-- Drop indexes from the Guards table
24-
DROP INDEX IF EXISTS "signers_pubkey_transactionid_idx";
20+
ALTER TABLE "Events" DROP CONSTRAINT IF EXISTS "Events_pkey";
21+
DROP INDEX IF EXISTS "Events_pkey";
22+
DROP INDEX IF EXISTS events_module_name_idx;
23+
DROP INDEX IF EXISTS events_transactionid_idx;
2524

26-
-- Drop indexes from the Signers table
27-
DROP INDEX IF EXISTS "signers_pubkey_transactionid_idx";
25+
ALTER TABLE "Signers" DROP CONSTRAINT IF EXISTS "Signers_pkey";
26+
DROP INDEX IF EXISTS "Signers_pkey";
27+
DROP INDEX IF EXISTS signers_pubkey_idx;
28+
DROP INDEX IF EXISTS signers_pubkey_transactionid_idx;
29+
DROP INDEX IF EXISTS signers_transaction_id_idx;
2830

29-
-- Drop indexes from the Transactions table
30-
DROP INDEX IF EXISTS "transactions_requestkey_idx";
31+
ALTER TABLE "Transactions" DROP CONSTRAINT IF EXISTS "Transactions_pkey" CASCADE;
32+
DROP INDEX IF EXISTS "Transactions_pkey";
3133
DROP INDEX IF EXISTS "transactions_blockId_idx";
32-
DROP INDEX IF EXISTS "transactions_sender_idx";
34+
DROP INDEX IF EXISTS transactions_canonical_idx;
3335
DROP INDEX IF EXISTS "transactions_chainId_idx";
34-
DROP INDEX IF EXISTS "transactions_chainid_blockid_idx";
35-
DROP INDEX IF EXISTS "transactions_hash_idx";
36-
DROP INDEX IF EXISTS "transactions_canonical_idx";
37-
DROP INDEX IF EXISTS "transactions_trgm_requestkey_idx";
38-
DROP INDEX IF EXISTS "transactions_trgm_hash_idx";
39-
DROP INDEX IF EXISTS "transactions_trgm_txid_idx";
40-
DROP INDEX IF EXISTS "transactions_trgm_pactid_idx";
41-
DROP INDEX IF EXISTS "transactions_trgm_sender_idx";
36+
DROP INDEX IF EXISTS transactions_chainid_blockid_idx;
37+
DROP INDEX IF EXISTS transactions_hash_idx;
38+
DROP INDEX IF EXISTS transactions_requestkey_idx;
39+
DROP INDEX IF EXISTS transactions_sender_id_idx;
40+
DROP INDEX IF EXISTS transactions_sender_idx;
41+
DROP INDEX IF EXISTS transactions_trgm_hash_idx;
42+
DROP INDEX IF EXISTS transactions_trgm_pactid_idx;
43+
DROP INDEX IF EXISTS transactions_trgm_requestkey_idx;
44+
DROP INDEX IF EXISTS transactions_trgm_sender_idx;
45+
DROP INDEX IF EXISTS transactions_trgm_txid_idx;
4246

43-
-- Drop indexes from the Transfers table
44-
DROP INDEX IF EXISTS "transfers_type_idx";
45-
DROP INDEX IF EXISTS "transfers_transactionid_idx";
47+
ALTER TABLE "Transfers" DROP CONSTRAINT IF EXISTS "Transfers_pkey";
48+
DROP INDEX IF EXISTS "Transfers_pkey";
49+
DROP INDEX IF EXISTS from_acct_idx;
50+
DROP INDEX IF EXISTS to_acct_idx;
51+
DROP INDEX IF EXISTS transfers_chainid_from_acct_modulename_idx;
52+
DROP INDEX IF EXISTS transfers_chainid_to_acct_modulename_idx;
53+
DROP INDEX IF EXISTS transfers_contractid_idx;
54+
DROP INDEX IF EXISTS transfers_from_acct_modulename_idx;
4655
DROP INDEX IF EXISTS "transfers_hasTokenId_idx";
47-
DROP INDEX IF EXISTS "transfers_contractid_idx";
48-
DROP INDEX IF EXISTS "transfers_modulename_idx";
49-
DROP INDEX IF EXISTS "transfers_from_acct_modulename_idx";
50-
51-
-- Drop constraints from the Balances table
52-
ALTER TABLE "Balances" DROP CONSTRAINT IF EXISTS "Balances_pkey";
53-
54-
-- Drop constraints from the Blocks table
55-
ALTER TABLE "Blocks" DROP CONSTRAINT IF EXISTS "Blocks_pkey";
56-
57-
-- Drop constraints from the Contracts table
58-
ALTER TABLE "Contracts" DROP CONSTRAINT IF EXISTS "Contracts_pkey";
56+
DROP INDEX IF EXISTS transfers_modulename_idx;
57+
DROP INDEX IF EXISTS transfers_transactionid_idx;
58+
DROP INDEX IF EXISTS transfers_type_idx;
5959

60-
-- Drop constraints from the Events table
61-
ALTER TABLE "Events" DROP CONSTRAINT IF EXISTS "Events_pkey";
62-
63-
-- Drop constraints from the Guards table
64-
ALTER TABLE "Guards" DROP CONSTRAINT IF EXISTS "Guards_pkey";
65-
66-
-- Drop constraints from the Signers table
67-
ALTER TABLE "Signers" DROP CONSTRAINT IF EXISTS "Signers_pkey";
68-
69-
-- Drop constraints from the Transactions table
70-
ALTER TABLE "Transactions" DROP CONSTRAINT IF EXISTS "Transactions_pkey";
60+
ALTER TABLE "Blocks" DROP CONSTRAINT IF EXISTS "Blocks_pkey" CASCADE;
61+
DROP INDEX IF EXISTS "Blocks_pkey";
62+
DROP INDEX IF EXISTS blocks_canonical_idx;
63+
DROP INDEX IF EXISTS blocks_chainid_height_idx;
64+
DROP INDEX IF EXISTS blocks_chainid_idx;
65+
DROP INDEX IF EXISTS "blocks_chainwebVersion_chainid_hash_unique_idx";
66+
DROP INDEX IF EXISTS blocks_hash_idx;
67+
DROP INDEX IF EXISTS blocks_height_id_idx;
68+
DROP INDEX IF EXISTS blocks_height_idx;
69+
DROP INDEX IF EXISTS blocks_trgm_parent_idx;
7170

72-
-- Drop constraints from the Transfers table
73-
ALTER TABLE "Transfers" DROP CONSTRAINT IF EXISTS "Transfers_pkey";
71+
COMMIT;

0 commit comments

Comments
 (0)