Skip to content

Commit 5f0e575

Browse files
kobayuriikobayurii
committed
add new migrate script
1 parent 5728b89 commit 5f0e575

File tree

6 files changed

+277
-0
lines changed

6 files changed

+277
-0
lines changed

‎database/migrate_compact_with_state/migrate_access_keys.sh

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting compact for partition state_changes_access_key_$partition at $start_time"
10+
echo "[INFO] Starting compact for partition state_changes_access_key_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
WITH ordered_data AS (
14+
SELECT
15+
account_id,
16+
data_key,
17+
data_value,
18+
block_height AS block_height_from,
19+
LAG(block_height) OVER (
20+
PARTITION BY account_id, data_key
21+
ORDER BY block_height DESC
22+
) AS block_height_to
23+
FROM state_changes_access_key_1300_$partition
24+
),
25+
insert_1300_compact AS (
26+
INSERT INTO state_changes_access_key_1300_compact_$partition (
27+
account_id, data_key, data_value, block_height_from, block_height_to
28+
)
29+
SELECT
30+
account_id, data_key, data_value, block_height_from, block_height_to
31+
FROM ordered_data
32+
WHERE data_value IS NOT NULL
33+
ON CONFLICT (account_id, data_key, block_height_from) DO NOTHING
34+
)
35+
INSERT INTO state_changes_access_key_1305_$partition (
36+
account_id, block_height, data_key, data_value
37+
)
38+
SELECT
39+
account_id,
40+
block_height_from AS block_height,
41+
data_key,
42+
data_value
43+
FROM ordered_data
44+
WHERE data_value IS NOT NULL
45+
AND block_height_from <= 130500000
46+
AND (block_height_to IS NULL OR block_height_to > 130500000)
47+
ON CONFLICT (account_id, data_key, block_height) DO NOTHING;
48+
" 2>&1 | tee -a "$LOG_FILE"
49+
50+
# shellcheck disable=SC2155
51+
local end_time=$(date +"%T")
52+
echo "[INFO] Finished compact for partition state_changes_access_key_$partition at $end_time"
53+
echo "[INFO] Finished compact for partition state_changes_access_key_$partition at $end_time" >> "$LOG_FILE"
54+
}
55+
56+
# Run migrations in parallel for partitions 0 to 99
57+
for i in $(seq 0 99); do
58+
migrate_partition "$i" &
59+
done
60+
61+
# Wait for all background jobs to finish
62+
wait

‎database/migrate_compact_with_state/migrate_accounts.sh

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting compact for partition state_changes_account_1300_$partition at $start_time"
10+
echo "[INFO] Starting compact for partition state_changes_account_1300_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
WITH ordered_data AS (
14+
SELECT
15+
account_id,
16+
data_value,
17+
block_height AS block_height_from,
18+
LAG(block_height) OVER (
19+
PARTITION BY account_id
20+
ORDER BY block_height DESC
21+
) AS block_height_to
22+
FROM state_changes_account_1300_$partition
23+
),
24+
insert_1300_compact AS (
25+
INSERT INTO state_changes_account_1300_compact_$partition (
26+
account_id, data_value, block_height_from, block_height_to
27+
)
28+
SELECT
29+
account_id, data_value, block_height_from, block_height_to
30+
FROM ordered_data
31+
WHERE data_value IS NOT NULL
32+
ON CONFLICT (account_id, block_height_from) DO NOTHING
33+
)
34+
INSERT INTO state_changes_account_1305_$partition (
35+
account_id, block_height, data_value
36+
)
37+
SELECT
38+
account_id,
39+
block_height_from AS block_height,
40+
data_value
41+
FROM ordered_data
42+
WHERE data_value IS NOT NULL
43+
AND block_height_from <= 130500000
44+
AND (block_height_to IS NULL OR block_height_to > 130500000)
45+
ON CONFLICT (account_id, block_height) DO NOTHING;
46+
" 2>&1 | tee -a "$LOG_FILE"
47+
48+
# shellcheck disable=SC2155
49+
local end_time=$(date +"%T")
50+
echo "[INFO] Finished compact for partition state_changes_account_1300_$partition at $end_time"
51+
echo "[INFO] Finished compact for partition state_changes_account_1300_$partition at $end_time" >> "$LOG_FILE"
52+
}
53+
54+
# Run migrations in parallel for partitions 0 to 99
55+
for i in $(seq 0 99); do
56+
migrate_partition "$i" &
57+
done
58+
59+
# Wait for all background jobs to finish
60+
wait

‎database/migrate_compact_with_state/migrate_contracts.sh

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting compact for partition state_changes_contract_$partition at $start_time"
10+
echo "[INFO] Starting compact for partition state_changes_contract_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
WITH ordered_data AS (
14+
SELECT
15+
account_id,
16+
data_value,
17+
block_height AS block_height_from,
18+
LAG(block_height) OVER (
19+
PARTITION BY account_id
20+
ORDER BY block_height DESC
21+
) AS block_height_to
22+
FROM state_changes_contract_1300_$partition
23+
),
24+
insert_1300_compact AS (
25+
INSERT INTO state_changes_contract_1300_compact_$partition (
26+
account_id, data_value, block_height_from, block_height_to
27+
)
28+
SELECT
29+
account_id, data_value, block_height_from, block_height_to
30+
FROM ordered_data
31+
WHERE data_value IS NOT NULL
32+
ON CONFLICT (account_id, block_height_from) DO NOTHING
33+
)
34+
INSERT INTO state_changes_contract_1305_$partition (
35+
account_id, block_height, data_value
36+
)
37+
SELECT
38+
account_id,
39+
block_height_from AS block_height,
40+
data_value
41+
FROM ordered_data
42+
WHERE data_value IS NOT NULL
43+
AND block_height_from <= 130500000
44+
AND (block_height_to IS NULL OR block_height_to > 130500000)
45+
ON CONFLICT (account_id, block_height) DO NOTHING;
46+
" 2>&1 | tee -a "$LOG_FILE"
47+
48+
# shellcheck disable=SC2155
49+
local end_time=$(date +"%T")
50+
echo "[INFO] Finished compact for partition state_changes_contract_$partition at $end_time"
51+
echo "[INFO] Finished compact for partition state_changes_contract_$partition at $end_time" >> "$LOG_FILE"
52+
}
53+
54+
# Run migrations in parallel for partitions 0 to 99
55+
for i in $(seq 0 99); do
56+
migrate_partition "$i" &
57+
done
58+
59+
# Wait for all background jobs to finish
60+
wait

‎database/migrate_compact_with_state/migrate_state_changes.sh

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting compact for partition state_changes_data_$partition at $start_time"
10+
echo "[INFO] Starting compact for partition state_changes_data_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
WITH ordered_data AS (
14+
SELECT
15+
account_id,
16+
data_key,
17+
data_value,
18+
block_height AS block_height_from,
19+
LAG(block_height) OVER (
20+
PARTITION BY account_id, data_key
21+
ORDER BY block_height DESC
22+
) AS block_height_to
23+
FROM state_changes_data_1300_$partition
24+
),
25+
insert_1300_compact AS (
26+
INSERT INTO state_changes_data_1300_compact_$partition (
27+
account_id, data_key, data_value, block_height_from, block_height_to
28+
)
29+
SELECT
30+
account_id, data_key, data_value, block_height_from, block_height_to
31+
FROM ordered_data
32+
WHERE data_value IS NOT NULL
33+
ON CONFLICT (account_id, data_key, block_height_from) DO NOTHING
34+
)
35+
INSERT INTO state_changes_data_1305_$partition (
36+
account_id, block_height, data_key, data_value
37+
)
38+
SELECT
39+
account_id,
40+
block_height_from AS block_height,
41+
data_key,
42+
data_value
43+
FROM ordered_data
44+
WHERE data_value IS NOT NULL
45+
AND block_height_from <= 130500000
46+
AND (block_height_to IS NULL OR block_height_to > 130500000)
47+
ON CONFLICT (account_id, data_key, block_height) DO NOTHING;
48+
" 2>&1 | tee -a "$LOG_FILE"
49+
50+
# shellcheck disable=SC2155
51+
local end_time=$(date +"%T")
52+
echo "[INFO] Finished compact for partition state_changes_data_$partition at $end_time"
53+
echo "[INFO] Finished compact for partition state_changes_data_$partition at $end_time" >> "$LOG_FILE"
54+
}
55+
56+
# Run migrations in parallel for partitions 0 to 99
57+
for i in $(seq 0 99); do
58+
migrate_partition "$i" &
59+
done
60+
61+
# Wait for all background jobs to finish
62+
wait

‎database/migrate_compact_with_state/shard_migration.sh

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
3+
# Set database credentials
4+
export PGPASSWORD="S8TSz8vz4sQ80D7ycWSnT5B4"
5+
export DB_NAME="shard0"
6+
export DB_USER="readrpc"
7+
export DB_HOST="localhost"
8+
export DB_PORT=5432
9+
10+
# Set log file
11+
export LOG_FILE="migration0.log"
12+
# Remove old log file if it exists
13+
rm -f "$LOG_FILE"
14+
touch "$LOG_FILE"
15+
16+
echo "Starting compact at $(date)" | tee -a "$LOG_FILE"
17+
18+
./migrate_access_keys.sh &
19+
./migrate_accounts.sh &
20+
./migrate_contracts.sh &
21+
./migrate_state_changes.sh &
22+
23+
wait
24+
25+
echo "Compact completed at $(date)" | tee -a "$LOG_FILE"

‎database/sql_databases/drop.sql

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
drop table state_changes_contract_1305;
2+
drop table state_changes_account_1305;
3+
drop table state_changes_access_key_1305;
4+
drop table state_changes_data_1305;
5+
drop table state_changes_access_key_1300_compact;
6+
drop table state_changes_contract_1300_compact;
7+
drop table state_changes_account_1300_compact;
8+
drop table state_changes_data_1300_compact;

0 commit comments

Comments
 (0)