Skip to content

Commit 5728b89

Browse files
kobayuriikobayurii
committed
save testing migration data scripts
1 parent 83916cf commit 5728b89

22 files changed

+1045
-0
lines changed

‎database/migrate_compact/migrate_access_keys.sh

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting compact for partition state_changes_access_key_$partition at $start_time"
10+
echo "[INFO] Starting compact for partition state_changes_access_key_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
WITH ordered_data AS (
14+
SELECT
15+
account_id,
16+
data_key,
17+
data_value,
18+
block_height AS block_height_from,
19+
LAG(block_height) OVER (PARTITION BY account_id, data_key ORDER BY block_height DESC) AS block_height_to
20+
FROM state_changes_access_key_1300_$partition
21+
)
22+
INSERT INTO state_changes_access_key_1300_compact_$partition (account_id, data_key, data_value, block_height_from, block_height_to)
23+
SELECT
24+
account_id,
25+
data_key,
26+
data_value,
27+
block_height_from,
28+
block_height_to
29+
FROM ordered_data
30+
WHERE data_value IS NOT NULL
31+
ON CONFLICT (account_id, data_key, block_height_from) DO NOTHING;
32+
" 2>&1 | tee -a "$LOG_FILE"
33+
34+
# shellcheck disable=SC2155
35+
local end_time=$(date +"%T")
36+
echo "[INFO] Finished compact for partition state_changes_access_key_$partition at $end_time"
37+
echo "[INFO] Finished compact for partition state_changes_access_key_$partition at $end_time" >> "$LOG_FILE"
38+
}
39+
40+
# Run migrations in parallel for partitions 0 to 99
41+
for i in $(seq 0 99); do
42+
migrate_partition "$i" &
43+
done
44+
45+
# Wait for all background jobs to finish
46+
wait

‎database/migrate_compact/migrate_accounts.sh

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting compact for partition state_changes_account_1300_$partition at $start_time"
10+
echo "[INFO] Starting compact for partition state_changes_account_1300_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
WITH ordered_data AS (
14+
SELECT
15+
account_id,
16+
data_value,
17+
block_height AS block_height_from,
18+
LAG(block_height) OVER (PARTITION BY account_id ORDER BY block_height DESC) AS block_height_to
19+
FROM state_changes_account_1300_$partition
20+
)
21+
INSERT INTO state_changes_account_1300_compact_$partition (account_id, data_value, block_height_from, block_height_to)
22+
SELECT
23+
account_id,
24+
data_value,
25+
block_height_from,
26+
block_height_to
27+
FROM ordered_data
28+
WHERE data_value IS NOT NULL
29+
ON CONFLICT (account_id, block_height_from) DO NOTHING;
30+
" 2>&1 | tee -a "$LOG_FILE"
31+
32+
# shellcheck disable=SC2155
33+
local end_time=$(date +"%T")
34+
echo "[INFO] Finished compact for partition state_changes_account_1300_$partition at $end_time"
35+
echo "[INFO] Finished compact for partition state_changes_account_1300_$partition at $end_time" >> "$LOG_FILE"
36+
}
37+
38+
# Run migrations in parallel for partitions 0 to 99
39+
for i in $(seq 0 99); do
40+
migrate_partition "$i" &
41+
done
42+
43+
# Wait for all background jobs to finish
44+
wait

‎database/migrate_compact/migrate_contracts.sh

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting compact for partition state_changes_contract_$partition at $start_time"
10+
echo "[INFO] Starting compact for partition state_changes_contract_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
WITH ordered_data AS (
14+
SELECT
15+
account_id,
16+
data_value,
17+
block_height AS block_height_from,
18+
LAG(block_height) OVER (PARTITION BY account_id ORDER BY block_height DESC) AS block_height_to
19+
FROM state_changes_contract_1300_$partition
20+
)
21+
INSERT INTO state_changes_contract_1300_compact_$partition (account_id, data_value, block_height_from, block_height_to)
22+
SELECT
23+
account_id,
24+
data_value,
25+
block_height_from,
26+
block_height_to
27+
FROM ordered_data
28+
WHERE data_value IS NOT NULL
29+
ON CONFLICT (account_id, block_height_from) DO NOTHING;
30+
" 2>&1 | tee -a "$LOG_FILE"
31+
32+
# shellcheck disable=SC2155
33+
local end_time=$(date +"%T")
34+
echo "[INFO] Finished compact for partition state_changes_contract_$partition at $end_time"
35+
echo "[INFO] Finished compact for partition state_changes_contract_$partition at $end_time" >> "$LOG_FILE"
36+
}
37+
38+
# Run migrations in parallel for partitions 0 to 99
39+
for i in $(seq 0 99); do
40+
migrate_partition "$i" &
41+
done
42+
43+
# Wait for all background jobs to finish
44+
wait

‎database/migrate_compact/migrate_state_changes.sh

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting compact for partition state_changes_data_$partition at $start_time"
10+
echo "[INFO] Starting compact for partition state_changes_data_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
WITH ordered_data AS (
14+
SELECT
15+
account_id,
16+
data_key,
17+
data_value,
18+
block_height AS block_height_from,
19+
LAG(block_height) OVER (PARTITION BY account_id, data_key ORDER BY block_height DESC) AS block_height_to
20+
FROM state_changes_data_1300_$partition
21+
)
22+
INSERT INTO state_changes_data_1300_compact_$partition (account_id, data_key, data_value, block_height_from, block_height_to)
23+
SELECT
24+
account_id,
25+
data_key,
26+
data_value,
27+
block_height_from,
28+
block_height_to
29+
FROM ordered_data
30+
WHERE data_value IS NOT NULL
31+
ON CONFLICT (account_id, data_key, block_height_from) DO NOTHING;
32+
" 2>&1 | tee -a "$LOG_FILE"
33+
34+
# shellcheck disable=SC2155
35+
local end_time=$(date +"%T")
36+
echo "[INFO] Finished compact for partition state_changes_data_$partition at $end_time"
37+
echo "[INFO] Finished compact for partition state_changes_data_$partition at $end_time" >> "$LOG_FILE"
38+
}
39+
40+
# Run migrations in parallel for partitions 0 to 99
41+
for i in $(seq 0 99); do
42+
migrate_partition "$i" &
43+
done
44+
45+
# Wait for all background jobs to finish
46+
wait

‎database/migrate_compact/shard_migration.sh

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
3+
# Set database credentials
4+
export PGPASSWORD="S8TSz8vz4sQ80D7ycWSnT5B4"
5+
export DB_NAME="shard5"
6+
export DB_USER="readrpc"
7+
export DB_HOST="localhost"
8+
export DB_PORT=5432
9+
10+
# Set log file
11+
export LOG_FILE="migration5.log"
12+
# Remove old log file if it exists
13+
rm -f "$LOG_FILE"
14+
touch "$LOG_FILE"
15+
16+
echo "Starting compact at $(date)" | tee -a "$LOG_FILE"
17+
18+
./migrate_access_keys.sh &
19+
./migrate_accounts.sh &
20+
./migrate_contracts.sh &
21+
./migrate_state_changes.sh &
22+
23+
wait
24+
25+
echo "Compact completed at $(date)" | tee -a "$LOG_FILE"

‎database/migrate_from_prev/migrate_access_keys.sh

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting migration for partition state_changes_access_key_$partition at $start_time"
10+
echo "[INFO] Starting migration for partition state_changes_access_key_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
INSERT INTO state_changes_access_key_1305_$partition (account_id, block_height, data_key, data_value)
14+
WITH latest_blocks AS (
15+
SELECT
16+
account_id,
17+
data_key,
18+
MAX(block_height) AS max_block_height
19+
FROM
20+
state_changes_access_key_1300_$partition
21+
WHERE
22+
block_height < 130500000
23+
GROUP BY
24+
account_id, data_key
25+
)
26+
SELECT
27+
sc.account_id,
28+
sc.block_height,
29+
sc.data_key,
30+
sc.data_value
31+
FROM
32+
state_changes_access_key_1300_$partition sc
33+
INNER JOIN latest_blocks lb
34+
ON
35+
sc.account_id = lb.account_id
36+
AND sc.data_key = lb.data_key
37+
AND sc.block_height = lb.max_block_height
38+
WHERE
39+
sc.data_value IS NOT NULL
40+
ON CONFLICT (account_id, data_key, block_height) DO NOTHING;
41+
" 2>&1 | tee -a "$LOG_FILE"
42+
43+
# shellcheck disable=SC2155
44+
local end_time=$(date +"%T")
45+
echo "[INFO] Finished migration for partition state_changes_access_key_$partition at $end_time"
46+
echo "[INFO] Finished migration for partition state_changes_access_key_$partition at $end_time" >> "$LOG_FILE"
47+
}
48+
49+
# Run migrations in parallel for partitions 0 to 99
50+
for i in $(seq 0 99); do
51+
migrate_partition "$i" &
52+
done
53+
54+
# Wait for all background jobs to finish
55+
wait

‎database/migrate_from_prev/migrate_accounts.sh

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting migration for partition state_changes_account_$partition at $start_time"
10+
echo "[INFO] Starting migration for partition state_changes_account_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
INSERT INTO state_changes_account_1305_$partition (account_id, block_height, data_value)
14+
WITH latest_blocks AS (
15+
SELECT
16+
account_id,
17+
MAX(block_height) AS max_block_height
18+
FROM
19+
state_changes_account_1300_$partition
20+
WHERE
21+
block_height < 130500000
22+
GROUP BY
23+
account_id
24+
)
25+
SELECT
26+
sc.account_id,
27+
sc.block_height,
28+
sc.data_value
29+
FROM
30+
state_changes_account_1300_$partition sc
31+
INNER JOIN latest_blocks lb
32+
ON
33+
sc.account_id = lb.account_id
34+
AND sc.block_height = lb.max_block_height
35+
WHERE
36+
sc.data_value IS NOT NULL
37+
ON CONFLICT (account_id, block_height) DO NOTHING;
38+
" 2>&1 | tee -a "$LOG_FILE"
39+
40+
# shellcheck disable=SC2155
41+
local end_time=$(date +"%T")
42+
echo "[INFO] Finished migration for partition state_changes_account_$partition at $end_time"
43+
echo "[INFO] Finished migration for partition state_changes_account_$partition at $end_time" >> "$LOG_FILE"
44+
}
45+
46+
# Run migrations in parallel for partitions 0 to 99
47+
for i in $(seq 0 99); do
48+
migrate_partition "$i" &
49+
done
50+
51+
# Wait for all background jobs to finish
52+
wait

‎database/migrate_from_prev/migrate_contracts.sh

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting migration for partition state_changes_contract_$partition at $start_time"
10+
echo "[INFO] Starting migration for partition state_changes_contract_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
INSERT INTO state_changes_contract_1305_$partition (account_id, block_height, data_value)
14+
WITH latest_blocks AS (
15+
SELECT
16+
account_id,
17+
MAX(block_height) AS max_block_height
18+
FROM
19+
state_changes_contract_1300_$partition
20+
WHERE
21+
block_height < 130500000
22+
GROUP BY
23+
account_id
24+
)
25+
SELECT
26+
sc.account_id,
27+
sc.block_height,
28+
sc.data_value
29+
FROM
30+
state_changes_contract_1300_$partition sc
31+
INNER JOIN latest_blocks lb
32+
ON
33+
sc.account_id = lb.account_id
34+
AND sc.block_height = lb.max_block_height
35+
WHERE
36+
sc.data_value IS NOT NULL
37+
ON CONFLICT (account_id, block_height) DO NOTHING;
38+
" 2>&1 | tee -a "$LOG_FILE"
39+
40+
# shellcheck disable=SC2155
41+
local end_time=$(date +"%T")
42+
echo "[INFO] Finished migration for partition state_changes_contract_$partition at $end_time"
43+
echo "[INFO] Finished migration for partition state_changes_contract_$partition at $end_time" >> "$LOG_FILE"
44+
}
45+
46+
# Run migrations in parallel for partitions 0 to 99
47+
for i in $(seq 0 99); do
48+
migrate_partition "$i" &
49+
done
50+
51+
# Wait for all background jobs to finish
52+
wait

0 commit comments

Comments
 (0)