Skip to content

Commit 23e5a95

Browse files
kobayuriikobayurii
committed
add new migrate script
1 parent 5728b89 commit 23e5a95

File tree

7 files changed

+369
-0
lines changed

7 files changed

+369
-0
lines changed

‎database/migrate_compact_with_state/migrate_access_keys.sh

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting compact for partition state_changes_access_key_$partition at $start_time"
10+
echo "[INFO] Starting compact for partition state_changes_access_key_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
WITH ordered_data AS (
14+
SELECT
15+
account_id,
16+
data_key,
17+
data_value,
18+
block_height AS block_height_from,
19+
LAG(block_height) OVER (
20+
PARTITION BY account_id, data_key
21+
ORDER BY block_height DESC
22+
) AS block_height_to
23+
FROM state_changes_access_key_1300_$partition
24+
),
25+
insert_1300_compact AS (
26+
INSERT INTO state_changes_access_key_1300_compact_$partition (
27+
account_id, data_key, data_value, block_height_from, block_height_to
28+
)
29+
SELECT
30+
account_id, data_key, data_value, block_height_from, block_height_to
31+
FROM ordered_data
32+
WHERE data_value IS NOT NULL
33+
ON CONFLICT (account_id, data_key, block_height_from) DO NOTHING
34+
)
35+
INSERT INTO state_changes_access_key_1305_$partition (
36+
account_id, block_height, data_key, data_value
37+
)
38+
SELECT
39+
account_id,
40+
block_height_from AS block_height,
41+
data_key,
42+
data_value
43+
FROM ordered_data
44+
WHERE data_value IS NOT NULL
45+
AND block_height_from <= 130500000
46+
AND (block_height_to IS NULL OR block_height_to > 130500000)
47+
ON CONFLICT (account_id, data_key, block_height) DO NOTHING;
48+
" 2>&1 | tee -a "$LOG_FILE"
49+
50+
# shellcheck disable=SC2155
51+
local end_time=$(date +"%T")
52+
echo "[INFO] Finished compact for partition state_changes_access_key_$partition at $end_time"
53+
echo "[INFO] Finished compact for partition state_changes_access_key_$partition at $end_time" >> "$LOG_FILE"
54+
}
55+
56+
# Run migrations in parallel for partitions 0 to 99
57+
for i in $(seq 0 99); do
58+
migrate_partition "$i" &
59+
done
60+
61+
# Wait for all background jobs to finish
62+
wait

‎database/migrate_compact_with_state/migrate_accounts.sh

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting compact for partition state_changes_account_1300_$partition at $start_time"
10+
echo "[INFO] Starting compact for partition state_changes_account_1300_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
WITH ordered_data AS (
14+
SELECT
15+
account_id,
16+
data_value,
17+
block_height AS block_height_from,
18+
LAG(block_height) OVER (
19+
PARTITION BY account_id
20+
ORDER BY block_height DESC
21+
) AS block_height_to
22+
FROM state_changes_account_1300_$partition
23+
),
24+
insert_1300_compact AS (
25+
INSERT INTO state_changes_account_1300_compact_$partition (
26+
account_id, data_value, block_height_from, block_height_to
27+
)
28+
SELECT
29+
account_id, data_value, block_height_from, block_height_to
30+
FROM ordered_data
31+
WHERE data_value IS NOT NULL
32+
ON CONFLICT (account_id, block_height_from) DO NOTHING
33+
)
34+
INSERT INTO state_changes_account_1305_$partition (
35+
account_id, block_height, data_value
36+
)
37+
SELECT
38+
account_id,
39+
block_height_from AS block_height,
40+
data_value
41+
FROM ordered_data
42+
WHERE data_value IS NOT NULL
43+
AND block_height_from <= 130500000
44+
AND (block_height_to IS NULL OR block_height_to > 130500000)
45+
ON CONFLICT (account_id, block_height) DO NOTHING;
46+
" 2>&1 | tee -a "$LOG_FILE"
47+
48+
# shellcheck disable=SC2155
49+
local end_time=$(date +"%T")
50+
echo "[INFO] Finished compact for partition state_changes_account_1300_$partition at $end_time"
51+
echo "[INFO] Finished compact for partition state_changes_account_1300_$partition at $end_time" >> "$LOG_FILE"
52+
}
53+
54+
# Run migrations in parallel for partitions 0 to 99
55+
for i in $(seq 0 99); do
56+
migrate_partition "$i" &
57+
done
58+
59+
# Wait for all background jobs to finish
60+
wait

‎database/migrate_compact_with_state/migrate_contracts.sh

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting compact for partition state_changes_contract_$partition at $start_time"
10+
echo "[INFO] Starting compact for partition state_changes_contract_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
WITH ordered_data AS (
14+
SELECT
15+
account_id,
16+
data_value,
17+
block_height AS block_height_from,
18+
LAG(block_height) OVER (
19+
PARTITION BY account_id
20+
ORDER BY block_height DESC
21+
) AS block_height_to
22+
FROM state_changes_contract_1300_$partition
23+
),
24+
insert_1300_compact AS (
25+
INSERT INTO state_changes_contract_1300_compact_$partition (
26+
account_id, data_value, block_height_from, block_height_to
27+
)
28+
SELECT
29+
account_id, data_value, block_height_from, block_height_to
30+
FROM ordered_data
31+
WHERE data_value IS NOT NULL
32+
ON CONFLICT (account_id, block_height_from) DO NOTHING
33+
)
34+
INSERT INTO state_changes_contract_1305_$partition (
35+
account_id, block_height, data_value
36+
)
37+
SELECT
38+
account_id,
39+
block_height_from AS block_height,
40+
data_value
41+
FROM ordered_data
42+
WHERE data_value IS NOT NULL
43+
AND block_height_from <= 130500000
44+
AND (block_height_to IS NULL OR block_height_to > 130500000)
45+
ON CONFLICT (account_id, block_height) DO NOTHING;
46+
" 2>&1 | tee -a "$LOG_FILE"
47+
48+
# shellcheck disable=SC2155
49+
local end_time=$(date +"%T")
50+
echo "[INFO] Finished compact for partition state_changes_contract_$partition at $end_time"
51+
echo "[INFO] Finished compact for partition state_changes_contract_$partition at $end_time" >> "$LOG_FILE"
52+
}
53+
54+
# Run migrations in parallel for partitions 0 to 99
55+
for i in $(seq 0 99); do
56+
migrate_partition "$i" &
57+
done
58+
59+
# Wait for all background jobs to finish
60+
wait

‎database/migrate_compact_with_state/migrate_state_changes.sh

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash
2+
3+
# Function to migrate a single partition
4+
migrate_partition() {
5+
local partition=$1
6+
# shellcheck disable=SC2155
7+
local start_time=$(date +"%T")
8+
9+
echo "[INFO] Starting compact for partition state_changes_data_$partition at $start_time"
10+
echo "[INFO] Starting compact for partition state_changes_data_$partition at $start_time" >> "$LOG_FILE"
11+
12+
psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
13+
WITH ordered_data AS (
14+
SELECT
15+
account_id,
16+
data_key,
17+
data_value,
18+
block_height AS block_height_from,
19+
LAG(block_height) OVER (
20+
PARTITION BY account_id, data_key
21+
ORDER BY block_height DESC
22+
) AS block_height_to
23+
FROM state_changes_data_1300_$partition
24+
),
25+
insert_1300_compact AS (
26+
INSERT INTO state_changes_data_1300_compact_$partition (
27+
account_id, data_key, data_value, block_height_from, block_height_to
28+
)
29+
SELECT
30+
account_id, data_key, data_value, block_height_from, block_height_to
31+
FROM ordered_data
32+
WHERE data_value IS NOT NULL
33+
ON CONFLICT (account_id, data_key, block_height_from) DO NOTHING
34+
)
35+
INSERT INTO state_changes_data_1305_$partition (
36+
account_id, block_height, data_key, data_value
37+
)
38+
SELECT
39+
account_id,
40+
block_height_from AS block_height,
41+
data_key,
42+
data_value
43+
FROM ordered_data
44+
WHERE data_value IS NOT NULL
45+
AND block_height_from <= 130500000
46+
AND (block_height_to IS NULL OR block_height_to > 130500000)
47+
ON CONFLICT (account_id, data_key, block_height) DO NOTHING;
48+
" 2>&1 | tee -a "$LOG_FILE"
49+
50+
# shellcheck disable=SC2155
51+
local end_time=$(date +"%T")
52+
echo "[INFO] Finished compact for partition state_changes_data_$partition at $end_time"
53+
echo "[INFO] Finished compact for partition state_changes_data_$partition at $end_time" >> "$LOG_FILE"
54+
}
55+
56+
# Run migrations in parallel for partitions 0 to 99
57+
for i in $(seq 0 99); do
58+
migrate_partition "$i" &
59+
done
60+
61+
# Wait for all background jobs to finish
62+
wait

‎database/migrate_compact_with_state/shard_migration.sh

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
3+
# Set database credentials
4+
export PGPASSWORD="S8TSz8vz4sQ80D7ycWSnT5B4"
5+
export DB_NAME="shard0"
6+
export DB_USER="readrpc"
7+
export DB_HOST="localhost"
8+
export DB_PORT=5432
9+
10+
# Set log file
11+
export LOG_FILE="migration0.log"
12+
# Remove old log file if it exists
13+
rm -f "$LOG_FILE"
14+
touch "$LOG_FILE"
15+
16+
echo "Starting compact at $(date)" | tee -a "$LOG_FILE"
17+
18+
./migrate_access_keys.sh &
19+
./migrate_accounts.sh &
20+
./migrate_contracts.sh &
21+
./migrate_state_changes.sh &
22+
23+
wait
24+
25+
echo "Compact completed at $(date)" | tee -a "$LOG_FILE"

‎database/sql_databases/drop.sql

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
drop table state_changes_contract_1305;
2+
drop table state_changes_account_1305;
3+
drop table state_changes_access_key_1305;
4+
drop table state_changes_data_1305;
5+
drop table state_changes_access_key_1300_compact;
6+
drop table state_changes_contract_1300_compact;
7+
drop table state_changes_account_1300_compact;
8+
drop table state_changes_data_1300_compact;

‎state-indexer/genesis_config.json

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
{
2+
"avg_hidden_validator_seats_per_shard": [
3+
0
4+
],
5+
"block_producer_kickout_threshold": 90,
6+
"chain_id": "mainnet",
7+
"chunk_producer_assignment_changes_limit": 5,
8+
"chunk_producer_kickout_threshold": 90,
9+
"chunk_validator_only_kickout_threshold": 80,
10+
"dynamic_resharding": false,
11+
"epoch_length": 43200,
12+
"fishermen_threshold": "340282366920938463463374607431768211455",
13+
"gas_limit": 1000000000000000,
14+
"gas_price_adjustment_rate": [
15+
1,
16+
100
17+
],
18+
"genesis_height": 9820210,
19+
"genesis_time": "2020-07-21T16:55:51.591948Z",
20+
"max_gas_price": "10000000000000000000000",
21+
"max_inflation_rate": [
22+
0,
23+
1
24+
],
25+
"max_kickout_stake_perc": 100,
26+
"min_gas_price": "1000000000",
27+
"minimum_stake_divisor": 10,
28+
"minimum_stake_ratio": [
29+
1,
30+
6250
31+
],
32+
"minimum_validators_per_shard": 1,
33+
"num_block_producer_seats": 100,
34+
"num_block_producer_seats_per_shard": [
35+
100
36+
],
37+
"num_blocks_per_year": 31536000,
38+
"num_chunk_only_producer_seats": 300,
39+
"num_chunk_producer_seats": 100,
40+
"num_chunk_validator_seats": 300,
41+
"online_max_threshold": [
42+
99,
43+
100
44+
],
45+
"online_min_threshold": [
46+
90,
47+
100
48+
],
49+
"protocol_reward_rate": [
50+
0,
51+
1
52+
],
53+
"protocol_treasury_account": "treasury.near",
54+
"protocol_upgrade_stake_threshold": [
55+
4,
56+
5
57+
],
58+
"protocol_version": 29,
59+
"shard_layout": {
60+
"V0": {
61+
"num_shards": 1,
62+
"version": 0
63+
}
64+
},
65+
"shuffle_shard_assignment_for_chunk_producers": false,
66+
"target_validator_mandates_per_shard": 68,
67+
"total_supply": "999999999792372916156395166000000",
68+
"transaction_validity_period": 86400,
69+
"use_production_config": false,
70+
"validators": [
71+
{
72+
"account_id": "nfvalidator1.near",
73+
"amount": "50000000000000000000000000000",
74+
"public_key": "ed25519:14pWWRutZtGFKX4B8q89KVFaUWY1Cqu1JcqYXhCDeFh1"
75+
},
76+
{
77+
"account_id": "nfvalidator2.near",
78+
"amount": "50000000000000000000000000000",
79+
"public_key": "ed25519:BwZk4bkYJxo79P2vSRw2uk1nfiqEfVkHvr5p8eVsqASC"
80+
},
81+
{
82+
"account_id": "nfvalidator3.near",
83+
"amount": "50000000000000000000000000000",
84+
"public_key": "ed25519:DMz11tmPvhdqpi7CzP2JULeeSE8SxYRD8pys5nKke4FS"
85+
},
86+
{
87+
"account_id": "nfvalidator4.near",
88+
"amount": "50000000000000000000000000000",
89+
"public_key": "ed25519:Fi3CQDHJoviKazVR27YmfFzWcFnvmoPBKEDd9ouq5Tjx"
90+
}
91+
]
92+
}

0 commit comments

Comments
 (0)