refactor(database, state-indexer): State schema improvements for reads and updates

database/Cargo.toml

@@ -28,6 +28,13 @@ sqlx = { version = "0.8.2", features = [
 num-bigint = "0.3.3"
 num-traits = "0.2.19"
 scylla = { version = "0.15.1", features = ["ssl", "full-serialization"] }
+tokio = { version = "1.36.0", features = [
+    "sync",
+    "time",
+    "macros",
+    "rt-multi-thread",
+] }
+tracing = "0.1.34"
 
 configuration.workspace = true
 readnode-primitives.workspace = true

database/database_migrations/README.md

@@ -0,0 +1,100 @@
+> **Note:** If you are starting the project from scratch (with a new, empty database), you do not need to run the migration scripts in this directory. These scripts are only necessary when migrating data from an existing database or upgrading shards. For new deployments, follow the standard database initialization procedures instead.
+
+Database Migration Scripts
+
+This directory contains scripts for migrating database shards and related data.
+
+## Shard Migration Script
+
+The `shard_migration.sh` script is the main migration orchestrator that runs multiple migration scripts in parallel.
+
+### Usage
+
+The script accepts the following command-line arguments:
+
+- `--db_name`: Database name
+- `--db_user`: Database username  
+- `--db_password`: Database password
+- `--host`: Database host
+- `--port`: Database port
+
+### Examples
+
+#### Basic Example:
+```bash
+./shard_migration.sh --db_name my_database --db_user postgres --db_password mypassword --host localhost --port 5432
+```
+
+#### Local PostgreSQL Database:
+```bash
+./shard_migration.sh \
+  --db_name read_rpc_db \
+  --db_user postgres \
+  --db_password secretpassword \
+  --host localhost \
+  --port 5432
+```
+
+#### Remote Database:
+```bash
+./shard_migration.sh \
+  --db_name production_db \
+  --db_user readrpc_user \
+  --db_password prod_password123 \
+  --host db.example.com \
+  --port 5432
+```
+
+#### Using Environment Variables:
+```bash
+# Set environment variables first
+export DB_NAME="my_database"
+export DB_USER="postgres" 
+export PGPASSWORD="mypassword"
+export DB_HOST="localhost"
+export DB_PORT="5432"
+
+# Then run the script (it will use the environment variables)
+./shard_migration.sh
+```
+
+### Prerequisites
+
+1. **Make the script executable:**
+   ```bash
+   chmod +x shard_migration.sh
+   ```
+
+2. **Make all migration scripts executable:**
+   ```bash
+   chmod +x migrate_*.sh
+   ```
+
+3. **Ensure all required migration scripts exist:**
+   - `migrate_access_keys.sh`
+   - `migrate_accounts.sh`
+   - `migrate_contracts.sh`
+   - `migrate_state_changes.sh`
+
+### How it Works
+
+The `shard_migration.sh` script:
+
+1. Parses command-line arguments and sets environment variables
+2. Creates a log file named `migration_${DB_NAME}.log`
+3. Runs four migration scripts in parallel using the `&` operator
+4. Waits for all migrations to complete using the `wait` command
+5. Logs start and completion times
+
+### Output
+
+- Migration progress and results are logged to `migration_${DB_NAME}.log`
+- Console output shows start and completion timestamps
+- Each individual migration script may produce its own output
+
+### Notes
+
+- All arguments are required for the script to function properly
+- The script runs migrations in parallel to improve performance
+- Make sure you have proper database permissions before running the migration
+- Review the individual migration scripts to understand what data will be migrated

database/database_migrations/migrate_access_keys.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Function to migrate a single partition
+migrate_partition() {
+    local partition=$1
+    # shellcheck disable=SC2155
+    local start_time=$(date +"%T")
+
+    echo "[INFO] Starting migration for partition state_changes_access_key_$partition at $start_time"
+    echo "[INFO] Starting migration for partition state_changes_access_key_$partition at $start_time" >> "$LOG_FILE"
+
+    psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
+        WITH ordered_data AS (
+            SELECT
+                account_id,
+                data_key,
+                data_value,
+                block_height AS block_height_from,
+                LAG(block_height) OVER (PARTITION BY account_id, data_key ORDER BY block_height DESC) AS block_height_to
+            FROM state_changes_access_key_$partition
+        )
+        INSERT INTO state_changes_access_key_compact_$partition (account_id, data_key, data_value, block_height_from, block_height_to)
+        SELECT
+            account_id,
+            data_key,
+            data_value,
+            block_height_from::bigint,
+            block_height_to::bigint
+        FROM ordered_data
+        WHERE data_value IS NOT NULL
+        ON CONFLICT (account_id, data_key, block_height_from) DO NOTHING;
+    " 2>&1 | tee -a "$LOG_FILE"
+
+    # shellcheck disable=SC2155
+    local end_time=$(date +"%T")
+    echo "[INFO] Finished migration for partition state_changes_access_key_$partition at $end_time"
+    echo "[INFO] Finished migration for partition state_changes_access_key_$partition at $end_time" >> "$LOG_FILE"
+}
+
+# Run migrations in parallel for partitions 0 to 99
+for i in $(seq 0 99); do
+    migrate_partition "$i" &
+done
+
+# Wait for all background jobs to finish
+wait

database/database_migrations/migrate_accounts.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Function to migrate a single partition
+migrate_partition() {
+    local partition=$1
+    # shellcheck disable=SC2155
+    local start_time=$(date +"%T")
+
+    echo "[INFO] Starting migration for partition state_changes_account_$partition at $start_time"
+    echo "[INFO] Starting migration for partition state_changes_account_$partition at $start_time" >> "$LOG_FILE"
+
+    psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
+        WITH ordered_data AS (
+            SELECT
+                account_id,
+                data_value,
+                block_height AS block_height_from,
+                LAG(block_height) OVER (PARTITION BY account_id ORDER BY block_height DESC) AS block_height_to
+            FROM state_changes_account_$partition
+        )
+        INSERT INTO state_changes_account_compact_$partition (account_id, data_value, block_height_from, block_height_to)
+        SELECT
+            account_id,
+            data_value,
+            block_height_from::bigint,
+            block_height_to::bigint
+        FROM ordered_data
+        WHERE data_value IS NOT NULL
+        ON CONFLICT (account_id, block_height_from) DO NOTHING;
+    " 2>&1 | tee -a "$LOG_FILE"
+
+    # shellcheck disable=SC2155
+    local end_time=$(date +"%T")
+    echo "[INFO] Finished migration for partition state_changes_account_$partition at $end_time"
+    echo "[INFO] Finished migration for partition state_changes_account_$partition at $end_time" >> "$LOG_FILE"
+}
+
+# Run migrations in parallel for partitions 0 to 99
+for i in $(seq 0 99); do
+    migrate_partition "$i" &
+done
+
+# Wait for all background jobs to finish
+wait

database/database_migrations/migrate_contracts.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Function to migrate a single partition
+migrate_partition() {
+    local partition=$1
+    # shellcheck disable=SC2155
+    local start_time=$(date +"%T")
+
+    echo "[INFO] Starting migration for partition state_changes_contract_$partition at $start_time"
+    echo "[INFO] Starting migration for partition state_changes_contract_$partition at $start_time" >> "$LOG_FILE"
+
+    psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
+        WITH ordered_data AS (
+            SELECT
+                account_id,
+                data_value,
+                block_height AS block_height_from,
+                LAG(block_height) OVER (PARTITION BY account_id ORDER BY block_height DESC) AS block_height_to
+            FROM state_changes_contract_$partition
+        )
+        INSERT INTO state_changes_contract_compact_$partition (account_id, data_value, block_height_from, block_height_to)
+        SELECT
+            account_id,
+            data_value,
+            block_height_from::bigint,
+            block_height_to::bigint
+        FROM ordered_data
+        WHERE data_value IS NOT NULL
+        ON CONFLICT (account_id, block_height_from) DO NOTHING;
+    " 2>&1 | tee -a "$LOG_FILE"
+
+    # shellcheck disable=SC2155
+    local end_time=$(date +"%T")
+    echo "[INFO] Finished migration for partition state_changes_contract_$partition at $end_time"
+    echo "[INFO] Finished migration for partition state_changes_contract_$partition at $end_time" >> "$LOG_FILE"
+}
+
+# Run migrations in parallel for partitions 0 to 99
+for i in $(seq 0 99); do
+    migrate_partition "$i" &
+done
+
+# Wait for all background jobs to finish
+wait

database/database_migrations/migrate_state_changes.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Function to migrate a single partition
+migrate_partition() {
+    local partition=$1
+    # shellcheck disable=SC2155
+    local start_time=$(date +"%T")
+
+    echo "[INFO] Starting migration for partition state_changes_data_$partition at $start_time"
+    echo "[INFO] Starting migration for partition state_changes_data_$partition at $start_time" >> "$LOG_FILE"
+
+    psql -U "$DB_USER" -d "$DB_NAME" -h "$DB_HOST" -p "$DB_PORT" -c "
+        WITH ordered_data AS (
+            SELECT
+                account_id,
+                data_key,
+                data_value,
+                block_height AS block_height_from,
+                LAG(block_height) OVER (PARTITION BY account_id, data_key ORDER BY block_height DESC) AS block_height_to
+            FROM state_changes_data_$partition
+        )
+        INSERT INTO state_changes_data_compact_$partition (account_id, data_key, data_value, block_height_from, block_height_to)
+        SELECT
+            account_id,
+            data_key,
+            data_value,
+            block_height_from::bigint,
+            block_height_to::bigint
+        FROM ordered_data
+        WHERE data_value IS NOT NULL
+        ON CONFLICT (account_id, data_key, block_height_from) DO NOTHING;
+    " 2>&1 | tee -a "$LOG_FILE"
+
+    # shellcheck disable=SC2155
+    local end_time=$(date +"%T")
+    echo "[INFO] Finished migration for partition state_changes_data_$partition at $end_time"
+    echo "[INFO] Finished migration for partition state_changes_data_$partition at $end_time" >> "$LOG_FILE"
+}
+
+# Run migrations in parallel for partitions 0 to 99
+for i in $(seq 0 99); do
+    migrate_partition "$i" &
+done
+
+# Wait for all background jobs to finish
+wait

database/database_migrations/shard_migration.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+# Parse arguments or use environment variables
+while [[ $# -gt 0 ]]; do
+    key="$1"
+    case $key in
+        --db_name)
+            export DB_NAME="$2"
+            shift 2
+            ;;
+        --db_user)
+            export DB_USER="$2"
+            shift 2
+            ;;
+        --db_password)
+            export PGPASSWORD="$2"
+            shift 2
+            ;;
+        --host)
+            export DB_HOST="$2"
+            shift 2
+            ;;
+        --port)
+            export DB_PORT="$2"
+            shift 2
+            ;;
+        *)
+            echo "Unknown option: $1"
+            exit 1
+            ;;
+    esac
+done
+
+# Set defaults from environment if not set by args
+: "${DB_NAME:=${DB_NAME}}"
+: "${DB_USER:=${DB_USER}}"
+: "${PGPASSWORD:=${PGPASSWORD}}"
+: "${DB_HOST:=${DB_HOST}}"
+: "${DB_PORT:=${DB_PORT}}"
+
+# Check required variables
+if [[ -z "$DB_NAME" || -z "$DB_USER" || -z "$PGPASSWORD" || -z "$DB_HOST" || -z "$DB_PORT" ]]; then
+    echo "All arguments are required: --db_name, --db_user, --db_password, --host, --port (or set corresponding env vars)"
+    exit 1
+fi
+
+
+# Set log file
+export LOG_FILE="migration_${DB_NAME}.log"
+# Remove old log file if it exists
+rm -f "$LOG_FILE"
+touch "$LOG_FILE"
+
+echo "Starting migration at $(date)" | tee -a "$LOG_FILE"
+
+./migrate_access_keys.sh &
+./migrate_accounts.sh &
+./migrate_contracts.sh &
+./migrate_state_changes.sh &
+
+wait
+
+echo "Migration completed at $(date)" | tee -a "$LOG_FILE"