add docs about apply migrations via sqlx

README.md

@@ -27,6 +27,46 @@ The indexer built on top of Lake Framework that watches the network and stores t
 
 The configuration module is responsible for managing the configuration settings of the NEAR ReadRPC project.
 
+### [database](database/README.md)
+
+The database module provides database abstractions and implementations for storing and retrieving data.
+
+### [cache-storage](cache-storage/README.md)
+
+The cache storage module provides caching functionality for improved performance.
+
+### [tx-details-storage](tx-details-storage/README.md)
+
+The transaction details storage module handles storage of detailed transaction information.
+
+### [logic-state-indexer](logic-state-indexer/README.md)
+
+The logic state indexer module provides state indexing functionality.
+
+### [readnode-primitives](readnode-primitives/README.md)
+
+The readnode primitives module contains common data structures and utilities.
+
+### [perf-testing](perf-testing/README.md)
+
+The performance testing module provides tools for testing and benchmarking.
+
+## Documentation
+
+### Project Documentation
+- [CHANGELOG.md](CHANGELOG.md) - Project changelog and version history
+- [Examples](examples/README.md) - Usage examples and sample configurations
+
+### Technical Documentation
+- [RPC Methods](docs/RPC_METHODS.md) - Available RPC methods and their specifications
+- [Custom RPC Methods](docs/CUSTOM_RPC_METHODS.md) - Custom RPC methods specific to Read RPC
+- [Database Migrations](docs/DATABASE_MIGRATIONS.md) - Database migration procedures and guidelines
+- [Tracing](docs/TRACING.md) - Distributed tracing setup and configuration
+
+### Database Documentation
+- [PostgreSQL Setup](database/src/postgres/README.md) - PostgreSQL-specific configuration and setup
+- [Database Migrations](docs/DATABASE_MIGRATIONS.md) - Database migration procedures and guidelines
+
 ## Docker compose
 
 **Note!** The docker compose is not fully ready yet. It's still in progress. However, you can run the entire project to play around with it. It is still not convenient for development or debugging purposes. We are working on improving it.

configuration/src/configs/database.rs

@@ -22,9 +22,6 @@ pub struct DatabaseConfig {
     pub shards_config:
         std::collections::HashMap<near_primitives::types::ShardId, DatabaseConnectUrl>,
     pub max_connections: u32,
-    // Migrations cannot be applied to read-only replicas
-    // We should run rpc-server only on read-only replicas
-    pub read_only: bool,
     pub shard_layout: Option<near_primitives::shard_layout::ShardLayout>,
 }
 
@@ -34,7 +31,6 @@ impl DatabaseConfig {
             database_url: self.database_url.clone(),
             shards_config: self.shards_config.clone(),
             max_connections: self.max_connections,
-            read_only: true,
             shard_layout: self.shard_layout.clone(),
         }
     }
@@ -77,7 +73,6 @@ impl From<CommonDatabaseConfig> for DatabaseConfig {
             max_connections: database_config
                 .max_connections
                 .unwrap_or_else(CommonDatabaseConfig::default_max_connections),
-            read_only: false,
             shard_layout: crate::shard_layout().ok(),
         }
     }

database/README.md

@@ -3,3 +3,15 @@
 This is a helper crate that provides db manager.
 
 ## [Postgres DB](src/postgres/README.md)
+
+## [Database Migrations with SQLx](../docs/DATABASE_MIGRATIONS.md)
+
+Comprehensive guide for managing database migrations using SQLx CLI. This covers:
+- Setting up and applying migrations for all database types (meta, shard, transaction details)
+- Creating new migrations
+- Migration best practices and troubleshooting
+- Automated migration scripts
+
+## [Legacy Database Migration Scripts](database_migrations/README.md)
+
+Scripts for migrating data from existing databases (only needed for upgrading existing deployments).

database/src/postgres/mod.rs

@@ -2,11 +2,6 @@ mod rpc_server;
 mod state_indexer;
 mod tx_indexer;
 
-static META_DB_MIGRATOR: sqlx::migrate::Migrator =
-    sqlx::migrate!("src/postgres/migrations/meta_db");
-static SHARD_DB_MIGRATOR: sqlx::migrate::Migrator =
-    sqlx::migrate!("src/postgres/migrations/shard_db");
-
 #[derive(borsh::BorshSerialize, borsh::BorshDeserialize, Clone, Debug)]
 struct PageState {
     pub last_data_key: Option<String>,
@@ -45,31 +40,23 @@ pub struct PostgresDBManager {
 impl PostgresDBManager {
     async fn create_meta_db_pool(
         database_url: &str,
-        read_only: bool,
         max_connections: u32,
     ) -> anyhow::Result<sqlx::Pool<sqlx::Postgres>> {
         let pool = sqlx::postgres::PgPoolOptions::new()
             .max_connections(max_connections)
             .connect(database_url)
             .await?;
-        if !read_only {
-            Self::run_migrations(&META_DB_MIGRATOR, &pool).await?;
-        }
         Ok(pool)
     }
 
     async fn create_shard_db_pool(
         database_url: &str,
-        read_only: bool,
         max_connections: u32,
     ) -> anyhow::Result<sqlx::Pool<sqlx::Postgres>> {
         let pool = sqlx::postgres::PgPoolOptions::new()
             .max_connections(max_connections)
             .connect(database_url)
             .await?;
-        if !read_only {
-            Self::run_migrations(&SHARD_DB_MIGRATOR, &pool).await?;
-        }
         Ok(pool)
     }
 
@@ -99,25 +86,13 @@ impl PostgresDBManager {
             ))?,
         })
     }
-
-    async fn run_migrations(
-        migrator: &sqlx::migrate::Migrator,
-        pool: &sqlx::Pool<sqlx::Postgres>,
-    ) -> anyhow::Result<()> {
-        migrator.run(pool).await?;
-        Ok(())
-    }
 }
 
 #[async_trait::async_trait]
 impl crate::BaseDbManager for PostgresDBManager {
     async fn new(config: &configuration::DatabaseConfig) -> anyhow::Result<Box<Self>> {
-        let meta_db_pool = Self::create_meta_db_pool(
-            &config.database_url,
-            config.read_only,
-            config.max_connections,
-        )
-        .await?;
+        let meta_db_pool =
+            Self::create_meta_db_pool(&config.database_url, config.max_connections).await?;
         let mut shards_pool = std::collections::HashMap::new();
         let shard_layout = config
             .shard_layout
@@ -128,9 +103,7 @@ impl crate::BaseDbManager for PostgresDBManager {
                 .shards_config
                 .get(&shard_id)
                 .unwrap_or_else(|| panic!("Shard_{shard_id} - database config not found"));
-            let pool =
-                Self::create_shard_db_pool(database_url, config.read_only, config.max_connections)
-                    .await?;
+            let pool = Self::create_shard_db_pool(database_url, config.max_connections).await?;
             shards_pool.insert(shard_id, pool);
         }
         Ok(Box::new(Self {

database/src/postgres/tx_indexer.rs

@@ -4,23 +4,11 @@ use bigdecimal::num_traits::ToPrimitive;
 use bigdecimal::BigDecimal;
 use sqlx::QueryBuilder;
 
-static META_RECEIPTS_AND_OUTCOMES_MIGRATOR: sqlx::migrate::Migrator =
-    sqlx::migrate!("src/postgres/migrations/tx_details/receipts_and_outcomes");
-static SHARDS_TRANSACTIONS_MIGRATOR: sqlx::migrate::Migrator =
-    sqlx::migrate!("src/postgres/migrations/tx_details/transactions");
-
 #[async_trait]
 impl crate::base::tx_indexer::TxIndexerDbManager for crate::postgres::PostgresDBManager {
     async fn create_tx_tables(&self) -> Result<()> {
-        // Transactions table and partitions on each shard
-        for pool in self.shards_pool.values() {
-            SHARDS_TRANSACTIONS_MIGRATOR.run(pool).await?;
-        }
-
-        // Receipts and outcomes tables and partitions in meta_db_pool only
-        META_RECEIPTS_AND_OUTCOMES_MIGRATOR
-            .run(&self.meta_db_pool)
-            .await?;
+        // For POstgres please read the `README.md` in the `database/src/postgres` directory.
+        // The tables are created by the migrations, so this method is not needed.
         Ok(())
     }