Add support for filecoin data

Author: elmattic

scripts/tests/upload_rcpsnaps.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# This script compresses all `.rpcsnap.json` files in a given directory using zstd,
+# then uploads the compressed files to our DigitalOcean Spaces.
+
+SPACE_NAME="forest-snapshots"
+REGION="fra1"
+DEST_DIR="rpc_test"
+
+TEST_SNAPSHOTS="src/tool/subcommands/api_cmd/test_snapshots.txt"
+
+if [ -z "$1" ]; then
+    echo "❌ Please provide the directory path as an argument."
+    echo "Usage: $0 <directory_path>"
+    exit 1
+fi
+
+DIR_PATH="$1"
+
+if [ ! -d "$DIR_PATH" ]; then
+    echo "❌ Directory does not exist: ${DIR_PATH}"
+    exit 1
+fi
+
+for FILE_PATH in "${DIR_PATH}"/*.rpcsnap.json; do
+    if [ ! -f "$FILE_PATH" ]; then
+        echo "❌ No .rpcsnap.json files found in ${DIR_PATH}"
+        break
+    fi
+
+    FILE_NAME=$(basename "$FILE_PATH")
+    COMPRESSED_FILE="${FILE_PATH}.zst"
+    DEST_PATH="${DEST_DIR}/${FILE_NAME}.zst"
+    BUCKET_URL="s3://${SPACE_NAME}/${DEST_PATH}"
+
+    zstd -f "$FILE_PATH" -o "$COMPRESSED_FILE"
+
+    if [ $? -eq 0 ]; then
+        s3cmd --quiet --no-progress put "${COMPRESSED_FILE}" "${BUCKET_URL}" --acl-public --mime-type="application/json"
+        if [ $? -eq 0 ]; then
+            echo "✅ Uploaded: ${COMPRESSED_FILE}"
+            # echo "   📎 URL: https://${SPACE_NAME}.${REGION}.digitaloceanspaces.com/${DEST_PATH}"
+
+            BASE_NAME=$(basename "$COMPRESSED_FILE")
+            echo "$BASE_NAME" >> "$TEST_SNAPSHOTS"
+        else
+            echo "❌ Failed to upload: ${COMPRESSED_FILE}"
+        fi
+    else
+        echo "❌ Failed to compress: ${FILE_NAME}"
+        continue
+    fi
+done
+
+# Sort the file lexicographically
+sort -o "$TEST_SNAPSHOTS" "$TEST_SNAPSHOTS"
+
+read -p "🧪 Do you want to run 'cargo test --lib -- --test rpc_regression_tests --nocapture'? [y/N] " answer
+case "$answer" in
+    [yY][eE][sS]|[yY])
+        echo "▶ Running tests..."
+        cargo test --lib -- --test rpc_regression_tests --nocapture
+        ;;
+    *)
+        echo "Skipping test run."
+        ;;
+esac

src/tool/subcommands/api_cmd/generate_test_snapshot.rs

@@ -75,6 +75,10 @@ pub(super) fn build_index(db: Arc<ReadOpsTrackingStore<ManyCar<ParityDb>>>) -> O
     for (k, v) in reader.iter() {
         index.eth_mappings.insert(k.to_string(), Payload(v.clone()));
     }
+    let reader = db.tracker.indices_db.read();
+    for (k, v) in reader.iter() {
+        index.eth_mappings.insert(k.to_string(), Payload(v.clone()));
+    }
     if index == Index::default() {
         None
     } else {

src/tool/subcommands/api_cmd/test_snapshot.rs

@@ -21,6 +21,7 @@ use crate::{
     shim::address::{CurrentNetwork, Network},
     state_manager::StateManager,
 };
+use cid::Cid;
 use openrpc_types::ParamStructure;
 use parking_lot::RwLock;
 use serde::{Deserialize, Serialize};
@@ -50,9 +51,17 @@ pub struct RpcTestSnapshot {
 fn backfill_eth_mappings(db: &MemoryDB, index: Option<Index>) -> anyhow::Result<()> {
     if let Some(index) = index {
         if let Some(mut guard) = db.eth_mappings_db.try_write() {
-            for (k, v) in index.eth_mappings.into_iter() {
-                let hash = EthHash::from_str(&k)?;
-                guard.insert(hash, v.0);
+            for (k, v) in index.eth_mappings.iter() {
+                if let Ok(hash) = EthHash::from_str(&k) {
+                    guard.insert(hash, v.0.clone());
+                }
+            }
+        }
+        if let Some(mut guard) = db.indices_db.try_write() {
+            for (k, v) in index.eth_mappings.iter() {
+                if let Ok(cid) = Cid::from_str(&k) {
+                    guard.insert(cid, v.0.clone());
+                }
             }
         }
     }

src/tool/subcommands/api_cmd/test_snapshots.txt

@@ -1,6 +1,16 @@
 filecoin_beacongetentry_1741270283524366.rpcsnap.json.zst
 filecoin_chaingetblock_1736937164811210.rpcsnap.json.zst
 filecoin_chaingetblockmessages_1736937164799678.rpcsnap.json.zst
+filecoin_chaingetevents_1745400136273439.rpcsnap.json.zst
+filecoin_chaingetevents_1745400136273611.rpcsnap.json.zst
+filecoin_chaingetevents_1745400136273670.rpcsnap.json.zst
+filecoin_chaingetevents_1745400136273739.rpcsnap.json.zst
+filecoin_chaingetevents_1745400136273790.rpcsnap.json.zst
+filecoin_chaingetevents_1745400136273834.rpcsnap.json.zst
+filecoin_chaingetevents_1745400136273881.rpcsnap.json.zst
+filecoin_chaingetevents_1745400136273923.rpcsnap.json.zst
+filecoin_chaingetevents_1745400136273969.rpcsnap.json.zst
+filecoin_chaingetevents_1745400136274014.rpcsnap.json.zst
 filecoin_chaingetgenesis_1736937286915866.rpcsnap.json.zst
 filecoin_chaingetmessage_1741270616352800.rpcsnap.json.zst
 filecoin_chaingetmessagesintipset_1741270616353560.rpcsnap.json.zst
@@ -91,6 +101,7 @@ filecoin_statemarketbalance_1737021768960382.rpcsnap.json.zst
 filecoin_statemarketdeals_1737021771457252.rpcsnap.json.zst
 filecoin_statemarketparticipants_1737021768960732.rpcsnap.json.zst
 filecoin_statemarketstoragedeal_1737021768962584.rpcsnap.json.zst
+filecoin_statemineractivesectors_1743095344878935.rpcsnap.json.zst
 filecoin_stateminerallocated_1737022538682774.rpcsnap.json.zst
 filecoin_statemineravailablebalance_1737022538683759.rpcsnap.json.zst
 filecoin_stateminerdeadlines_1742515166713076.rpcsnap.json.zst
@@ -99,13 +110,12 @@ filecoin_stateminerinfo_1737022538684987.rpcsnap.json.zst
 filecoin_stateminerinitialpledgecollateral_1741784141639426.rpcsnap.json.zst
 filecoin_stateminerinitialpledgeforsector_1737022538685276.rpcsnap.json.zst
 filecoin_stateminerpartitions_1737022538685304.rpcsnap.json.zst
-filecoin_stateminerprecommitdepositforpower_1741784287224778.rpcsnap.json.zst
 filecoin_stateminerpower_1737022538685392.rpcsnap.json.zst
+filecoin_stateminerprecommitdepositforpower_1741784287224778.rpcsnap.json.zst
 filecoin_stateminerprovingdeadline_1737022538685431.rpcsnap.json.zst
 filecoin_stateminerrecoveries_1737022538685471.rpcsnap.json.zst
 filecoin_stateminersectorallocated_1737022538685862.rpcsnap.json.zst
 filecoin_stateminersectorcount_1737022538685937.rpcsnap.json.zst
-filecoin_statemineractivesectors_1743095344878935.rpcsnap.json.zst
 filecoin_stateminersectors_1743095345529547.rpcsnap.json.zst
 filecoin_statenetworkname_1737546933390169.rpcsnap.json.zst
 filecoin_statenetworkversion_1737546933390184.rpcsnap.json.zst