Skip to content

Commit a067199

Browse files
committed
Add support for filecoin data
1 parent d5e7f79 commit a067199

File tree

4 files changed

+94
-5
lines changed

4 files changed

+94
-5
lines changed

scripts/tests/upload_rcpsnaps.sh

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/usr/bin/env bash
2+
# This script compresses all `.rpcsnap.json` files in a given directory using zstd,
3+
# then uploads the compressed files to our DigitalOcean Spaces.
4+
5+
SPACE_NAME="forest-snapshots"
6+
REGION="fra1"
7+
DEST_DIR="rpc_test"
8+
9+
TEST_SNAPSHOTS="src/tool/subcommands/api_cmd/test_snapshots.txt"
10+
11+
if [ -z "$1" ]; then
12+
echo "❌ Please provide the directory path as an argument."
13+
echo "Usage: $0 <directory_path>"
14+
exit 1
15+
fi
16+
17+
DIR_PATH="$1"
18+
19+
if [ ! -d "$DIR_PATH" ]; then
20+
echo "❌ Directory does not exist: ${DIR_PATH}"
21+
exit 1
22+
fi
23+
24+
for FILE_PATH in "${DIR_PATH}"/*.rpcsnap.json; do
25+
if [ ! -f "$FILE_PATH" ]; then
26+
echo "❌ No .rpcsnap.json files found in ${DIR_PATH}"
27+
break
28+
fi
29+
30+
FILE_NAME=$(basename "$FILE_PATH")
31+
COMPRESSED_FILE="${FILE_PATH}.zst"
32+
DEST_PATH="${DEST_DIR}/${FILE_NAME}.zst"
33+
BUCKET_URL="s3://${SPACE_NAME}/${DEST_PATH}"
34+
35+
zstd -f "$FILE_PATH" -o "$COMPRESSED_FILE"
36+
37+
if [ $? -eq 0 ]; then
38+
s3cmd --quiet --no-progress put "${COMPRESSED_FILE}" "${BUCKET_URL}" --acl-public --mime-type="application/json"
39+
if [ $? -eq 0 ]; then
40+
echo "✅ Uploaded: ${COMPRESSED_FILE}"
41+
# echo " 📎 URL: https://${SPACE_NAME}.${REGION}.digitaloceanspaces.com/${DEST_PATH}"
42+
43+
BASE_NAME=$(basename "$COMPRESSED_FILE")
44+
echo "$BASE_NAME" >> "$TEST_SNAPSHOTS"
45+
else
46+
echo "❌ Failed to upload: ${COMPRESSED_FILE}"
47+
fi
48+
else
49+
echo "❌ Failed to compress: ${FILE_NAME}"
50+
continue
51+
fi
52+
done
53+
54+
# Sort the file lexicographically
55+
sort -o "$TEST_SNAPSHOTS" "$TEST_SNAPSHOTS"
56+
57+
read -p "🧪 Do you want to run 'cargo test --lib -- --test rpc_regression_tests --nocapture'? [y/N] " answer
58+
case "$answer" in
59+
[yY][eE][sS]|[yY])
60+
echo "▶ Running tests..."
61+
cargo test --lib -- --test rpc_regression_tests --nocapture
62+
;;
63+
*)
64+
echo "Skipping test run."
65+
;;
66+
esac

src/tool/subcommands/api_cmd/generate_test_snapshot.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,10 @@ pub(super) fn build_index(db: Arc<ReadOpsTrackingStore<ManyCar<ParityDb>>>) -> O
7575
for (k, v) in reader.iter() {
7676
index.eth_mappings.insert(k.to_string(), Payload(v.clone()));
7777
}
78+
let reader = db.tracker.indices_db.read();
79+
for (k, v) in reader.iter() {
80+
index.eth_mappings.insert(k.to_string(), Payload(v.clone()));
81+
}
7882
if index == Index::default() {
7983
None
8084
} else {

src/tool/subcommands/api_cmd/test_snapshot.rs

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ use crate::{
2121
shim::address::{CurrentNetwork, Network},
2222
state_manager::StateManager,
2323
};
24+
use cid::Cid;
2425
use openrpc_types::ParamStructure;
2526
use parking_lot::RwLock;
2627
use serde::{Deserialize, Serialize};
@@ -50,9 +51,17 @@ pub struct RpcTestSnapshot {
5051
fn backfill_eth_mappings(db: &MemoryDB, index: Option<Index>) -> anyhow::Result<()> {
5152
if let Some(index) = index {
5253
if let Some(mut guard) = db.eth_mappings_db.try_write() {
53-
for (k, v) in index.eth_mappings.into_iter() {
54-
let hash = EthHash::from_str(&k)?;
55-
guard.insert(hash, v.0);
54+
for (k, v) in index.eth_mappings.iter() {
55+
if let Ok(hash) = EthHash::from_str(&k) {
56+
guard.insert(hash, v.0.clone());
57+
}
58+
}
59+
}
60+
if let Some(mut guard) = db.indices_db.try_write() {
61+
for (k, v) in index.eth_mappings.iter() {
62+
if let Ok(cid) = Cid::from_str(&k) {
63+
guard.insert(cid, v.0.clone());
64+
}
5665
}
5766
}
5867
}

src/tool/subcommands/api_cmd/test_snapshots.txt

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
11
filecoin_beacongetentry_1741270283524366.rpcsnap.json.zst
22
filecoin_chaingetblock_1736937164811210.rpcsnap.json.zst
33
filecoin_chaingetblockmessages_1736937164799678.rpcsnap.json.zst
4+
filecoin_chaingetevents_1745400136273439.rpcsnap.json.zst
5+
filecoin_chaingetevents_1745400136273611.rpcsnap.json.zst
6+
filecoin_chaingetevents_1745400136273670.rpcsnap.json.zst
7+
filecoin_chaingetevents_1745400136273739.rpcsnap.json.zst
8+
filecoin_chaingetevents_1745400136273790.rpcsnap.json.zst
9+
filecoin_chaingetevents_1745400136273834.rpcsnap.json.zst
10+
filecoin_chaingetevents_1745400136273881.rpcsnap.json.zst
11+
filecoin_chaingetevents_1745400136273923.rpcsnap.json.zst
12+
filecoin_chaingetevents_1745400136273969.rpcsnap.json.zst
13+
filecoin_chaingetevents_1745400136274014.rpcsnap.json.zst
414
filecoin_chaingetgenesis_1736937286915866.rpcsnap.json.zst
515
filecoin_chaingetmessage_1741270616352800.rpcsnap.json.zst
616
filecoin_chaingetmessagesintipset_1741270616353560.rpcsnap.json.zst
@@ -91,6 +101,7 @@ filecoin_statemarketbalance_1737021768960382.rpcsnap.json.zst
91101
filecoin_statemarketdeals_1737021771457252.rpcsnap.json.zst
92102
filecoin_statemarketparticipants_1737021768960732.rpcsnap.json.zst
93103
filecoin_statemarketstoragedeal_1737021768962584.rpcsnap.json.zst
104+
filecoin_statemineractivesectors_1743095344878935.rpcsnap.json.zst
94105
filecoin_stateminerallocated_1737022538682774.rpcsnap.json.zst
95106
filecoin_statemineravailablebalance_1737022538683759.rpcsnap.json.zst
96107
filecoin_stateminerdeadlines_1742515166713076.rpcsnap.json.zst
@@ -99,13 +110,12 @@ filecoin_stateminerinfo_1737022538684987.rpcsnap.json.zst
99110
filecoin_stateminerinitialpledgecollateral_1741784141639426.rpcsnap.json.zst
100111
filecoin_stateminerinitialpledgeforsector_1737022538685276.rpcsnap.json.zst
101112
filecoin_stateminerpartitions_1737022538685304.rpcsnap.json.zst
102-
filecoin_stateminerprecommitdepositforpower_1741784287224778.rpcsnap.json.zst
103113
filecoin_stateminerpower_1737022538685392.rpcsnap.json.zst
114+
filecoin_stateminerprecommitdepositforpower_1741784287224778.rpcsnap.json.zst
104115
filecoin_stateminerprovingdeadline_1737022538685431.rpcsnap.json.zst
105116
filecoin_stateminerrecoveries_1737022538685471.rpcsnap.json.zst
106117
filecoin_stateminersectorallocated_1737022538685862.rpcsnap.json.zst
107118
filecoin_stateminersectorcount_1737022538685937.rpcsnap.json.zst
108-
filecoin_statemineractivesectors_1743095344878935.rpcsnap.json.zst
109119
filecoin_stateminersectors_1743095345529547.rpcsnap.json.zst
110120
filecoin_statenetworkname_1737546933390169.rpcsnap.json.zst
111121
filecoin_statenetworkversion_1737546933390184.rpcsnap.json.zst

0 commit comments

Comments
 (0)