From eef772f43ec8bcddadbbd57ffd897db10e68f61d Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 16 Jul 2025 00:01:41 +0800 Subject: [PATCH 01/35] feat: add `--v2` flag to `forest-cli snapshot export` --- .config/forest.dic | 3 + scripts/tests/calibnet_export_check.sh | 6 +- .../tests/snapshot_parity/docker-compose.yml | 4 +- src/chain/mod.rs | 105 +++++++++++++++++- src/cli/subcommands/snapshot_cmd.rs | 16 ++- src/db/car/forest.rs | 58 +++++++--- src/db/memory.rs | 2 +- src/rpc/methods/chain.rs | 73 ++++++++++++ src/rpc/mod.rs | 1 + .../api_cmd/test_snapshots_ignored.txt | 1 + src/tool/subcommands/archive_cmd.rs | 77 +++++++++---- src/utils/db/car_stream.rs | 38 +++++-- 12 files changed, 325 insertions(+), 59 deletions(-) diff --git a/.config/forest.dic b/.config/forest.dic index 28cd3d77edab..04d06fb1666a 100644 --- a/.config/forest.dic +++ b/.config/forest.dic @@ -129,6 +129,9 @@ unrepresentable untrusted URL UUID +v0 +v1 +v2 validator/S varint verifier diff --git a/scripts/tests/calibnet_export_check.sh b/scripts/tests/calibnet_export_check.sh index 3bbf402b08c8..621dd5e070c8 100755 --- a/scripts/tests/calibnet_export_check.sh +++ b/scripts/tests/calibnet_export_check.sh @@ -15,6 +15,9 @@ rm --force --verbose ./*.{car,car.zst,sha256sum} echo "Exporting zstd compressed snapshot" $FOREST_CLI_PATH snapshot export +echo "Exporting zstd compressed snapshot in the experimetal v2 format" +$FOREST_CLI_PATH snapshot export --v2 -o v2.forest.car.zst + echo "Testing snapshot validity" zstd --test ./*.car.zst @@ -22,8 +25,7 @@ echo "Verifying snapshot checksum" sha256sum --check ./*.sha256sum echo "Validating CAR files" -zstd --decompress ./*.car.zst -for f in *.car; do +for f in *.car.zst; do echo "Validating CAR file $f" $FOREST_TOOL_PATH snapshot validate "$f" done diff --git a/scripts/tests/snapshot_parity/docker-compose.yml b/scripts/tests/snapshot_parity/docker-compose.yml index a7dac263ab52..10872bdf0af9 100644 --- a/scripts/tests/snapshot_parity/docker-compose.yml +++ b/scripts/tests/snapshot_parity/docker-compose.yml @@ -98,10 +98,10 @@ services: set -euxo pipefail pushd /data/exported # Skip the CAR format line and the "Index size" line (only present in Forest snapshots) - forest-tool archive info forest.car.zst | tail -n +2 | grep -v "Index size" > forest.txt + forest-tool archive info forest.car.zst | grep -v "CAR format" | grep -v "Index size" > forest.txt cat forest.txt # Skip the CAR format line - forest-tool archive info lotus.car | tail -n +2 > lotus.txt + forest-tool archive info lotus.car | grep -v "CAR format" > lotus.txt cat lotus.txt diff forest.txt lotus.txt # Do byte-to-byte comparison diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 6658ec45d764..85f8a33c52f4 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -4,14 +4,23 @@ pub mod store; mod weight; use crate::blocks::{Tipset, TipsetKey}; use crate::cid_collections::CidHashSet; -use crate::db::car::forest; +use crate::db::car::forest::{self, finalize_frame}; use crate::db::{SettingsStore, SettingsStoreExt}; use crate::ipld::stream_chain; +use crate::utils::db::car_stream::{CarBlock, CarBlockWrite}; use crate::utils::io::{AsyncWriterWithChecksum, Checksum}; +use crate::utils::multihash::MultihashCode; use crate::utils::stream::par_buffer; use anyhow::Context as _; +use cid::Cid; use digest::Digest; +use futures::StreamExt as _; use fvm_ipld_blockstore::Blockstore; +use fvm_ipld_encoding::DAG_CBOR; +use multihash_derive::MultihashDigest as _; +use nunny::Vec as NonEmpty; +use serde::{Deserialize, Serialize}; +use std::fs::File; use std::sync::Arc; use tokio::io::{AsyncWrite, AsyncWriteExt, BufWriter}; @@ -73,3 +82,97 @@ pub async fn export( Ok(digest) } + +pub async fn export_v2( + db: &Arc, + f3: Option<(Cid, &mut File)>, + tipset: &Tipset, + lookup_depth: ChainEpochDelta, + writer: impl AsyncWrite + Unpin, + seen: CidHashSet, + skip_checksum: bool, +) -> anyhow::Result>, Error> { + let stateroot_lookup_limit = tipset.epoch() - lookup_depth; + let head = tipset.key().to_cids(); + let f3_cid = f3.as_ref().map(|(cid, _)| *cid); + let snap_meta = FilecoinSnapshotMetadata::new_v2(head, f3_cid); + let snap_meta_cbor_encoded = fvm_ipld_encoding::to_vec(&snap_meta)?; + let snap_meta_block = CarBlock { + cid: Cid::new_v1( + DAG_CBOR, + MultihashCode::Blake2b256.digest(&snap_meta_cbor_encoded), + ), + data: snap_meta_cbor_encoded, + }; + let roots = nunny::vec![snap_meta_block.cid]; + let mut prefix_data_frames = vec![{ + let mut encoder = forest::new_encoder(forest::DEFAULT_FOREST_CAR_COMPRESSION_LEVEL)?; + snap_meta_block.write(&mut encoder)?; + anyhow::Ok(( + vec![snap_meta_block.cid], + finalize_frame(forest::DEFAULT_FOREST_CAR_COMPRESSION_LEVEL, &mut encoder)?, + )) + }]; + + if let Some((f3_cid, f3_data)) = f3 { + prefix_data_frames.push({ + let mut encoder = forest::new_encoder(forest::DEFAULT_FOREST_CAR_COMPRESSION_LEVEL)?; + encoder.write_car_block(f3_cid, f3_data.metadata()?.len() as _, f3_data)?; + anyhow::Ok(( + vec![f3_cid], + finalize_frame(forest::DEFAULT_FOREST_CAR_COMPRESSION_LEVEL, &mut encoder)?, + )) + }); + } + + // Wrap writer in optional checksum calculator + let mut writer = AsyncWriterWithChecksum::::new(BufWriter::new(writer), !skip_checksum); + + // Stream stateroots in range (stateroot_lookup_limit+1)..=tipset.epoch(). Also + // stream all block headers until genesis. + let blocks = par_buffer( + // Queue 1k blocks. This is enough to saturate the compressor and blocks + // are small enough that keeping 1k in memory isn't a problem. Average + // block size is between 1kb and 2kb. + 1024, + stream_chain( + Arc::clone(db), + tipset.clone().chain_owned(Arc::clone(db)), + stateroot_lookup_limit, + ) + .with_seen(seen), + ); + + // Encode Ipld key-value pairs in zstd frames + let block_frames = forest::Encoder::compress_stream_default(blocks); + let frames = futures::stream::iter(prefix_data_frames).chain(block_frames); + + // Write zstd frames and include a skippable index + forest::Encoder::write(&mut writer, roots, frames).await?; + + // Flush to ensure everything has been successfully written + writer.flush().await.context("failed to flush")?; + + let digest = writer.finalize().map_err(|e| Error::Other(e.to_string()))?; + + Ok(digest) +} + +/// +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "PascalCase")] +pub struct FilecoinSnapshotMetadata { + pub version: u64, + pub head_tipset_key: NonEmpty, + pub f3_data: Option, +} + +impl FilecoinSnapshotMetadata { + pub fn new_v2(head_tipset_key: NonEmpty, f3_data: Option) -> Self { + Self { + version: 2, + head_tipset_key, + f3_data, + } + } +} diff --git a/src/cli/subcommands/snapshot_cmd.rs b/src/cli/subcommands/snapshot_cmd.rs index d938504e10e5..44631fc2cb94 100644 --- a/src/cli/subcommands/snapshot_cmd.rs +++ b/src/cli/subcommands/snapshot_cmd.rs @@ -35,6 +35,9 @@ pub enum SnapshotCommands { /// How many state-roots to include. Lower limit is 900 for `calibnet` and `mainnet`. #[arg(short, long)] depth: Option, + /// Export snapshot in the experimental v2 format(FRC-0108). + #[arg(long)] + v2: bool, }, } @@ -47,6 +50,7 @@ impl SnapshotCommands { dry_run, tipset, depth, + v2, } => { let chain_head = ChainHead::call(&client, ()).await?; @@ -119,9 +123,15 @@ impl SnapshotCommands { }); // Manually construct RpcRequest because snapshot export could // take a few hours on mainnet - let hash_result = client - .call(ChainExport::request((params,))?.with_timeout(Duration::MAX)) - .await?; + let hash_result = if v2 { + client + .call(ChainExportV2::request((params,))?.with_timeout(Duration::MAX)) + .await? + } else { + client + .call(ChainExport::request((params,))?.with_timeout(Duration::MAX)) + .await? + }; handle.abort(); let _ = handle.await; diff --git a/src/db/car/forest.rs b/src/db/car/forest.rs index df4d16b81fcc..6abee3881ba0 100644 --- a/src/db/car/forest.rs +++ b/src/db/car/forest.rs @@ -48,6 +48,7 @@ use super::{CacheKey, ZstdFrameCache}; use crate::blocks::{Tipset, TipsetKey}; +use crate::chain::FilecoinSnapshotMetadata; use crate::db::PersistentStore; use crate::db::car::RandomAccessFileReader; use crate::db::car::plain::write_skip_frame_header_async; @@ -58,15 +59,15 @@ use ahash::{HashMap, HashMapExt}; use byteorder::LittleEndian; use bytes::{BufMut as _, Bytes, BytesMut, buf::Writer}; use cid::Cid; -use futures::{Stream, TryStream, TryStreamExt as _}; +use futures::{Stream, TryStreamExt as _}; use fvm_ipld_blockstore::Blockstore; -use fvm_ipld_encoding::to_vec; +use fvm_ipld_encoding::{CborStore, to_vec}; use nunny::Vec as NonEmpty; use parking_lot::{Mutex, RwLock}; use positioned_io::{Cursor, ReadAt, ReadBytesAtExt, SizeCursor}; use std::io::{Seek, SeekFrom}; use std::path::Path; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use std::task::Poll; use std::{ io, @@ -97,7 +98,8 @@ pub struct ForestCar { index_size_bytes: u32, frame_cache: Arc>, write_cache: Arc>>>, - roots: NonEmpty, + header: CarV1Header, + metadata: OnceLock>, } impl ForestCar { @@ -117,7 +119,22 @@ impl ForestCar { index_size_bytes, frame_cache: Arc::new(Mutex::new(ZstdFrameCache::default())), write_cache: Arc::new(RwLock::new(ahash::HashMap::default())), - roots: header.roots, + header, + metadata: OnceLock::new(), + }) + } + + pub fn metadata(&self) -> &Option { + self.metadata.get_or_init(|| { + if self.header.roots.len() == 1 { + let maybe_metadata_cid = self.header.roots.first(); + if let Ok(Some(metadata)) = + self.get_cbor::(maybe_metadata_cid) + { + return Some(metadata); + } + } + None }) } @@ -149,8 +166,12 @@ impl ForestCar { Ok((header, footer)) } - pub fn roots(&self) -> &NonEmpty { - &self.roots + pub fn head_tipset_key(&self) -> &NonEmpty { + if let Some(metadata) = self.metadata() { + &metadata.head_tipset_key + } else { + &self.header.roots + } } pub fn index_size_bytes(&self) -> u32 { @@ -158,7 +179,7 @@ impl ForestCar { } pub fn heaviest_tipset_key(&self) -> TipsetKey { - TipsetKey::from(self.roots().clone()) + TipsetKey::from(self.head_tipset_key().clone()) } pub fn heaviest_tipset(&self) -> anyhow::Result { @@ -179,7 +200,8 @@ impl ForestCar { index_size_bytes: self.index_size_bytes, frame_cache: self.frame_cache, write_cache: self.write_cache, - roots: self.roots, + header: self.header, + metadata: self.metadata, } } @@ -283,7 +305,7 @@ impl Encoder { pub async fn write( mut sink: impl AsyncWrite + Unpin, roots: NonEmpty, - mut stream: impl TryStream, Bytes), Error = anyhow::Error> + Unpin, + mut stream: impl Stream, Bytes)>> + Unpin, ) -> anyhow::Result<()> { let mut offset = 0; @@ -324,8 +346,8 @@ impl Encoder { /// `compress_stream` with [`DEFAULT_FOREST_CAR_FRAME_SIZE`] as default frame size and [`DEFAULT_FOREST_CAR_COMPRESSION_LEVEL`] as default compression level. pub fn compress_stream_default( - stream: impl TryStream, - ) -> impl TryStream, Bytes), Error = anyhow::Error> { + stream: impl Stream>, + ) -> impl Stream, Bytes)>> { Self::compress_stream( DEFAULT_FOREST_CAR_FRAME_SIZE, DEFAULT_FOREST_CAR_COMPRESSION_LEVEL, @@ -338,8 +360,8 @@ impl Encoder { pub fn compress_stream( zstd_frame_size_tripwire: usize, zstd_compression_level: u16, - stream: impl TryStream, - ) -> impl TryStream, Bytes), Error = anyhow::Error> { + stream: impl Stream>, + ) -> impl Stream, Bytes)>> { let mut encoder_store = new_encoder(zstd_compression_level); let mut frame_cids = vec![]; @@ -399,7 +421,7 @@ fn compressed_len(encoder: &zstd::Encoder<'static, Writer>) -> usize { encoder.get_ref().get_ref().len() } -fn finalize_frame( +pub fn finalize_frame( zstd_compression_level: u16, encoder: &mut zstd::Encoder<'static, Writer>, ) -> io::Result { @@ -407,7 +429,7 @@ fn finalize_frame( Ok(prev_encoder.finish()?.into_inner().freeze()) } -fn new_encoder( +pub fn new_encoder( zstd_compression_level: u16, ) -> io::Result>> { zstd::Encoder::new(BytesMut::new().writer(), i32::from(zstd_compression_level)) @@ -485,7 +507,7 @@ mod tests { let roots = nonempty!(blocks.first().cid); let forest_car = ForestCar::new(mk_encoded_car(1024 * 4, 3, roots.clone(), blocks.clone())).unwrap(); - assert_eq!(forest_car.roots(), &roots); + assert_eq!(forest_car.head_tipset_key(), &roots); for block in blocks { assert_eq!(forest_car.get(&block.cid).unwrap(), Some(block.data)); } @@ -507,7 +529,7 @@ mod tests { blocks.clone(), )) .unwrap(); - assert_eq!(forest_car.roots(), &roots); + assert_eq!(forest_car.head_tipset_key(), &roots); for block in blocks { assert_eq!(forest_car.get(&block.cid).unwrap(), Some(block.data)); } diff --git a/src/db/memory.rs b/src/db/memory.rs index 44aadb62f5cb..d091ebc9ba8f 100644 --- a/src/db/memory.rs +++ b/src/db/memory.rs @@ -215,7 +215,7 @@ mod tests { db.export_forest_car(&mut car_db_bytes).await.unwrap(); let car = ForestCar::new(car_db_bytes).unwrap(); - assert_eq!(car.roots(), &nonempty![key1]); + assert_eq!(car.head_tipset_key(), &nonempty![key1]); assert!(car.has(&key1).unwrap()); assert!(car.has(&key2).unwrap()); } diff --git a/src/rpc/methods/chain.rs b/src/rpc/methods/chain.rs index b0704c041995..ddbd2382b0b6 100644 --- a/src/rpc/methods/chain.rs +++ b/src/rpc/methods/chain.rs @@ -293,6 +293,79 @@ impl RpcMethod<1> for ChainExport { } } +pub enum ChainExportV2 {} +impl RpcMethod<1> for ChainExportV2 { + const NAME: &'static str = "Filecoin.ChainExportV2"; + const PARAM_NAMES: [&'static str; 1] = ["params"]; + const API_PATHS: BitFlags = ApiPaths::all(); + const PERMISSION: Permission = Permission::Read; + + type Params = (ChainExportParams,); + type Ok = Option; + + async fn handle( + ctx: Ctx, + (params,): Self::Params, + ) -> Result { + let ChainExportParams { + epoch, + recent_roots, + output_path, + tipset_keys: ApiTipsetKey(tsk), + skip_checksum, + dry_run, + } = params; + + static LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); + + let _locked = LOCK.try_lock(); + if _locked.is_err() { + return Err(anyhow::anyhow!("Another chain export job is still in progress").into()); + } + + let chain_finality = ctx.chain_config().policy.chain_finality; + if recent_roots < chain_finality { + return Err(anyhow::anyhow!(format!( + "recent-stateroots must be greater than {chain_finality}" + )) + .into()); + } + + let head = ctx.chain_store().load_required_tipset_or_heaviest(&tsk)?; + let start_ts = + ctx.chain_index() + .tipset_by_height(epoch, head, ResolveNullTipset::TakeOlder)?; + + match if dry_run { + crate::chain::export_v2::( + &ctx.store_owned(), + None, + &start_ts, + recent_roots, + VoidAsyncWriter, + CidHashSet::default(), + skip_checksum, + ) + .await + } else { + let file = tokio::fs::File::create(&output_path).await?; + crate::chain::export_v2::( + &ctx.store_owned(), + None, + &start_ts, + recent_roots, + file, + CidHashSet::default(), + skip_checksum, + ) + .await + } { + Ok(checksum_opt) => Ok(checksum_opt.map(|hash| hash.encode_hex())), + Err(e) => Err(anyhow::anyhow!(e).into()), + } + } +} + pub enum ChainReadObj {} impl RpcMethod<1> for ChainReadObj { const NAME: &'static str = "Filecoin.ChainReadObj"; diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index d37221c23a5c..96df3820591c 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -59,6 +59,7 @@ macro_rules! for_each_rpc_method { // chain vertical $callback!($crate::rpc::chain::ChainPruneSnapshot); $callback!($crate::rpc::chain::ChainExport); + $callback!($crate::rpc::chain::ChainExportV2); $callback!($crate::rpc::chain::ChainGetBlock); $callback!($crate::rpc::chain::ChainGetBlockMessages); $callback!($crate::rpc::chain::ChainGetEvents); diff --git a/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt b/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt index d665bb42e907..b605ab1af2e5 100644 --- a/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt +++ b/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt @@ -11,6 +11,7 @@ F3.SignMessage Filecoin.AuthNew Filecoin.AuthVerify Filecoin.ChainExport +Filecoin.ChainExportV2 Filecoin.ChainGetEvents Filecoin.ChainSetHead Filecoin.EthEstimateGas diff --git a/src/tool/subcommands/archive_cmd.rs b/src/tool/subcommands/archive_cmd.rs index a94ab544786a..816ece05eb15 100644 --- a/src/tool/subcommands/archive_cmd.rs +++ b/src/tool/subcommands/archive_cmd.rs @@ -171,9 +171,25 @@ impl ArchiveCommands { let variant = store.variant().to_string(); let heaviest = store.heaviest_tipset()?; let index_size_bytes = store.index_size_bytes(); + let snapshot_version = match &store { + AnyCar::Forest(car) => { + if let Some(metadata) = car.metadata() { + metadata.version + } else { + 1 + } + } + _ => 1, + }; println!( "{}", - ArchiveInfo::from_store(&store, variant, heaviest, index_size_bytes)? + ArchiveInfo::from_store( + &store, + variant, + heaviest, + snapshot_version, + index_size_bytes + )? ); Ok(()) } @@ -230,29 +246,31 @@ pub struct ArchiveInfo { epoch: ChainEpoch, tipsets: ChainEpoch, messages: ChainEpoch, - root: Tipset, + head: Tipset, + snapshot_version: u64, index_size_bytes: Option, } impl std::fmt::Display for ArchiveInfo { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - writeln!(f, "CAR format: {}", self.variant)?; - writeln!(f, "Network: {}", self.network)?; - writeln!(f, "Epoch: {}", self.epoch)?; - writeln!(f, "State-roots: {}", self.epoch - self.tipsets + 1)?; - writeln!(f, "Messages sets: {}", self.epoch - self.messages + 1)?; - let root_cids_string = self - .root + writeln!(f, "CAR format: {}", self.variant)?; + writeln!(f, "Snapshot version: {}", self.snapshot_version)?; + writeln!(f, "Network: {}", self.network)?; + writeln!(f, "Epoch: {}", self.epoch)?; + writeln!(f, "State-roots: {}", self.epoch - self.tipsets + 1)?; + writeln!(f, "Messages sets: {}", self.epoch - self.messages + 1)?; + let head_tipset_key_string = self + .head .cids() .iter() .map(Cid::to_string) - .join("\n "); - write!(f, "Root CIDs: {root_cids_string}")?; + .join("\n "); + write!(f, "Head Tipset: {head_tipset_key_string}")?; if let Some(index_size_bytes) = self.index_size_bytes { writeln!(f)?; write!( f, - "Index size: {}", + "Index size: {}", human_bytes::human_bytes(index_size_bytes) )?; } @@ -267,9 +285,17 @@ impl ArchiveInfo { store: &impl Blockstore, variant: String, heaviest_tipset: Tipset, + snapshot_version: u64, index_size_bytes: Option, ) -> anyhow::Result { - Self::from_store_with(store, variant, heaviest_tipset, index_size_bytes, true) + Self::from_store_with( + store, + variant, + heaviest_tipset, + snapshot_version, + index_size_bytes, + true, + ) } // Scan a CAR archive to identify which network it belongs to and how many @@ -279,15 +305,16 @@ impl ArchiveInfo { store: &impl Blockstore, variant: String, heaviest_tipset: Tipset, + snapshot_version: u64, index_size_bytes: Option, progress: bool, ) -> anyhow::Result { - let root = heaviest_tipset; - let root_epoch = root.epoch(); + let head = heaviest_tipset; + let root_epoch = head.epoch(); - let tipsets = root.clone().chain(store); + let tipsets = head.clone().chain(store); - let windowed = (std::iter::once(root.clone()).chain(tipsets)).tuple_windows(); + let windowed = (std::iter::once(head.clone()).chain(tipsets)).tuple_windows(); let mut network: String = "unknown".into(); let mut lowest_stateroot_epoch = root_epoch; @@ -353,7 +380,8 @@ impl ArchiveInfo { epoch: root_epoch, tipsets: lowest_stateroot_epoch, messages: lowest_message_epoch, - root, + head, + snapshot_version, index_size_bytes, }) } @@ -862,8 +890,13 @@ async fn sync_bucket( let store = Arc::new(ManyCar::try_from(snapshot_files)?); let heaviest_tipset = store.heaviest_tipset()?; - let info = - ArchiveInfo::from_store(&store, "ManyCAR".to_string(), heaviest_tipset.clone(), None)?; + let info = ArchiveInfo::from_store( + &store, + "ManyCAR".to_string(), + heaviest_tipset.clone(), + 1, + None, + )?; let genesis_timestamp = heaviest_tipset.genesis(&store)?.timestamp; @@ -998,7 +1031,7 @@ mod tests { let variant = store.variant().to_string(); let ts = store.heaviest_tipset().unwrap(); let index_size_bytes = store.index_size_bytes(); - let info = ArchiveInfo::from_store(&store, variant, ts, index_size_bytes).unwrap(); + let info = ArchiveInfo::from_store(&store, variant, ts, 1, index_size_bytes).unwrap(); assert_eq!(info.network, "calibnet"); assert_eq!(info.epoch, 0); } @@ -1009,7 +1042,7 @@ mod tests { let variant = store.variant().to_string(); let ts = store.heaviest_tipset().unwrap(); let index_size_bytes = store.index_size_bytes(); - let info = ArchiveInfo::from_store(&store, variant, ts, index_size_bytes).unwrap(); + let info = ArchiveInfo::from_store(&store, variant, ts, 1, index_size_bytes).unwrap(); assert_eq!(info.network, "mainnet"); assert_eq!(info.epoch, 0); } diff --git a/src/utils/db/car_stream.rs b/src/utils/db/car_stream.rs index 434d25ab122d..e5bc41cd9e0c 100644 --- a/src/utils/db/car_stream.rs +++ b/src/utils/db/car_stream.rs @@ -13,7 +13,7 @@ use integer_encoding::VarInt; use nunny::Vec as NonEmpty; use pin_project_lite::pin_project; use serde::{Deserialize, Serialize}; -use std::io::{self, SeekFrom}; +use std::io::{self, Cursor, Read, SeekFrom, Write}; use std::pin::Pin; use std::task::{Context, Poll}; use tokio::io::{ @@ -53,15 +53,8 @@ pub struct CarBlock { impl CarBlock { // Write a varint frame containing the cid and the data - pub fn write(&self, mut writer: &mut impl io::Write) -> io::Result<()> { - let frame_length = self.cid.encoded_len() + self.data.len(); - writer.write_all(&frame_length.encode_var_vec())?; - #[allow(clippy::needless_borrows_for_generic_args)] - self.cid - .write_bytes(&mut writer) - .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; - writer.write_all(&self.data)?; - Ok(()) + pub fn write(&self, writer: &mut impl Write) -> io::Result<()> { + writer.write_car_block(self.cid, self.data.len(), &mut Cursor::new(&self.data)) } pub fn from_bytes(bytes: impl Into) -> io::Result { @@ -94,6 +87,31 @@ impl CarBlock { } } +pub trait CarBlockWrite { + fn write_car_block( + &mut self, + cid: Cid, + data_len: usize, + data: &mut impl Read, + ) -> io::Result<()>; +} + +impl CarBlockWrite for T { + fn write_car_block( + &mut self, + cid: Cid, + data_len: usize, + data: &mut impl Read, + ) -> io::Result<()> { + let frame_length = cid.encoded_len() + data_len; + self.write_all(&frame_length.encode_var_vec())?; + cid.write_bytes(&mut *self) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + std::io::copy(data, self)?; + Ok(()) + } +} + pin_project! { /// Stream of CAR blocks. If the input data is compressed with zstd, it will /// automatically be decompressed. From 972914f140fe8e3dbaa864abed7768f211e51e33 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 16 Jul 2025 18:21:54 +0800 Subject: [PATCH 02/35] snapshot metadata API for plain CAR --- src/db/car/any.rs | 9 ++++++++ src/db/car/forest.rs | 7 ++++-- src/db/car/plain.rs | 36 ++++++++++++++++++++++++----- src/tool/subcommands/archive_cmd.rs | 13 ++++------- 4 files changed, 48 insertions(+), 17 deletions(-) diff --git a/src/db/car/any.rs b/src/db/car/any.rs index 36e544fd40dd..334a4248bb32 100644 --- a/src/db/car/any.rs +++ b/src/db/car/any.rs @@ -10,6 +10,7 @@ use super::{CacheKey, RandomAccessFileReader, ZstdFrameCache}; use crate::blocks::{Tipset, TipsetKey}; +use crate::chain::FilecoinSnapshotMetadata; use crate::db::PersistentStore; use crate::utils::io::EitherMmapOrRandomAccessFile; use cid::Cid; @@ -52,6 +53,14 @@ impl AnyCar { )) } + pub fn metadata(&self) -> &Option { + match self { + AnyCar::Forest(forest) => forest.metadata(), + AnyCar::Plain(plain) => plain.metadata(), + AnyCar::Memory(mem) => mem.metadata(), + } + } + pub fn heaviest_tipset_key(&self) -> TipsetKey { match self { AnyCar::Forest(forest) => forest.heaviest_tipset_key(), diff --git a/src/db/car/forest.rs b/src/db/car/forest.rs index 6abee3881ba0..68d96e8be5b2 100644 --- a/src/db/car/forest.rs +++ b/src/db/car/forest.rs @@ -61,7 +61,7 @@ use bytes::{BufMut as _, Bytes, BytesMut, buf::Writer}; use cid::Cid; use futures::{Stream, TryStreamExt as _}; use fvm_ipld_blockstore::Blockstore; -use fvm_ipld_encoding::{CborStore, to_vec}; +use fvm_ipld_encoding::CborStore as _; use nunny::Vec as NonEmpty; use parking_lot::{Mutex, RwLock}; use positioned_io::{Cursor, ReadAt, ReadBytesAtExt, SizeCursor}; @@ -314,7 +314,10 @@ impl Encoder { let header = CarV1Header { roots, version: 1 }; let mut header_uvi_frame = BytesMut::new(); - UviBytes::default().encode(Bytes::from(to_vec(&header)?), &mut header_uvi_frame)?; + UviBytes::default().encode( + Bytes::from(fvm_ipld_encoding::to_vec(&header)?), + &mut header_uvi_frame, + )?; header_encoder.write_all(&header_uvi_frame)?; let header_bytes = header_encoder.finish()?.into_inner().freeze(); diff --git a/src/db/car/plain.rs b/src/db/car/plain.rs index ac4c541b819b..1f33dad07cc5 100644 --- a/src/db/car/plain.rs +++ b/src/db/car/plain.rs @@ -60,6 +60,7 @@ //! - CARv2 support //! - A wrapper that abstracts over car formats for reading. +use crate::chain::FilecoinSnapshotMetadata; use crate::cid_collections::{CidHashMap, hash_map::Entry as CidHashMapEntry}; use crate::db::PersistentStore; use crate::utils::db::car_stream::{CarV1Header, CarV2Header}; @@ -70,11 +71,11 @@ use crate::{ use CidHashMapEntry::{Occupied, Vacant}; use cid::Cid; use fvm_ipld_blockstore::Blockstore; +use fvm_ipld_encoding::CborStore as _; use integer_encoding::{FixedIntReader, VarIntReader}; use nunny::Vec as NonEmpty; use parking_lot::RwLock; use positioned_io::ReadAt; -use std::ops::DerefMut; use std::{ any::Any, io::{ @@ -83,6 +84,8 @@ use std::{ Read, Seek, SeekFrom, }, iter, + ops::DerefMut, + sync::OnceLock, }; use tokio::io::{AsyncWrite, AsyncWriteExt}; use tracing::{debug, trace}; @@ -115,6 +118,7 @@ pub struct PlainCar { version: u64, header_v1: CarV1Header, header_v2: Option, + metadata: OnceLock>, } impl PlainCar { @@ -167,13 +171,32 @@ impl PlainCar { version, header_v1, header_v2, + metadata: OnceLock::new(), }) } } } - pub fn roots(&self) -> &NonEmpty { - &self.header_v1.roots + pub fn metadata(&self) -> &Option { + self.metadata.get_or_init(|| { + if self.header_v1.roots.len() == 1 { + let maybe_metadata_cid = self.header_v1.roots.first(); + if let Ok(Some(metadata)) = + self.get_cbor::(maybe_metadata_cid) + { + return Some(metadata); + } + } + None + }) + } + + pub fn head_tipset_key(&self) -> &NonEmpty { + if let Some(metadata) = self.metadata() { + &metadata.head_tipset_key + } else { + &self.header_v1.roots + } } pub fn version(&self) -> u64 { @@ -181,7 +204,7 @@ impl PlainCar { } pub fn heaviest_tipset_key(&self) -> TipsetKey { - TipsetKey::from(self.roots().clone()) + TipsetKey::from(self.head_tipset_key().clone()) } pub fn heaviest_tipset(&self) -> anyhow::Result { @@ -202,6 +225,7 @@ impl PlainCar { version: self.version, header_v1: self.header_v1, header_v2: self.header_v2, + metadata: self.metadata, } } } @@ -503,7 +527,7 @@ mod tests { let car_backed = PlainCar::new(car).unwrap(); assert_eq!(car_backed.version(), 1); - assert_eq!(car_backed.roots().len(), 1); + assert_eq!(car_backed.head_tipset_key().len(), 1); assert_eq!(car_backed.cids().len(), 1222); let reference_car = reference(Cursor::new(car)); @@ -526,7 +550,7 @@ mod tests { let car_backed = PlainCar::new(car).unwrap(); assert_eq!(car_backed.version(), 2); - assert_eq!(car_backed.roots().len(), 1); + assert_eq!(car_backed.head_tipset_key().len(), 1); assert_eq!(car_backed.cids().len(), 7153); let reference_car = reference(Cursor::new(car)); diff --git a/src/tool/subcommands/archive_cmd.rs b/src/tool/subcommands/archive_cmd.rs index 816ece05eb15..e7c3642984bd 100644 --- a/src/tool/subcommands/archive_cmd.rs +++ b/src/tool/subcommands/archive_cmd.rs @@ -171,15 +171,10 @@ impl ArchiveCommands { let variant = store.variant().to_string(); let heaviest = store.heaviest_tipset()?; let index_size_bytes = store.index_size_bytes(); - let snapshot_version = match &store { - AnyCar::Forest(car) => { - if let Some(metadata) = car.metadata() { - metadata.version - } else { - 1 - } - } - _ => 1, + let snapshot_version = if let Some(metadata) = store.metadata() { + metadata.version + } else { + 1 }; println!( "{}", From 6408398d1a3fed3ec932995f405630335a48e99d Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 16 Jul 2025 18:56:45 +0800 Subject: [PATCH 03/35] forest-tool archive metadata --- CHANGELOG.md | 6 +++++- scripts/tests/calibnet_export_check.sh | 10 +++++++++- src/chain/mod.rs | 21 +++++++++++++++++++++ src/tool/subcommands/archive_cmd.rs | 16 +++++++++++++++- 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a25d2eec3449..d1433998578a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,11 @@ - [#5739](https://github.com/ChainSafe/forest/issues/5739) Add `--export-mode` flag to the `forest-tool archive sync-bucket` subcommand. This allows exporting and uploading only the required files. -- [#5778](https://github.com/ChainSafe/forest/pull/5778) Feat generate a detailed test report in `api compare` command through `--report-dir` and `--report-mode` +- [#5835](https://github.com/ChainSafe/forest/issues/5835) Add `--v2` flag to the `forest-cli snapshot export` subcommand. This allows exporting a Filecoin snapshot in v2 format(FRC-0108). + +- [#5835](https://github.com/ChainSafe/forest/issues/5835) Add `forest-tool archive metadata` subcommand for inspecting snapshot metadata of a Filecoin snapshot in v2 format(FRC-0108). + +- [#5778](https://github.com/ChainSafe/forest/pull/5778) Feat generate a detailed test report in `api compare` command through `--report-dir` and `--report-mode`. ### Changed diff --git a/scripts/tests/calibnet_export_check.sh b/scripts/tests/calibnet_export_check.sh index 621dd5e070c8..2460ffbbeae2 100755 --- a/scripts/tests/calibnet_export_check.sh +++ b/scripts/tests/calibnet_export_check.sh @@ -13,11 +13,19 @@ echo "Cleaning up the initial snapshot" rm --force --verbose ./*.{car,car.zst,sha256sum} echo "Exporting zstd compressed snapshot" -$FOREST_CLI_PATH snapshot export +$FOREST_CLI_PATH snapshot export -o v1.forest.car.zst echo "Exporting zstd compressed snapshot in the experimetal v2 format" $FOREST_CLI_PATH snapshot export --v2 -o v2.forest.car.zst +echo "Inspecting archive info and metadata" +for f in *.car.zst; do + echo "Inspecting archive info $f" + $FOREST_TOOL_PATH archive info "$f" + echo "Inspecting archive metadata $f" + $FOREST_TOOL_PATH archive metadata "$f" +done + echo "Testing snapshot validity" zstd --test ./*.car.zst diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 85f8a33c52f4..b845f31f39fe 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -17,6 +17,7 @@ use digest::Digest; use futures::StreamExt as _; use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::DAG_CBOR; +use itertools::Itertools as _; use multihash_derive::MultihashDigest as _; use nunny::Vec as NonEmpty; use serde::{Deserialize, Serialize}; @@ -176,3 +177,23 @@ impl FilecoinSnapshotMetadata { } } } + +impl std::fmt::Display for FilecoinSnapshotMetadata { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f, "Snapshot version: {}", self.version)?; + let head_tipset_key_string = self + .head_tipset_key + .iter() + .map(Cid::to_string) + .join("\n "); + writeln!(f, "Head Tipset: {head_tipset_key_string}")?; + write!( + f, + "F3 data: {}", + self.f3_data + .map(|c| c.to_string()) + .unwrap_or_else(|| "not found".into()) + )?; + Ok(()) + } +} diff --git a/src/tool/subcommands/archive_cmd.rs b/src/tool/subcommands/archive_cmd.rs index e7c3642984bd..8c4c277b6b4e 100644 --- a/src/tool/subcommands/archive_cmd.rs +++ b/src/tool/subcommands/archive_cmd.rs @@ -82,7 +82,12 @@ impl ExportMode { pub enum ArchiveCommands { /// Show basic information about an archive. Info { - /// Path to an uncompressed archive (CAR) + /// Path to an archive (`.car` or `.car.zst`). + snapshot: PathBuf, + }, + /// Show FRC-0108 metadata of an Filecoin snapshot archive. + Metadata { + /// Path to an archive (`.car` or `.car.zst`). snapshot: PathBuf, }, /// Trim a snapshot of the chain and write it to `` @@ -188,6 +193,15 @@ impl ArchiveCommands { ); Ok(()) } + Self::Metadata { snapshot } => { + let store = AnyCar::try_from(snapshot.as_path())?; + if let Some(metadata) = store.metadata() { + println!("{metadata}"); + } else { + println!("No FRC-0108 metadata found in the snapshot"); + } + Ok(()) + } Self::Export { snapshot_files, output_path, From d07fa89ff0d83f22b6cec6e7cc104571c63cc95c Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 16 Jul 2025 19:01:57 +0800 Subject: [PATCH 04/35] fix link checker --- .config/lychee.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.config/lychee.toml b/.config/lychee.toml index f19f47c0dc8c..a2d5d01419f7 100644 --- a/.config/lychee.toml +++ b/.config/lychee.toml @@ -17,6 +17,8 @@ exclude = [ "filecoin.io/slack", # Bot protection / 403 Forbidden errors "crates.io", + # Bot protection / 403 Forbidden errors + "linuxhint.com", ] timeout = 30 max_retries = 6 From c370d04ecd62e6c3708732b7cfba6c55fde75078 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 16 Jul 2025 19:32:40 +0800 Subject: [PATCH 05/35] fix typo --- scripts/tests/calibnet_export_check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/calibnet_export_check.sh b/scripts/tests/calibnet_export_check.sh index 2460ffbbeae2..4114880447e6 100755 --- a/scripts/tests/calibnet_export_check.sh +++ b/scripts/tests/calibnet_export_check.sh @@ -15,7 +15,7 @@ rm --force --verbose ./*.{car,car.zst,sha256sum} echo "Exporting zstd compressed snapshot" $FOREST_CLI_PATH snapshot export -o v1.forest.car.zst -echo "Exporting zstd compressed snapshot in the experimetal v2 format" +echo "Exporting zstd compressed snapshot in the experimental v2 format" $FOREST_CLI_PATH snapshot export --v2 -o v2.forest.car.zst echo "Inspecting archive info and metadata" From 918d5906e5846f736a1eb85f89abb95576cffb65 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 22 Jul 2025 16:09:00 +0800 Subject: [PATCH 06/35] resolve comments --- CHANGELOG.md | 2 +- scripts/tests/calibnet_export_check.sh | 2 +- src/chain/mod.rs | 60 ++++++++++++++++++- src/cli/subcommands/snapshot_cmd.rs | 31 ++++++---- src/db/car/forest.rs | 2 + src/db/car/plain.rs | 2 + src/rpc/methods/chain.rs | 2 +- .../api_cmd/test_snapshots_ignored.txt | 2 +- src/tool/subcommands/archive_cmd.rs | 31 +++++++--- 9 files changed, 108 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18d9519036ff..d755bb1e51ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ - [#5739](https://github.com/ChainSafe/forest/issues/5739) Add `--export-mode` flag to the `forest-tool archive sync-bucket` subcommand. This allows exporting and uploading only the required files. -- [#5835](https://github.com/ChainSafe/forest/issues/5835) Add `--v2` flag to the `forest-cli snapshot export` subcommand. This allows exporting a Filecoin snapshot in v2 format(FRC-0108). +- [#5835](https://github.com/ChainSafe/forest/issues/5835) Add `--format` flag to the `forest-cli snapshot export` subcommand. This allows exporting a Filecoin snapshot in v2 format(FRC-0108). - [#5835](https://github.com/ChainSafe/forest/issues/5835) Add `forest-tool archive metadata` subcommand for inspecting snapshot metadata of a Filecoin snapshot in v2 format(FRC-0108). diff --git a/scripts/tests/calibnet_export_check.sh b/scripts/tests/calibnet_export_check.sh index 4114880447e6..e9102daf1913 100755 --- a/scripts/tests/calibnet_export_check.sh +++ b/scripts/tests/calibnet_export_check.sh @@ -16,7 +16,7 @@ echo "Exporting zstd compressed snapshot" $FOREST_CLI_PATH snapshot export -o v1.forest.car.zst echo "Exporting zstd compressed snapshot in the experimental v2 format" -$FOREST_CLI_PATH snapshot export --v2 -o v2.forest.car.zst +$FOREST_CLI_PATH snapshot export --format v2 -o v2.forest.car.zst echo "Inspecting archive info and metadata" for f in *.car.zst; do diff --git a/src/chain/mod.rs b/src/chain/mod.rs index b845f31f39fe..6113b11e2377 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -19,6 +19,8 @@ use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::DAG_CBOR; use itertools::Itertools as _; use multihash_derive::MultihashDigest as _; +use num::FromPrimitive as _; +use num_derive::FromPrimitive; use nunny::Vec as NonEmpty; use serde::{Deserialize, Serialize}; use std::fs::File; @@ -159,19 +161,51 @@ pub async fn export_v2( Ok(digest) } +#[derive(Debug, Copy, FromPrimitive, Clone, PartialEq, Eq)] +#[repr(u64)] +pub enum FilecoinSnapshotVersion { + V1 = 1, + V2 = 2, +} + +impl Serialize for FilecoinSnapshotVersion { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_u64(*self as u64) + } +} + +impl<'de> Deserialize<'de> for FilecoinSnapshotVersion { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let i = u64::deserialize(deserializer)?; + match FilecoinSnapshotVersion::from_u64(i) { + Some(v) => Ok(v), + None => Err(serde::de::Error::custom("invalid snapshot version {i}")), + } + } +} + /// #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "PascalCase")] pub struct FilecoinSnapshotMetadata { - pub version: u64, + /// Snapshot version + pub version: FilecoinSnapshotVersion, + /// Chain head tipset key pub head_tipset_key: NonEmpty, + /// F3 snapshot `CID` pub f3_data: Option, } impl FilecoinSnapshotMetadata { pub fn new_v2(head_tipset_key: NonEmpty, f3_data: Option) -> Self { Self { - version: 2, + version: FilecoinSnapshotVersion::V2, head_tipset_key, f3_data, } @@ -180,7 +214,7 @@ impl FilecoinSnapshotMetadata { impl std::fmt::Display for FilecoinSnapshotMetadata { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - writeln!(f, "Snapshot version: {}", self.version)?; + writeln!(f, "Snapshot version: {}", self.version as u64)?; let head_tipset_key_string = self .head_tipset_key .iter() @@ -197,3 +231,23 @@ impl std::fmt::Display for FilecoinSnapshotMetadata { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_snapshot_version_cbor_serde() { + assert_eq!( + fvm_ipld_encoding::to_vec(&FilecoinSnapshotVersion::V2), + fvm_ipld_encoding::to_vec(&2_u64) + ); + assert_eq!( + fvm_ipld_encoding::from_slice::( + &fvm_ipld_encoding::to_vec(&2_u64).unwrap() + ) + .unwrap(), + FilecoinSnapshotVersion::V2 + ); + } +} diff --git a/src/cli/subcommands/snapshot_cmd.rs b/src/cli/subcommands/snapshot_cmd.rs index 44631fc2cb94..1b2959c8f2f9 100644 --- a/src/cli/subcommands/snapshot_cmd.rs +++ b/src/cli/subcommands/snapshot_cmd.rs @@ -16,6 +16,12 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use tokio::io::AsyncWriteExt; +#[derive(Debug, Clone, clap::ValueEnum)] +pub enum SnapshotFormat { + V1, + V2, +} + #[derive(Debug, Subcommand)] pub enum SnapshotCommands { /// Export a snapshot of the chain to `` @@ -36,8 +42,8 @@ pub enum SnapshotCommands { #[arg(short, long)] depth: Option, /// Export snapshot in the experimental v2 format(FRC-0108). - #[arg(long)] - v2: bool, + #[arg(long, value_enum, default_value_t = SnapshotFormat::V1)] + format: SnapshotFormat, }, } @@ -50,7 +56,7 @@ impl SnapshotCommands { dry_run, tipset, depth, - v2, + format, } => { let chain_head = ChainHead::call(&client, ()).await?; @@ -123,14 +129,17 @@ impl SnapshotCommands { }); // Manually construct RpcRequest because snapshot export could // take a few hours on mainnet - let hash_result = if v2 { - client - .call(ChainExportV2::request((params,))?.with_timeout(Duration::MAX)) - .await? - } else { - client - .call(ChainExport::request((params,))?.with_timeout(Duration::MAX)) - .await? + let hash_result = match format { + SnapshotFormat::V1 => { + client + .call(ChainExport::request((params,))?.with_timeout(Duration::MAX)) + .await? + } + SnapshotFormat::V2 => { + client + .call(ChainExportV2::request((params,))?.with_timeout(Duration::MAX)) + .await? + } }; handle.abort(); diff --git a/src/db/car/forest.rs b/src/db/car/forest.rs index 68d96e8be5b2..d7df2632fe6d 100644 --- a/src/db/car/forest.rs +++ b/src/db/car/forest.rs @@ -126,6 +126,7 @@ impl ForestCar { pub fn metadata(&self) -> &Option { self.metadata.get_or_init(|| { + // if self.header.roots.len() == 1 { let maybe_metadata_cid = self.header.roots.first(); if let Ok(Some(metadata)) = @@ -167,6 +168,7 @@ impl ForestCar { } pub fn head_tipset_key(&self) -> &NonEmpty { + // if let Some(metadata) = self.metadata() { &metadata.head_tipset_key } else { diff --git a/src/db/car/plain.rs b/src/db/car/plain.rs index 1f33dad07cc5..cc26720bb450 100644 --- a/src/db/car/plain.rs +++ b/src/db/car/plain.rs @@ -179,6 +179,7 @@ impl PlainCar { pub fn metadata(&self) -> &Option { self.metadata.get_or_init(|| { + // if self.header_v1.roots.len() == 1 { let maybe_metadata_cid = self.header_v1.roots.first(); if let Ok(Some(metadata)) = @@ -192,6 +193,7 @@ impl PlainCar { } pub fn head_tipset_key(&self) -> &NonEmpty { + // if let Some(metadata) = self.metadata() { &metadata.head_tipset_key } else { diff --git a/src/rpc/methods/chain.rs b/src/rpc/methods/chain.rs index ddbd2382b0b6..2d5e934442b1 100644 --- a/src/rpc/methods/chain.rs +++ b/src/rpc/methods/chain.rs @@ -295,7 +295,7 @@ impl RpcMethod<1> for ChainExport { pub enum ChainExportV2 {} impl RpcMethod<1> for ChainExportV2 { - const NAME: &'static str = "Filecoin.ChainExportV2"; + const NAME: &'static str = "Forest.ChainExportV2"; const PARAM_NAMES: [&'static str; 1] = ["params"]; const API_PATHS: BitFlags = ApiPaths::all(); const PERMISSION: Permission = Permission::Read; diff --git a/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt b/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt index b605ab1af2e5..2aa8acd36975 100644 --- a/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt +++ b/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt @@ -11,7 +11,6 @@ F3.SignMessage Filecoin.AuthNew Filecoin.AuthVerify Filecoin.ChainExport -Filecoin.ChainExportV2 Filecoin.ChainGetEvents Filecoin.ChainSetHead Filecoin.EthEstimateGas @@ -79,6 +78,7 @@ Filecoin.WalletSignMessage Filecoin.WalletValidateAddress Filecoin.WalletVerify Filecoin.Web3ClientVersion +Forest.ChainExportV2 Forest.ChainGetMinBaseFee Forest.NetInfo Forest.SnapshotGC diff --git a/src/tool/subcommands/archive_cmd.rs b/src/tool/subcommands/archive_cmd.rs index 8c4c277b6b4e..b8d7f6714511 100644 --- a/src/tool/subcommands/archive_cmd.rs +++ b/src/tool/subcommands/archive_cmd.rs @@ -27,6 +27,7 @@ //! Additional reading: [`crate::db::car::plain`] use crate::blocks::Tipset; +use crate::chain::FilecoinSnapshotVersion; use crate::chain::{ ChainEpochDelta, index::{ChainIndex, ResolveNullTipset}, @@ -179,7 +180,7 @@ impl ArchiveCommands { let snapshot_version = if let Some(metadata) = store.metadata() { metadata.version } else { - 1 + FilecoinSnapshotVersion::V1 }; println!( "{}", @@ -256,14 +257,14 @@ pub struct ArchiveInfo { tipsets: ChainEpoch, messages: ChainEpoch, head: Tipset, - snapshot_version: u64, + snapshot_version: FilecoinSnapshotVersion, index_size_bytes: Option, } impl std::fmt::Display for ArchiveInfo { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { writeln!(f, "CAR format: {}", self.variant)?; - writeln!(f, "Snapshot version: {}", self.snapshot_version)?; + writeln!(f, "Snapshot version: {}", self.snapshot_version as u64)?; writeln!(f, "Network: {}", self.network)?; writeln!(f, "Epoch: {}", self.epoch)?; writeln!(f, "State-roots: {}", self.epoch - self.tipsets + 1)?; @@ -294,7 +295,7 @@ impl ArchiveInfo { store: &impl Blockstore, variant: String, heaviest_tipset: Tipset, - snapshot_version: u64, + snapshot_version: FilecoinSnapshotVersion, index_size_bytes: Option, ) -> anyhow::Result { Self::from_store_with( @@ -314,7 +315,7 @@ impl ArchiveInfo { store: &impl Blockstore, variant: String, heaviest_tipset: Tipset, - snapshot_version: u64, + snapshot_version: FilecoinSnapshotVersion, index_size_bytes: Option, progress: bool, ) -> anyhow::Result { @@ -903,7 +904,7 @@ async fn sync_bucket( &store, "ManyCAR".to_string(), heaviest_tipset.clone(), - 1, + FilecoinSnapshotVersion::V1, None, )?; @@ -1040,7 +1041,14 @@ mod tests { let variant = store.variant().to_string(); let ts = store.heaviest_tipset().unwrap(); let index_size_bytes = store.index_size_bytes(); - let info = ArchiveInfo::from_store(&store, variant, ts, 1, index_size_bytes).unwrap(); + let info = ArchiveInfo::from_store( + &store, + variant, + ts, + FilecoinSnapshotVersion::V1, + index_size_bytes, + ) + .unwrap(); assert_eq!(info.network, "calibnet"); assert_eq!(info.epoch, 0); } @@ -1051,7 +1059,14 @@ mod tests { let variant = store.variant().to_string(); let ts = store.heaviest_tipset().unwrap(); let index_size_bytes = store.index_size_bytes(); - let info = ArchiveInfo::from_store(&store, variant, ts, 1, index_size_bytes).unwrap(); + let info = ArchiveInfo::from_store( + &store, + variant, + ts, + FilecoinSnapshotVersion::V1, + index_size_bytes, + ) + .unwrap(); assert_eq!(info.network, "mainnet"); assert_eq!(info.epoch, 0); } From b8f8e6d10663588883bf038c4c806949e5f5b99d Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 22 Jul 2025 19:25:19 +0800 Subject: [PATCH 07/35] address AI comment --- src/chain/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 6113b11e2377..2f957f0051e5 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -185,7 +185,9 @@ impl<'de> Deserialize<'de> for FilecoinSnapshotVersion { let i = u64::deserialize(deserializer)?; match FilecoinSnapshotVersion::from_u64(i) { Some(v) => Ok(v), - None => Err(serde::de::Error::custom("invalid snapshot version {i}")), + None => Err(serde::de::Error::custom(format!( + "invalid snapshot version {i}" + ))), } } } From f3ccbdbbd2619e6372d4748673f3d383f064b585 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 29 Jul 2025 20:18:41 +0800 Subject: [PATCH 08/35] resolve comments --- CHANGELOG.md | 4 +- src/chain/mod.rs | 91 ++++++++++------- src/cli/subcommands/snapshot_cmd.rs | 34 ++----- src/db/car/forest.rs | 14 ++- src/rpc/methods/chain.rs | 147 +++++++++++++--------------- src/tool/subcommands/archive_cmd.rs | 6 +- 6 files changed, 144 insertions(+), 152 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad8cc4de9857..b583b84d10b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ - [#5835](https://github.com/ChainSafe/forest/issues/5835) Add `--format` flag to the `forest-cli snapshot export` subcommand. This allows exporting a Filecoin snapshot in v2 format(FRC-0108). +- [#5835](https://github.com/ChainSafe/forest/issues/5835) Add `forest-tool archive metadata` subcommand for inspecting snapshot metadata of a Filecoin snapshot in v2 format(FRC-0108). + - [#5859](https://github.com/ChainSafe/forest/pull/5859) Added size metrics for zstd frame cache and made max size configurable via `FOREST_ZSTD_FRAME_CACHE_DEFAULT_MAX_SIZE` environment variable. ### Changed @@ -53,8 +55,6 @@ This is a non-mandatory release recommended for all node operators. It includes - [#5739](https://github.com/ChainSafe/forest/issues/5739) Add `--export-mode` flag to the `forest-tool archive sync-bucket` subcommand. This allows exporting and uploading only the required files. -- [#5835](https://github.com/ChainSafe/forest/issues/5835) Add `forest-tool archive metadata` subcommand for inspecting snapshot metadata of a Filecoin snapshot in v2 format(FRC-0108). - - [#5778](https://github.com/ChainSafe/forest/pull/5778) Feat generate a detailed test report in `api compare` command through `--report-dir` and `--report-mode`. ### Changed diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 2f957f0051e5..3209e2a865fb 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -4,9 +4,10 @@ pub mod store; mod weight; use crate::blocks::{Tipset, TipsetKey}; use crate::cid_collections::CidHashSet; -use crate::db::car::forest::{self, finalize_frame}; +use crate::db::car::forest::{self, ForestCarFrame, finalize_frame}; use crate::db::{SettingsStore, SettingsStoreExt}; use crate::ipld::stream_chain; +use crate::lotus_json::lotus_json_with_self; use crate::utils::db::car_stream::{CarBlock, CarBlockWrite}; use crate::utils::io::{AsyncWriterWithChecksum, Checksum}; use crate::utils::multihash::MultihashCode; @@ -22,6 +23,7 @@ use multihash_derive::MultihashDigest as _; use num::FromPrimitive as _; use num_derive::FromPrimitive; use nunny::Vec as NonEmpty; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::fs::File; use std::sync::Arc; @@ -51,39 +53,18 @@ pub async fn export( seen: CidHashSet, skip_checksum: bool, ) -> anyhow::Result>, Error> { - let stateroot_lookup_limit = tipset.epoch() - lookup_depth; let roots = tipset.key().to_cids(); - - // Wrap writer in optional checksum calculator - let mut writer = AsyncWriterWithChecksum::::new(BufWriter::new(writer), !skip_checksum); - - // Stream stateroots in range (stateroot_lookup_limit+1)..=tipset.epoch(). Also - // stream all block headers until genesis. - let blocks = par_buffer( - // Queue 1k blocks. This is enough to saturate the compressor and blocks - // are small enough that keeping 1k in memory isn't a problem. Average - // block size is between 1kb and 2kb. - 1024, - stream_chain( - Arc::clone(db), - tipset.clone().chain_owned(Arc::clone(db)), - stateroot_lookup_limit, - ) - .with_seen(seen), - ); - - // Encode Ipld key-value pairs in zstd frames - let frames = forest::Encoder::compress_stream_default(blocks); - - // Write zstd frames and include a skippable index - forest::Encoder::write(&mut writer, roots, frames).await?; - - // Flush to ensure everything has been successfully written - writer.flush().await.context("failed to flush")?; - - let digest = writer.finalize().map_err(|e| Error::Other(e.to_string()))?; - - Ok(digest) + export_to_forest_car::( + roots, + None, + db, + tipset, + lookup_depth, + writer, + seen, + skip_checksum, + ) + .await } pub async fn export_v2( @@ -95,7 +76,6 @@ pub async fn export_v2( seen: CidHashSet, skip_checksum: bool, ) -> anyhow::Result>, Error> { - let stateroot_lookup_limit = tipset.epoch() - lookup_depth; let head = tipset.key().to_cids(); let f3_cid = f3.as_ref().map(|(cid, _)| *cid); let snap_meta = FilecoinSnapshotMetadata::new_v2(head, f3_cid); @@ -128,6 +108,32 @@ pub async fn export_v2( }); } + export_to_forest_car::( + roots, + Some(prefix_data_frames), + db, + tipset, + lookup_depth, + writer, + seen, + skip_checksum, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn export_to_forest_car( + roots: NonEmpty, + prefix_data_frames: Option>>, + db: &Arc, + tipset: &Tipset, + lookup_depth: ChainEpochDelta, + writer: impl AsyncWrite + Unpin, + seen: CidHashSet, + skip_checksum: bool, +) -> anyhow::Result>, Error> { + let stateroot_lookup_limit = tipset.epoch() - lookup_depth; + // Wrap writer in optional checksum calculator let mut writer = AsyncWriterWithChecksum::::new(BufWriter::new(writer), !skip_checksum); @@ -148,7 +154,7 @@ pub async fn export_v2( // Encode Ipld key-value pairs in zstd frames let block_frames = forest::Encoder::compress_stream_default(blocks); - let frames = futures::stream::iter(prefix_data_frames).chain(block_frames); + let frames = futures::stream::iter(prefix_data_frames.unwrap_or_default()).chain(block_frames); // Write zstd frames and include a skippable index forest::Encoder::write(&mut writer, roots, frames).await?; @@ -161,12 +167,13 @@ pub async fn export_v2( Ok(digest) } -#[derive(Debug, Copy, FromPrimitive, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, clap::ValueEnum, FromPrimitive, Clone, PartialEq, Eq, JsonSchema)] #[repr(u64)] pub enum FilecoinSnapshotVersion { V1 = 1, V2 = 2, } +lotus_json_with_self!(FilecoinSnapshotVersion); impl Serialize for FilecoinSnapshotVersion { fn serialize(&self, serializer: S) -> Result @@ -205,13 +212,21 @@ pub struct FilecoinSnapshotMetadata { } impl FilecoinSnapshotMetadata { - pub fn new_v2(head_tipset_key: NonEmpty, f3_data: Option) -> Self { + pub fn new( + version: FilecoinSnapshotVersion, + head_tipset_key: NonEmpty, + f3_data: Option, + ) -> Self { Self { - version: FilecoinSnapshotVersion::V2, + version, head_tipset_key, f3_data, } } + + pub fn new_v2(head_tipset_key: NonEmpty, f3_data: Option) -> Self { + Self::new(FilecoinSnapshotVersion::V2, head_tipset_key, f3_data) + } } impl std::fmt::Display for FilecoinSnapshotMetadata { diff --git a/src/cli/subcommands/snapshot_cmd.rs b/src/cli/subcommands/snapshot_cmd.rs index ddf54fc5ec42..4fe9c5b59059 100644 --- a/src/cli/subcommands/snapshot_cmd.rs +++ b/src/cli/subcommands/snapshot_cmd.rs @@ -2,12 +2,12 @@ // SPDX-License-Identifier: Apache-2.0, MIT use super::*; +use crate::chain::FilecoinSnapshotVersion; use crate::chain_sync::SyncConfig; use crate::cli_shared::snapshot::{self, TrustedVendor}; use crate::db::car::forest::new_forest_car_temp_path_in; use crate::networks::calibnet; -use crate::rpc::types::ApiTipsetKey; -use crate::rpc::{self, chain::ChainExportParams, prelude::*}; +use crate::rpc::{self, chain::ForestChainExportParams, prelude::*, types::ApiTipsetKey}; use anyhow::Context as _; use chrono::DateTime; use clap::Subcommand; @@ -19,12 +19,6 @@ use std::{ }; use tokio::io::AsyncWriteExt; -#[derive(Debug, Clone, clap::ValueEnum)] -pub enum SnapshotFormat { - V1, - V2, -} - #[derive(Debug, Subcommand)] pub enum SnapshotCommands { /// Export a snapshot of the chain to `` @@ -45,8 +39,8 @@ pub enum SnapshotCommands { #[arg(short, long)] depth: Option, /// Export snapshot in the experimental v2 format(FRC-0108). - #[arg(long, value_enum, default_value_t = SnapshotFormat::V1)] - format: SnapshotFormat, + #[arg(long, value_enum, default_value_t = FilecoinSnapshotVersion::V1)] + format: FilecoinSnapshotVersion, }, } @@ -95,7 +89,8 @@ impl SnapshotCommands { let output_dir = output_path.parent().context("invalid output path")?; let temp_path = new_forest_car_temp_path_in(output_dir)?; - let params = ChainExportParams { + let params = ForestChainExportParams { + version: format, epoch, recent_roots: depth.unwrap_or(SyncConfig::default().recent_state_roots), output_path: temp_path.to_path_buf(), @@ -140,20 +135,9 @@ impl SnapshotCommands { }); // Manually construct RpcRequest because snapshot export could // take a few hours on mainnet - let hash_result = match format { - SnapshotFormat::V1 => { - client - .call(ChainExport::request((params,))?.with_timeout(Duration::MAX)) - .await? - } - SnapshotFormat::V2 => { - client - .call( - ForestChainExport::request((params,))?.with_timeout(Duration::MAX), - ) - .await? - } - }; + let hash_result = client + .call(ForestChainExport::request((params,))?.with_timeout(Duration::MAX)) + .await?; handle.abort(); let _ = handle.await; diff --git a/src/db/car/forest.rs b/src/db/car/forest.rs index 71fdfe350bb4..9dc466d281ba 100644 --- a/src/db/car/forest.rs +++ b/src/db/car/forest.rs @@ -88,6 +88,9 @@ pub const DEFAULT_FOREST_CAR_FRAME_SIZE: usize = 8000_usize.next_power_of_two(); pub const DEFAULT_FOREST_CAR_COMPRESSION_LEVEL: u16 = zstd::DEFAULT_COMPRESSION_LEVEL as _; const ZSTD_SKIP_FRAME_LEN: u64 = 8; +/// `zstd` frame of Forest CAR +pub type ForestCarFrame = (Vec, Bytes); + pub struct ForestCar { // Multiple `ForestCar` structures may share the same cache. The cache key is used to identify // the origin of a cached z-frame. @@ -122,8 +125,9 @@ impl ForestCar { pub fn metadata(&self) -> &Option { self.metadata.get_or_init(|| { - // - if self.header.roots.len() == 1 { + /// According to FRC-0108, v2 snapshots have exactly one root pointing to metadata + const V2_SNAPSHOT_ROOT_COUNT: usize = 1; + if self.header.roots.len() == V2_SNAPSHOT_ROOT_COUNT { let maybe_metadata_cid = self.header.roots.first(); if let Ok(Some(metadata)) = self.get_cbor::(maybe_metadata_cid) @@ -278,7 +282,7 @@ impl Encoder { pub async fn write( mut sink: impl AsyncWrite + Unpin, roots: NonEmpty, - mut stream: impl Stream, Bytes)>> + Unpin, + mut stream: impl Stream> + Unpin, ) -> anyhow::Result<()> { let mut offset = 0; @@ -323,7 +327,7 @@ impl Encoder { /// `compress_stream` with [`DEFAULT_FOREST_CAR_FRAME_SIZE`] as default frame size and [`DEFAULT_FOREST_CAR_COMPRESSION_LEVEL`] as default compression level. pub fn compress_stream_default( stream: impl Stream>, - ) -> impl Stream, Bytes)>> { + ) -> impl Stream> { Self::compress_stream( DEFAULT_FOREST_CAR_FRAME_SIZE, DEFAULT_FOREST_CAR_COMPRESSION_LEVEL, @@ -337,7 +341,7 @@ impl Encoder { zstd_frame_size_tripwire: usize, zstd_compression_level: u16, stream: impl Stream>, - ) -> impl Stream, Bytes)>> { + ) -> impl Stream> { let mut encoder_store = new_encoder(zstd_compression_level); let mut frame_cids = vec![]; diff --git a/src/rpc/methods/chain.rs b/src/rpc/methods/chain.rs index 647741b5d171..a2853b23cbbe 100644 --- a/src/rpc/methods/chain.rs +++ b/src/rpc/methods/chain.rs @@ -9,7 +9,7 @@ use types::*; use crate::blocks::RawBlockHeader; use crate::blocks::{Block, CachingBlockHeader, Tipset, TipsetKey}; use crate::chain::index::ResolveNullTipset; -use crate::chain::{ChainStore, HeadChange}; +use crate::chain::{ChainStore, FilecoinSnapshotVersion, HeadChange}; use crate::cid_collections::CidHashSet; use crate::ipld::DfsIter; use crate::lotus_json::{HasLotusJson, LotusJson, lotus_json_with_self}; @@ -222,21 +222,22 @@ impl RpcMethod<1> for ChainPruneSnapshot { } } -pub enum ChainExport {} -impl RpcMethod<1> for ChainExport { - const NAME: &'static str = "Filecoin.ChainExport"; +pub enum ForestChainExport {} +impl RpcMethod<1> for ForestChainExport { + const NAME: &'static str = "Forest.ChainExport"; const PARAM_NAMES: [&'static str; 1] = ["params"]; const API_PATHS: BitFlags = ApiPaths::all(); const PERMISSION: Permission = Permission::Read; - type Params = (ChainExportParams,); + type Params = (ForestChainExportParams,); type Ok = Option; async fn handle( ctx: Ctx, (params,): Self::Params, ) -> Result { - let ChainExportParams { + let ForestChainExportParams { + version, epoch, recent_roots, output_path, @@ -265,27 +266,35 @@ impl RpcMethod<1> for ChainExport { ctx.chain_index() .tipset_by_height(epoch, head, ResolveNullTipset::TakeOlder)?; - match if dry_run { - crate::chain::export::( - &ctx.store_owned(), - &start_ts, - recent_roots, - VoidAsyncWriter, - CidHashSet::default(), - skip_checksum, - ) - .await + let writer = if dry_run { + tokio_util::either::Either::Left(VoidAsyncWriter) } else { - let file = tokio::fs::File::create(&output_path).await?; - crate::chain::export::( - &ctx.store_owned(), - &start_ts, - recent_roots, - file, - CidHashSet::default(), - skip_checksum, - ) - .await + tokio_util::either::Either::Right(tokio::fs::File::create(&output_path).await?) + }; + match match version { + FilecoinSnapshotVersion::V1 => { + crate::chain::export::( + &ctx.store_owned(), + &start_ts, + recent_roots, + writer, + CidHashSet::default(), + skip_checksum, + ) + .await + } + FilecoinSnapshotVersion::V2 => { + crate::chain::export_v2::( + &ctx.store_owned(), + None, + &start_ts, + recent_roots, + writer, + CidHashSet::default(), + skip_checksum, + ) + .await + } } { Ok(checksum_opt) => Ok(checksum_opt.map(|hash| hash.encode_hex())), Err(e) => Err(anyhow::anyhow!(e).into()), @@ -293,9 +302,9 @@ impl RpcMethod<1> for ChainExport { } } -pub enum ForestChainExport {} -impl RpcMethod<1> for ForestChainExport { - const NAME: &'static str = "Forest.ChainExport"; +pub enum ChainExport {} +impl RpcMethod<1> for ChainExport { + const NAME: &'static str = "Filecoin.ChainExport"; const PARAM_NAMES: [&'static str; 1] = ["params"]; const API_PATHS: BitFlags = ApiPaths::all(); const PERMISSION: Permission = Permission::Read; @@ -305,64 +314,28 @@ impl RpcMethod<1> for ForestChainExport { async fn handle( ctx: Ctx, - (params,): Self::Params, - ) -> Result { - let ChainExportParams { + (ChainExportParams { epoch, recent_roots, output_path, - tipset_keys: ApiTipsetKey(tsk), + tipset_keys, skip_checksum, dry_run, - } = params; - - static LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); - - let _locked = LOCK.try_lock(); - if _locked.is_err() { - return Err(anyhow::anyhow!("Another chain export job is still in progress").into()); - } - - let chain_finality = ctx.chain_config().policy.chain_finality; - if recent_roots < chain_finality { - return Err(anyhow::anyhow!(format!( - "recent-stateroots must be greater than {chain_finality}" - )) - .into()); - } - - let head = ctx.chain_store().load_required_tipset_or_heaviest(&tsk)?; - let start_ts = - ctx.chain_index() - .tipset_by_height(epoch, head, ResolveNullTipset::TakeOlder)?; - - match if dry_run { - crate::chain::export_v2::( - &ctx.store_owned(), - None, - &start_ts, - recent_roots, - VoidAsyncWriter, - CidHashSet::default(), - skip_checksum, - ) - .await - } else { - let file = tokio::fs::File::create(&output_path).await?; - crate::chain::export_v2::( - &ctx.store_owned(), - None, - &start_ts, + },): Self::Params, + ) -> Result { + ForestChainExport::handle( + ctx, + (ForestChainExportParams { + version: FilecoinSnapshotVersion::V1, + epoch, recent_roots, - file, - CidHashSet::default(), + output_path, + tipset_keys, skip_checksum, - ) - .await - } { - Ok(checksum_opt) => Ok(checksum_opt.map(|hash| hash.encode_hex())), - Err(e) => Err(anyhow::anyhow!(e).into()), - } + dry_run, + },), + ) + .await } } @@ -913,6 +886,20 @@ pub struct ApiMessage { lotus_json_with_self!(ApiMessage); +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +pub struct ForestChainExportParams { + pub version: FilecoinSnapshotVersion, + pub epoch: ChainEpoch, + pub recent_roots: i64, + pub output_path: PathBuf, + #[schemars(with = "LotusJson")] + #[serde(with = "crate::lotus_json")] + pub tipset_keys: ApiTipsetKey, + pub skip_checksum: bool, + pub dry_run: bool, +} +lotus_json_with_self!(ForestChainExportParams); + #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct ChainExportParams { pub epoch: ChainEpoch, diff --git a/src/tool/subcommands/archive_cmd.rs b/src/tool/subcommands/archive_cmd.rs index b8d7f6714511..98e27bd5f9fd 100644 --- a/src/tool/subcommands/archive_cmd.rs +++ b/src/tool/subcommands/archive_cmd.rs @@ -199,7 +199,9 @@ impl ArchiveCommands { if let Some(metadata) = store.metadata() { println!("{metadata}"); } else { - println!("No FRC-0108 metadata found in the snapshot"); + println!( + "No metadata found (required by v2 snapshot) - this appears to be a v1 snapshot" + ); } Ok(()) } @@ -324,7 +326,7 @@ impl ArchiveInfo { let tipsets = head.clone().chain(store); - let windowed = (std::iter::once(head.clone()).chain(tipsets)).tuple_windows(); + let windowed = std::iter::once(head.clone()).chain(tipsets).tuple_windows(); let mut network: String = "unknown".into(); let mut lowest_stateroot_epoch = root_epoch; From c60e6b1475bc22b723cef71b76f9b20f66ea9694 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 30 Jul 2025 21:16:23 +0800 Subject: [PATCH 09/35] resolve comments --- src/chain/mod.rs | 2 +- src/db/car/forest.rs | 7 +++---- src/db/car/mod.rs | 3 +++ src/db/car/plain.rs | 6 +++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 3209e2a865fb..c9dd082002f8 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -199,7 +199,7 @@ impl<'de> Deserialize<'de> for FilecoinSnapshotVersion { } } -/// +/// Defined in #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "PascalCase")] pub struct FilecoinSnapshotMetadata { diff --git a/src/db/car/forest.rs b/src/db/car/forest.rs index 9dc466d281ba..868f6dc1596b 100644 --- a/src/db/car/forest.rs +++ b/src/db/car/forest.rs @@ -125,9 +125,7 @@ impl ForestCar { pub fn metadata(&self) -> &Option { self.metadata.get_or_init(|| { - /// According to FRC-0108, v2 snapshots have exactly one root pointing to metadata - const V2_SNAPSHOT_ROOT_COUNT: usize = 1; - if self.header.roots.len() == V2_SNAPSHOT_ROOT_COUNT { + if self.header.roots.len() == super::V2_SNAPSHOT_ROOT_COUNT { let maybe_metadata_cid = self.header.roots.first(); if let Ok(Some(metadata)) = self.get_cbor::(maybe_metadata_cid) @@ -168,7 +166,8 @@ impl ForestCar { } pub fn head_tipset_key(&self) -> &NonEmpty { - // + // head tipset key is stored in v2 snapshot metadata + // See if let Some(metadata) = self.metadata() { &metadata.head_tipset_key } else { diff --git a/src/db/car/mod.rs b/src/db/car/mod.rs index f68528d46823..c7fd4e69df64 100644 --- a/src/db/car/mod.rs +++ b/src/db/car/mod.rs @@ -33,6 +33,9 @@ pub type CacheKey = u64; type FrameOffset = u64; +/// According to FRC-0108, v2 snapshots have exactly one root pointing to metadata +const V2_SNAPSHOT_ROOT_COUNT: usize = 1; + pub static ZSTD_FRAME_CACHE_DEFAULT_MAX_SIZE: LazyLock = LazyLock::new(|| { const ENV_KEY: &str = "FOREST_ZSTD_FRAME_CACHE_DEFAULT_MAX_SIZE"; if let Ok(value) = std::env::var(ENV_KEY) { diff --git a/src/db/car/plain.rs b/src/db/car/plain.rs index 798a4f0a4b82..d67d2daefc79 100644 --- a/src/db/car/plain.rs +++ b/src/db/car/plain.rs @@ -174,8 +174,7 @@ impl PlainCar { pub fn metadata(&self) -> &Option { self.metadata.get_or_init(|| { - // - if self.header_v1.roots.len() == 1 { + if self.header_v1.roots.len() == super::V2_SNAPSHOT_ROOT_COUNT { let maybe_metadata_cid = self.header_v1.roots.first(); if let Ok(Some(metadata)) = self.get_cbor::(maybe_metadata_cid) @@ -188,7 +187,8 @@ impl PlainCar { } pub fn head_tipset_key(&self) -> &NonEmpty { - // + // head tipset key is stored in v2 snapshot metadata + // See if let Some(metadata) = self.metadata() { &metadata.head_tipset_key } else { From b6720fedcc1e8d8928be9509b52a2e6db75b79bb Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 30 Jul 2025 22:37:01 +0800 Subject: [PATCH 10/35] tests --- src/blocks/chain4u.rs | 10 ++++- src/blocks/header.rs | 30 ++++++++++++--- src/blocks/tipset.rs | 9 +++++ src/chain/mod.rs | 89 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 8 deletions(-) diff --git a/src/blocks/chain4u.rs b/src/blocks/chain4u.rs index a955c477f46a..5e7c79b24978 100644 --- a/src/blocks/chain4u.rs +++ b/src/blocks/chain4u.rs @@ -16,7 +16,7 @@ use crate::{ sector::PoStProof, }, }; -use chain4u::header::GENESIS_BLOCK_PARENTS; +use chain4u::header::{FILECOIN_GENESIS_BLOCK, FILECOIN_GENESIS_CID, GENESIS_BLOCK_PARENTS}; use cid::Cid; use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::CborStore; @@ -161,7 +161,13 @@ impl Chain4U { } impl Chain4U { - pub fn with_blockstore(blockstore: T) -> Self { + pub fn with_blockstore(blockstore: T) -> Self + where + T: Blockstore, + { + blockstore + .put_keyed(&FILECOIN_GENESIS_CID, &FILECOIN_GENESIS_BLOCK) + .unwrap(); Self { blockstore, inner: Default::default(), diff --git a/src/blocks/header.rs b/src/blocks/header.rs index dbbb19f3af6f..91fdd10c6d59 100644 --- a/src/blocks/header.rs +++ b/src/blocks/header.rs @@ -25,12 +25,18 @@ use serde_tuple::{Deserialize_tuple, Serialize_tuple}; // See // and #[cfg(test)] -static FILECOIN_GENESIS_CID: std::sync::LazyLock = std::sync::LazyLock::new(|| { +pub static FILECOIN_GENESIS_CID: std::sync::LazyLock = std::sync::LazyLock::new(|| { "bafyreiaqpwbbyjo4a42saasj36kkrpv4tsherf2e7bvezkert2a7dhonoi" .parse() .expect("Infallible") }); +#[cfg(test)] +pub static FILECOIN_GENESIS_BLOCK: std::sync::LazyLock> = std::sync::LazyLock::new(|| { + hex::decode("a5684461746574696d6573323031372d30352d30352030313a32373a3531674e6574776f726b6846696c65636f696e65546f6b656e6846696c65636f696e6c546f6b656e416d6f756e7473a36b546f74616c537570706c796d322c3030302c3030302c303030664d696e6572736d312c3430302c3030302c3030306c50726f746f636f6c4c616273a36b446576656c6f706d656e746b3330302c3030302c3030306b46756e6472616973696e676b3230302c3030302c3030306a466f756e646174696f6e6b3130302c3030302c303030674d657373616765784854686973206973207468652047656e6573697320426c6f636b206f66207468652046696c65636f696e20446563656e7472616c697a65642053746f72616765204e6574776f726b2e") + .expect("Infallible") +}); + #[cfg(test)] pub static GENESIS_BLOCK_PARENTS: std::sync::LazyLock = std::sync::LazyLock::new(|| nunny::vec![*FILECOIN_GENESIS_CID].into()); @@ -335,15 +341,16 @@ impl<'de> Deserialize<'de> for CachingBlockHeader { #[cfg(test)] mod tests { + use super::*; use crate::beacon::{BeaconEntry, BeaconPoint, BeaconSchedule, mock_beacon::MockBeacon}; + use crate::blocks::{CachingBlockHeader, Error}; use crate::shim::clock::ChainEpoch; use crate::shim::{address::Address, version::NetworkVersion}; use crate::utils::encoding::from_slice_with_fallback; - use fvm_ipld_encoding::to_vec; - - use crate::blocks::{CachingBlockHeader, Error}; - - use super::RawBlockHeader; + use crate::utils::multihash::MultihashCode; + use cid::Cid; + use fvm_ipld_encoding::{DAG_CBOR, to_vec}; + use multihash_derive::MultihashDigest as _; impl quickcheck::Arbitrary for CachingBlockHeader { fn arbitrary(g: &mut quickcheck::Gen) -> Self { @@ -405,4 +412,15 @@ mod tests { } } } + + #[test] + fn test_genesis_parent() { + assert_eq!( + Cid::new_v1( + DAG_CBOR, + MultihashCode::Sha2_256.digest(&FILECOIN_GENESIS_BLOCK) + ), + *FILECOIN_GENESIS_CID + ); + } } diff --git a/src/blocks/tipset.rs b/src/blocks/tipset.rs index 1d449079f0b7..a361f5996ef8 100644 --- a/src/blocks/tipset.rs +++ b/src/blocks/tipset.rs @@ -170,6 +170,15 @@ impl From for Tipset { } } +impl From> for Tipset { + fn from(headers: NonEmpty) -> Self { + Self { + headers, + key: OnceLock::new(), + } + } +} + impl PartialEq for Tipset { fn eq(&self, other: &Self) -> bool { self.headers.eq(&other.headers) diff --git a/src/chain/mod.rs b/src/chain/mod.rs index c9dd082002f8..6cc16b287b46 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -251,7 +251,14 @@ impl std::fmt::Display for FilecoinSnapshotMetadata { #[cfg(test)] mod tests { + use sha2::{Digest as _, Sha256}; + use super::*; + use crate::{ + blocks::{CachingBlockHeader, Chain4U, Tipset, TipsetKey, chain4u}, + db::{MemoryDB, car::ForestCar}, + utils::db::CborStoreExt, + }; #[test] fn test_snapshot_version_cbor_serde() { @@ -267,4 +274,86 @@ mod tests { FilecoinSnapshotVersion::V2 ); } + + #[tokio::test] + async fn test_export_v1() { + test_export_inner(FilecoinSnapshotVersion::V1) + .await + .unwrap() + } + + #[tokio::test] + async fn test_export_v2() { + test_export_inner(FilecoinSnapshotVersion::V2) + .await + .unwrap() + } + + async fn test_export_inner(version: FilecoinSnapshotVersion) -> anyhow::Result<()> { + let db = Arc::new(MemoryDB::default()); + let c4u = Chain4U::with_blockstore(db.clone()); + chain4u! { + in c4u; // select the context + [genesis] + -> [b_1] + -> [b_2_0, b_2_1] + -> [b_3] + -> [b_4] + -> [b_5_0, b_5_1] + }; + + let head_key_cids = nunny::vec![b_5_0.cid(), b_5_1.cid()]; + let head_key = TipsetKey::from(head_key_cids.clone()); + let head = Tipset::load_required(&db, &head_key)?; + + let mut car_bytes = vec![]; + + let checksum = match version { + FilecoinSnapshotVersion::V1 => { + export::(&db, &head, 0, &mut car_bytes, Default::default(), false).await? + } + FilecoinSnapshotVersion::V2 => { + export_v2::( + &db, + None, + &head, + 0, + &mut car_bytes, + Default::default(), + false, + ) + .await? + } + }; + + assert_eq!(Sha256::digest(&car_bytes), checksum.unwrap()); + + let car = ForestCar::new(car_bytes)?; + + assert_eq!(car.heaviest_tipset()?, head); + + match version { + FilecoinSnapshotVersion::V1 => { + assert_eq!(car.metadata(), &None); + } + FilecoinSnapshotVersion::V2 => { + assert_eq!( + car.metadata(), + &Some(FilecoinSnapshotMetadata { + version, + head_tipset_key: head_key_cids, + f3_data: None, + }) + ); + } + } + + for b in [&genesis, &b_1, &b_2_0, &b_2_1, &b_3, &b_4, &b_5_0, &b_5_1] { + let b_from_car: CachingBlockHeader = car.get_cbor_required(&b.cid())?; + let b_from_db: CachingBlockHeader = db.get_cbor_required(&b.cid())?; + assert_eq!(b_from_car, b_from_db); + } + + Ok(()) + } } From dc127ca22add4966f26288b4a3201349169f265b Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 30 Jul 2025 22:47:34 +0800 Subject: [PATCH 11/35] speed up snapshot export check --- .github/workflows/forest.yml | 26 ++++++++++++++++++++++++-- scripts/tests/calibnet_export_check.sh | 20 ++++++++------------ scripts/tests/harness.sh | 2 +- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/.github/workflows/forest.yml b/.github/workflows/forest.yml index f5b114c7a1d7..239d695403b0 100644 --- a/.github/workflows/forest.yml +++ b/.github/workflows/forest.yml @@ -281,8 +281,30 @@ jobs: - name: Set permissions run: | chmod +x ~/.cargo/bin/forest* - - name: Snapshot export check - run: ./scripts/tests/calibnet_export_check.sh + - name: Snapshot export check v1 + run: ./scripts/tests/calibnet_export_check.sh v1 + timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }} + calibnet-export-check-v2: + needs: + - build-ubuntu + name: Snapshot export checks v2 + runs-on: ubuntu-24.04 + steps: + - run: lscpu + - uses: actions/cache@v4 + with: + path: "${{ env.FIL_PROOFS_PARAMETER_CACHE }}" + key: proof-params-keys + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: "forest-${{ runner.os }}" + path: ~/.cargo/bin + - name: Set permissions + run: | + chmod +x ~/.cargo/bin/forest* + - name: Snapshot export check v2 + run: ./scripts/tests/calibnet_export_check.sh v2 timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }} calibnet-no-discovery-checks: needs: diff --git a/scripts/tests/calibnet_export_check.sh b/scripts/tests/calibnet_export_check.sh index e9102daf1913..44163f757bcb 100755 --- a/scripts/tests/calibnet_export_check.sh +++ b/scripts/tests/calibnet_export_check.sh @@ -5,6 +5,8 @@ set -eu +format="${1:-v1}" + source "$(dirname "$0")/harness.sh" forest_init "$@" @@ -12,13 +14,14 @@ forest_init "$@" echo "Cleaning up the initial snapshot" rm --force --verbose ./*.{car,car.zst,sha256sum} -echo "Exporting zstd compressed snapshot" -$FOREST_CLI_PATH snapshot export -o v1.forest.car.zst +echo "Exporting zstd compressed snapshot at genesis" +$FOREST_CLI_PATH snapshot export --tipset 0 --format "$format" + +echo "Exporting zstd compressed snapshot in $format format" +$FOREST_CLI_PATH snapshot export --format "$format" -echo "Exporting zstd compressed snapshot in the experimental v2 format" -$FOREST_CLI_PATH snapshot export --format v2 -o v2.forest.car.zst +$FOREST_CLI_PATH shutdown --force -echo "Inspecting archive info and metadata" for f in *.car.zst; do echo "Inspecting archive info $f" $FOREST_TOOL_PATH archive info "$f" @@ -32,14 +35,7 @@ zstd --test ./*.car.zst echo "Verifying snapshot checksum" sha256sum --check ./*.sha256sum -echo "Validating CAR files" for f in *.car.zst; do echo "Validating CAR file $f" $FOREST_TOOL_PATH snapshot validate "$f" done - -echo "Exporting zstd compressed snapshot at genesis" -$FOREST_CLI_PATH snapshot export --tipset 0 - -echo "Testing genesis snapshot validity" -zstd --test forest_snapshot_calibnet_2022-11-01_height_0.forest.car.zst diff --git a/scripts/tests/harness.sh b/scripts/tests/harness.sh index 2625ee7124fd..dc153f27a54f 100644 --- a/scripts/tests/harness.sh +++ b/scripts/tests/harness.sh @@ -141,7 +141,7 @@ function forest_print_logs_and_metrics { function forest_cleanup { if pkill -0 forest 2>/dev/null; then forest_print_logs_and_metrics - $FOREST_CLI_PATH shutdown --force + $FOREST_CLI_PATH shutdown --force || true timeout 10s sh -c "while pkill -0 forest 2>/dev/null; do sleep 1; done" fi } From 0203c9626046c5af1718b57baa7ca4649a2ca662 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 31 Jul 2025 19:36:59 +0800 Subject: [PATCH 12/35] chore(deps): bump go-f3 and Go deps --- f3-sidecar/go.mod | 57 ++++++------ f3-sidecar/go.sum | 119 ++++++++++++-------------- interop-tests/src/tests/go_app/go.mod | 58 ++++++------- interop-tests/src/tests/go_app/go.sum | 116 ++++++++++++------------- 4 files changed, 170 insertions(+), 180 deletions(-) diff --git a/f3-sidecar/go.mod b/f3-sidecar/go.mod index 6299e22620a9..1150e3cec7ea 100644 --- a/f3-sidecar/go.mod +++ b/f3-sidecar/go.mod @@ -3,14 +3,14 @@ module f3-sidecar/v2 go 1.24.5 require ( - github.com/filecoin-project/go-f3 v0.8.8 + github.com/filecoin-project/go-f3 v0.8.9 github.com/filecoin-project/go-jsonrpc v0.8.0 github.com/filecoin-project/go-state-types v0.16.0 - github.com/ihciah/rust2go v0.0.0-20250717023419-6e5b74b1f846 + github.com/ihciah/rust2go v0.0.0-20250726175549-557d7a3a4e27 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-ds-leveldb v0.5.2 - github.com/ipfs/go-log/v2 v2.6.0 - github.com/libp2p/go-libp2p v0.42.0 + github.com/ipfs/go-log/v2 v2.8.0 + github.com/libp2p/go-libp2p v0.42.1 github.com/libp2p/go-libp2p-kad-dht v0.33.1 github.com/libp2p/go-libp2p-pubsub v0.14.2 github.com/stretchr/testify v1.10.0 @@ -20,10 +20,9 @@ require ( require ( github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.13.0 // indirect + github.com/bits-and-blooms/bitset v1.22.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/consensys/bavard v0.1.13 // indirect - github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/consensys/gnark-crypto v0.18.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect @@ -31,12 +30,12 @@ require ( github.com/filecoin-project/go-clock v0.1.0 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/golang/snappy v1.0.0 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a // indirect github.com/google/uuid v1.6.0 // indirect @@ -50,11 +49,11 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/klauspost/compress v1.18.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/koron/go-ssdp v0.0.6 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect - github.com/libp2p/go-flow-metrics v0.2.0 // indirect + github.com/libp2p/go-flow-metrics v0.3.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect github.com/libp2p/go-libp2p-kbucket v0.7.0 // indirect github.com/libp2p/go-libp2p-record v0.3.1 // indirect @@ -62,14 +61,13 @@ require ( github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-netroute v0.2.2 // indirect github.com/libp2p/go-reuseport v0.4.0 // indirect - github.com/libp2p/go-yamux/v5 v5.0.1 // indirect + github.com/libp2p/go-yamux/v5 v5.1.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/miekg/dns v1.1.66 // indirect + github.com/miekg/dns v1.1.67 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.1 // indirect - github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect @@ -90,26 +88,26 @@ require ( github.com/pion/dtls/v3 v3.0.6 // indirect github.com/pion/ice/v4 v4.0.10 // indirect github.com/pion/interceptor v0.1.40 // indirect - github.com/pion/logging v0.2.3 // indirect + github.com/pion/logging v0.2.4 // indirect github.com/pion/mdns/v2 v2.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/rtcp v1.2.15 // indirect - github.com/pion/rtp v1.8.19 // indirect + github.com/pion/rtp v1.8.21 // indirect github.com/pion/sctp v1.8.39 // indirect - github.com/pion/sdp/v3 v3.0.13 // indirect + github.com/pion/sdp/v3 v3.0.15 // indirect github.com/pion/srtp/v3 v3.0.6 // indirect github.com/pion/stun v0.6.1 // indirect github.com/pion/stun/v3 v3.0.0 // indirect github.com/pion/transport/v2 v2.2.10 // indirect github.com/pion/transport/v3 v3.0.7 // indirect github.com/pion/turn/v4 v4.0.2 // indirect - github.com/pion/webrtc/v4 v4.1.2 // indirect + github.com/pion/webrtc/v4 v4.1.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect - github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_golang v1.23.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.64.0 // indirect - github.com/prometheus/procfs v0.16.1 // indirect + github.com/prometheus/common v0.65.0 // indirect + github.com/prometheus/procfs v0.17.0 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.52.0 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect @@ -121,27 +119,26 @@ require ( go.dedis.ch/kyber/v4 v4.0.0-pre2.0.20240924132404-4de33740016e // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel v1.35.0 // indirect - go.opentelemetry.io/otel/metric v1.35.0 // indirect - go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/dig v1.19.0 // indirect go.uber.org/fx v1.24.0 // indirect go.uber.org/mock v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect - golang.org/x/mod v0.25.0 // indirect - golang.org/x/net v0.41.0 // indirect + golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect + golang.org/x/mod v0.26.0 // indirect + golang.org/x/net v0.42.0 // indirect golang.org/x/sync v0.16.0 // indirect golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.12.0 // indirect - golang.org/x/tools v0.34.0 // indirect + golang.org/x/tools v0.35.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gonum.org/v1/gonum v0.16.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.4.1 // indirect - rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/f3-sidecar/go.sum b/f3-sidecar/go.sum index 2467ef08a1dd..e55c92e08444 100644 --- a/f3-sidecar/go.sum +++ b/f3-sidecar/go.sum @@ -14,8 +14,8 @@ github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= -github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= +github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -23,10 +23,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= -github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= -github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= -github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/consensys/gnark-crypto v0.18.0 h1:vIye/FqI50VeAr0B3dx+YjeIvmc3LWz4yEfbWBpTUf0= +github.com/consensys/gnark-crypto v0.18.0/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -47,8 +45,8 @@ github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= -github.com/filecoin-project/go-f3 v0.8.8 h1:1MFkgJIQVvE9U5Q2EDJ5djjAt3jq9wPUa+d1QpQOr3c= -github.com/filecoin-project/go-f3 v0.8.8/go.mod h1:TvVqGx98n6vdDZGZ1K2Vw13TtlsyJbiTiLGX2XSQeVk= +github.com/filecoin-project/go-f3 v0.8.9 h1:0SHqwWmcVAL02Or7uE4P7qG1feopyVBSlgrUxkHkQBM= +github.com/filecoin-project/go-f3 v0.8.9/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= github.com/filecoin-project/go-jsonrpc v0.8.0 h1:2yqlN3Vd8Gx5UtA3fib7tQu2aW1cSOJt253LEBWExo4= github.com/filecoin-project/go-jsonrpc v0.8.0/go.mod h1:p8WGOwQGYbFugSdK7qKIGhhb1VVcQ2rtBLdEiik1QWI= github.com/filecoin-project/go-state-types v0.16.0 h1:ajIREDzTGfq71ofIQ29iZR1WXxmkvd2nQNc6ApcP1wI= @@ -67,8 +65,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= @@ -78,8 +76,9 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -94,8 +93,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -113,7 +112,6 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a h1://KbezygeMJZCSHH+HgUZiTeSoiuFspbMg1ge+eFj18= github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA= -github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -135,8 +133,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyf github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= -github.com/ihciah/rust2go v0.0.0-20250717023419-6e5b74b1f846 h1:fwZvyVNjoVf4enaKAbWf5rlFFvAMPT72ta0YDlZfvgc= -github.com/ihciah/rust2go v0.0.0-20250717023419-6e5b74b1f846/go.mod h1:SpcZZoVYDXYhx373JZ1onR8dnEcalKA0H61x/FJ9O0s= +github.com/ihciah/rust2go v0.0.0-20250726175549-557d7a3a4e27 h1:UumvcTbl6Vg04nu3Q4fI27aO4VhXQX0BXY+nvLRRxCc= +github.com/ihciah/rust2go v0.0.0-20250726175549-557d7a3a4e27/go.mod h1:SpcZZoVYDXYhx373JZ1onR8dnEcalKA0H61x/FJ9O0s= github.com/ipfs/boxo v0.33.0 h1:9ow3chwkDzMj0Deq4AWRUEI7WnIIV7SZhPTzzG2mmfw= github.com/ipfs/boxo v0.33.0/go.mod h1:3IPh7YFcCIcKp6o02mCHovrPntoT5Pctj/7j4syh/RM= github.com/ipfs/go-block-format v0.2.2 h1:uecCTgRwDIXyZPgYspaLXoMiMmxQpSx2aq34eNc4YvQ= @@ -151,8 +149,8 @@ github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp0x0= github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= -github.com/ipfs/go-log/v2 v2.6.0 h1:2Nu1KKQQ2ayonKp4MPo6pXCjqw1ULc9iohRqWV5EYqg= -github.com/ipfs/go-log/v2 v2.6.0/go.mod h1:p+Efr3qaY5YXpx9TX7MoLCSEZX5boSWj9wh86P5HJa8= +github.com/ipfs/go-log/v2 v2.8.0 h1:SptNTPJQV3s5EF4FdrTu/yVdOKfGbDgn1EBZx4til2o= +github.com/ipfs/go-log/v2 v2.8.0/go.mod h1:2LEEhdv8BGubPeSFTyzbqhCqrwqxCbuTNTLWqgNAipo= github.com/ipfs/go-test v0.2.2 h1:1yjYyfbdt1w93lVzde6JZ2einh3DIV40at4rVoyEcE8= github.com/ipfs/go-test v0.2.2/go.mod h1:cmLisgVwkdRCnKu/CFZOk2DdhOcwghr5GsHeqwexoRA= github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= @@ -172,8 +170,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -185,16 +183,16 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4= +github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= -github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw= -github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc= -github.com/libp2p/go-libp2p v0.42.0 h1:A8foZk+ZEhZTv0Jb++7xUFlrFhBDv4j2Vh/uq4YX+KE= -github.com/libp2p/go-libp2p v0.42.0/go.mod h1:4NGcjbD9OIvFiSRb0XueCO19zJ4kSPK5vkyyOUYmMro= +github.com/libp2p/go-flow-metrics v0.3.0 h1:q31zcHUvHnwDO0SHaukewPYgwOBSxtt830uJtUx6784= +github.com/libp2p/go-flow-metrics v0.3.0/go.mod h1:nuhlreIwEguM1IvHAew3ij7A8BMlyHQJ279ao24eZZo= +github.com/libp2p/go-libp2p v0.42.1 h1:Rt8+5thie729NQk1gx1h/2t/+VIafWcqR1I+Kvw+UTg= +github.com/libp2p/go-libp2p v0.42.1/go.mod h1:4NGcjbD9OIvFiSRb0XueCO19zJ4kSPK5vkyyOUYmMro= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-kad-dht v0.33.1 h1:hKFhHMf7WH69LDjaxsJUWOU6qZm71uO47M/a5ijkiP0= @@ -215,8 +213,8 @@ github.com/libp2p/go-netroute v0.2.2 h1:Dejd8cQ47Qx2kRABg6lPwknU7+nBnFRpko45/fFP github.com/libp2p/go-netroute v0.2.2/go.mod h1:Rntq6jUAH0l9Gg17w5bFGhcC9a+vk4KNXs6s7IljKYE= github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= -github.com/libp2p/go-yamux/v5 v5.0.1 h1:f0WoX/bEF2E8SbE4c/k1Mo+/9z0O4oC/hWEA+nfYRSg= -github.com/libp2p/go-yamux/v5 v5.0.1/go.mod h1:en+3cdX51U0ZslwRdRLrvQsdayFt3TSUKvBGErzpWbU= +github.com/libp2p/go-yamux/v5 v5.1.0 h1:8Qlxj4E9JGJAQVW6+uj2o7mqkqsIVlSUGmTWhlXzoHE= +github.com/libp2p/go-yamux/v5 v5.1.0/go.mod h1:tgIQ07ObtRR/I0IWsFOyQIL9/dR5UXgc2s8xKmNZv1o= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= @@ -225,8 +223,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= -github.com/miekg/dns v1.1.66 h1:FeZXOS3VCVsKnEAd+wBkjMC3D2K+ww66Cq3VnCINuJE= -github.com/miekg/dns v1.1.66/go.mod h1:jGFzBsSNbJw6z1HYut1RKBKHA9PBdxeHrZG8J+gC2WE= +github.com/miekg/dns v1.1.67 h1:kg0EHj0G4bfT5/oOys6HhZw4vmMlnoZ+gDu8tJ/AlI0= +github.com/miekg/dns v1.1.67/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -238,9 +236,6 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= -github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= -github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= @@ -305,20 +300,20 @@ github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu1 github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4= github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= -github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI= -github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90= +github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= +github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo= github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0= -github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c= -github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= +github.com/pion/rtp v1.8.21 h1:3yrOwmZFyUpcIosNcWRpQaU+UXIJ6yxLuJ8Bx0mw37Y= +github.com/pion/rtp v1.8.21/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE= github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE= -github.com/pion/sdp/v3 v3.0.13 h1:uN3SS2b+QDZnWXgdr69SM8KB4EbcnPnPf2Laxhty/l4= -github.com/pion/sdp/v3 v3.0.13/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E= +github.com/pion/sdp/v3 v3.0.15 h1:F0I1zds+K/+37ZrzdADmx2Q44OFDOPRLhPnNTaUX9hk= +github.com/pion/sdp/v3 v3.0.15/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E= github.com/pion/srtp/v3 v3.0.6 h1:E2gyj1f5X10sB/qILUGIkL4C2CqK269Xq167PbGCc/4= github.com/pion/srtp/v3 v3.0.6/go.mod h1:BxvziG3v/armJHAaJ87euvkhHqWe9I7iiOy50K2QkhY= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= @@ -333,8 +328,8 @@ github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1 github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps= github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs= -github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54= -github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U= +github.com/pion/webrtc/v4 v4.1.3 h1:YZ67Boj9X/hk190jJZ8+HFGQ6DqSZ/fYP3sLAZv7c3c= +github.com/pion/webrtc/v4 v4.1.3/go.mod h1:rsq+zQ82ryfR9vbb0L1umPJ6Ogq7zm8mcn9fcGnxomM= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -343,18 +338,18 @@ github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= +github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= -github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= +github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= +github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.52.0 h1:/SlHrCRElyaU6MaEPKqKr9z83sBg2v4FLLvWM+Z47pA= @@ -439,12 +434,12 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= @@ -477,8 +472,8 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= -golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -489,8 +484,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= -golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -513,8 +508,8 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -589,8 +584,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= -golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -653,7 +648,5 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= -rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= -rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/interop-tests/src/tests/go_app/go.mod b/interop-tests/src/tests/go_app/go.mod index 25a1a22461c4..24c61bfe74b5 100644 --- a/interop-tests/src/tests/go_app/go.mod +++ b/interop-tests/src/tests/go_app/go.mod @@ -3,12 +3,12 @@ module test/v2 go 1.24.5 require ( - github.com/ihciah/rust2go v0.0.0-20250717023419-6e5b74b1f846 + github.com/ihciah/rust2go v0.0.0-20250726175549-557d7a3a4e27 github.com/ipfs/boxo v0.33.0 github.com/ipfs/go-cid v0.5.0 github.com/ipfs/go-datastore v0.8.2 - github.com/ipfs/go-log/v2 v2.6.0 - github.com/libp2p/go-libp2p v0.42.0 + github.com/ipfs/go-log/v2 v2.8.0 + github.com/libp2p/go-libp2p v0.42.1 github.com/libp2p/go-libp2p-kad-dht v0.33.1 github.com/multiformats/go-multiaddr v0.16.0 ) @@ -23,13 +23,13 @@ require ( github.com/filecoin-project/go-clock v0.1.0 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect - github.com/gammazero/chanqueue v1.1.0 // indirect - github.com/gammazero/deque v1.0.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/gammazero/chanqueue v1.1.1 // indirect + github.com/gammazero/deque v1.1.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a // indirect + github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect @@ -46,11 +46,11 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/klauspost/compress v1.18.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/koron/go-ssdp v0.0.6 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect - github.com/libp2p/go-flow-metrics v0.2.0 // indirect + github.com/libp2p/go-flow-metrics v0.3.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect github.com/libp2p/go-libp2p-kbucket v0.7.0 // indirect github.com/libp2p/go-libp2p-record v0.3.1 // indirect @@ -58,10 +58,10 @@ require ( github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-netroute v0.2.2 // indirect github.com/libp2p/go-reuseport v0.4.0 // indirect - github.com/libp2p/go-yamux/v5 v5.0.1 // indirect + github.com/libp2p/go-yamux/v5 v5.1.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/miekg/dns v1.1.66 // indirect + github.com/miekg/dns v1.1.67 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.1 // indirect @@ -83,25 +83,25 @@ require ( github.com/pion/dtls/v3 v3.0.6 // indirect github.com/pion/ice/v4 v4.0.10 // indirect github.com/pion/interceptor v0.1.40 // indirect - github.com/pion/logging v0.2.3 // indirect + github.com/pion/logging v0.2.4 // indirect github.com/pion/mdns/v2 v2.0.7 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/rtcp v1.2.15 // indirect - github.com/pion/rtp v1.8.19 // indirect + github.com/pion/rtp v1.8.21 // indirect github.com/pion/sctp v1.8.39 // indirect - github.com/pion/sdp/v3 v3.0.13 // indirect + github.com/pion/sdp/v3 v3.0.15 // indirect github.com/pion/srtp/v3 v3.0.6 // indirect github.com/pion/stun v0.6.1 // indirect github.com/pion/stun/v3 v3.0.0 // indirect github.com/pion/transport/v2 v2.2.10 // indirect github.com/pion/transport/v3 v3.0.7 // indirect github.com/pion/turn/v4 v4.0.2 // indirect - github.com/pion/webrtc/v4 v4.1.2 // indirect + github.com/pion/webrtc/v4 v4.1.3 // indirect github.com/polydawn/refmt v0.89.0 // indirect - github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_golang v1.23.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.64.0 // indirect - github.com/prometheus/procfs v0.16.1 // indirect + github.com/prometheus/common v0.65.0 // indirect + github.com/prometheus/procfs v0.17.0 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.52.0 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect @@ -109,24 +109,24 @@ require ( github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/wlynxg/anet v0.0.5 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel v1.35.0 // indirect - go.opentelemetry.io/otel/metric v1.35.0 // indirect - go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/dig v1.19.0 // indirect go.uber.org/fx v1.24.0 // indirect go.uber.org/mock v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect - golang.org/x/mod v0.25.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sync v0.15.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect + golang.org/x/mod v0.26.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.12.0 // indirect - golang.org/x/tools v0.34.0 // indirect + golang.org/x/tools v0.35.0 // indirect gonum.org/v1/gonum v0.16.0 // indirect google.golang.org/protobuf v1.36.6 // indirect lukechampine.com/blake3 v1.4.1 // indirect diff --git a/interop-tests/src/tests/go_app/go.sum b/interop-tests/src/tests/go_app/go.sum index 1ca9154bf512..d5199cf1e0b6 100644 --- a/interop-tests/src/tests/go_app/go.sum +++ b/interop-tests/src/tests/go_app/go.sum @@ -43,16 +43,16 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/gammazero/chanqueue v1.1.0 h1:yiwtloc1azhgGLFo2gMloJtQvkYD936Ai7tBfa+rYJw= -github.com/gammazero/chanqueue v1.1.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc= -github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= -github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= +github.com/gammazero/chanqueue v1.1.1 h1:n9Y+zbBxw2f7uUE9wpgs0rOSkP/I/yhDLiNuhyVjojQ= +github.com/gammazero/chanqueue v1.1.1/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc= +github.com/gammazero/deque v1.1.0 h1:OyiyReBbnEG2PP0Bnv1AASLIYvyKqIFN5xfl1t8oGLo= +github.com/gammazero/deque v1.1.0/go.mod h1:JVrR+Bj1NMQbPnYclvDlvSX0nVGReLrQZ0aUMuWLctg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= @@ -75,8 +75,8 @@ github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a h1://KbezygeMJZCSHH+HgUZiTeSoiuFspbMg1ge+eFj18= -github.com/google/pprof v0.0.0-20250607225305-033d6d78b36a/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA= +github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 h1:xhMrHhTJ6zxu3gA4enFM9MLn9AY7613teCdFnlUVbSQ= +github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= @@ -94,8 +94,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= -github.com/ihciah/rust2go v0.0.0-20250717023419-6e5b74b1f846 h1:fwZvyVNjoVf4enaKAbWf5rlFFvAMPT72ta0YDlZfvgc= -github.com/ihciah/rust2go v0.0.0-20250717023419-6e5b74b1f846/go.mod h1:SpcZZoVYDXYhx373JZ1onR8dnEcalKA0H61x/FJ9O0s= +github.com/ihciah/rust2go v0.0.0-20250726175549-557d7a3a4e27 h1:UumvcTbl6Vg04nu3Q4fI27aO4VhXQX0BXY+nvLRRxCc= +github.com/ihciah/rust2go v0.0.0-20250726175549-557d7a3a4e27/go.mod h1:SpcZZoVYDXYhx373JZ1onR8dnEcalKA0H61x/FJ9O0s= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.33.0 h1:9ow3chwkDzMj0Deq4AWRUEI7WnIIV7SZhPTzzG2mmfw= @@ -114,8 +114,8 @@ github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= github.com/ipfs/go-ipld-format v0.6.2 h1:bPZQ+A05ol0b3lsJSl0bLvwbuQ+HQbSsdGTy4xtYUkU= github.com/ipfs/go-ipld-format v0.6.2/go.mod h1:nni2xFdHKx5lxvXJ6brt/pndtGxKAE+FPR1rg4jTkyk= -github.com/ipfs/go-log/v2 v2.6.0 h1:2Nu1KKQQ2ayonKp4MPo6pXCjqw1ULc9iohRqWV5EYqg= -github.com/ipfs/go-log/v2 v2.6.0/go.mod h1:p+Efr3qaY5YXpx9TX7MoLCSEZX5boSWj9wh86P5HJa8= +github.com/ipfs/go-log/v2 v2.8.0 h1:SptNTPJQV3s5EF4FdrTu/yVdOKfGbDgn1EBZx4til2o= +github.com/ipfs/go-log/v2 v2.8.0/go.mod h1:2LEEhdv8BGubPeSFTyzbqhCqrwqxCbuTNTLWqgNAipo= github.com/ipfs/go-metrics-interface v0.3.0 h1:YwG7/Cy4R94mYDUuwsBfeziJCVm9pBMJ6q/JR9V40TU= github.com/ipfs/go-metrics-interface v0.3.0/go.mod h1:OxxQjZDGocXVdyTPocns6cOLwHieqej/jos7H4POwoY= github.com/ipfs/go-peertaskqueue v0.8.2 h1:PaHFRaVFdxQk1Qo3OKiHPYjmmusQy7gKQUaL8JDszAU= @@ -136,8 +136,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -153,10 +153,10 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= -github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw= -github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc= -github.com/libp2p/go-libp2p v0.42.0 h1:A8foZk+ZEhZTv0Jb++7xUFlrFhBDv4j2Vh/uq4YX+KE= -github.com/libp2p/go-libp2p v0.42.0/go.mod h1:4NGcjbD9OIvFiSRb0XueCO19zJ4kSPK5vkyyOUYmMro= +github.com/libp2p/go-flow-metrics v0.3.0 h1:q31zcHUvHnwDO0SHaukewPYgwOBSxtt830uJtUx6784= +github.com/libp2p/go-flow-metrics v0.3.0/go.mod h1:nuhlreIwEguM1IvHAew3ij7A8BMlyHQJ279ao24eZZo= +github.com/libp2p/go-libp2p v0.42.1 h1:Rt8+5thie729NQk1gx1h/2t/+VIafWcqR1I+Kvw+UTg= +github.com/libp2p/go-libp2p v0.42.1/go.mod h1:4NGcjbD9OIvFiSRb0XueCO19zJ4kSPK5vkyyOUYmMro= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-kad-dht v0.33.1 h1:hKFhHMf7WH69LDjaxsJUWOU6qZm71uO47M/a5ijkiP0= @@ -175,8 +175,8 @@ github.com/libp2p/go-netroute v0.2.2 h1:Dejd8cQ47Qx2kRABg6lPwknU7+nBnFRpko45/fFP github.com/libp2p/go-netroute v0.2.2/go.mod h1:Rntq6jUAH0l9Gg17w5bFGhcC9a+vk4KNXs6s7IljKYE= github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= -github.com/libp2p/go-yamux/v5 v5.0.1 h1:f0WoX/bEF2E8SbE4c/k1Mo+/9z0O4oC/hWEA+nfYRSg= -github.com/libp2p/go-yamux/v5 v5.0.1/go.mod h1:en+3cdX51U0ZslwRdRLrvQsdayFt3TSUKvBGErzpWbU= +github.com/libp2p/go-yamux/v5 v5.1.0 h1:8Qlxj4E9JGJAQVW6+uj2o7mqkqsIVlSUGmTWhlXzoHE= +github.com/libp2p/go-yamux/v5 v5.1.0/go.mod h1:tgIQ07ObtRR/I0IWsFOyQIL9/dR5UXgc2s8xKmNZv1o= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= @@ -185,8 +185,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= -github.com/miekg/dns v1.1.66 h1:FeZXOS3VCVsKnEAd+wBkjMC3D2K+ww66Cq3VnCINuJE= -github.com/miekg/dns v1.1.66/go.mod h1:jGFzBsSNbJw6z1HYut1RKBKHA9PBdxeHrZG8J+gC2WE= +github.com/miekg/dns v1.1.67 h1:kg0EHj0G4bfT5/oOys6HhZw4vmMlnoZ+gDu8tJ/AlI0= +github.com/miekg/dns v1.1.67/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -247,20 +247,20 @@ github.com/pion/ice/v4 v4.0.10/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu1 github.com/pion/interceptor v0.1.40 h1:e0BjnPcGpr2CFQgKhrQisBU7V3GXK6wrfYrGYaU6Jq4= github.com/pion/interceptor v0.1.40/go.mod h1:Z6kqH7M/FYirg3frjGJ21VLSRJGBXB/KqaTIrdqnOic= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= -github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI= -github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90= +github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= +github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo= github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0= -github.com/pion/rtp v1.8.19 h1:jhdO/3XhL/aKm/wARFVmvTfq0lC/CvN1xwYKmduly3c= -github.com/pion/rtp v1.8.19/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= +github.com/pion/rtp v1.8.21 h1:3yrOwmZFyUpcIosNcWRpQaU+UXIJ6yxLuJ8Bx0mw37Y= +github.com/pion/rtp v1.8.21/go.mod h1:bAu2UFKScgzyFqvUKmbvzSdPr+NGbZtv6UB2hesqXBk= github.com/pion/sctp v1.8.39 h1:PJma40vRHa3UTO3C4MyeJDQ+KIobVYRZQZ0Nt7SjQnE= github.com/pion/sctp v1.8.39/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE= -github.com/pion/sdp/v3 v3.0.13 h1:uN3SS2b+QDZnWXgdr69SM8KB4EbcnPnPf2Laxhty/l4= -github.com/pion/sdp/v3 v3.0.13/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E= +github.com/pion/sdp/v3 v3.0.15 h1:F0I1zds+K/+37ZrzdADmx2Q44OFDOPRLhPnNTaUX9hk= +github.com/pion/sdp/v3 v3.0.15/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E= github.com/pion/srtp/v3 v3.0.6 h1:E2gyj1f5X10sB/qILUGIkL4C2CqK269Xq167PbGCc/4= github.com/pion/srtp/v3 v3.0.6/go.mod h1:BxvziG3v/armJHAaJ87euvkhHqWe9I7iiOy50K2QkhY= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= @@ -275,8 +275,8 @@ github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1 github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps= github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs= -github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54= -github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U= +github.com/pion/webrtc/v4 v4.1.3 h1:YZ67Boj9X/hk190jJZ8+HFGQ6DqSZ/fYP3sLAZv7c3c= +github.com/pion/webrtc/v4 v4.1.3/go.mod h1:rsq+zQ82ryfR9vbb0L1umPJ6Ogq7zm8mcn9fcGnxomM= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -285,17 +285,17 @@ github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= +github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= -github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= +github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= +github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.52.0 h1:/SlHrCRElyaU6MaEPKqKr9z83sBg2v4FLLvWM+Z47pA= @@ -364,12 +364,12 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= @@ -397,11 +397,11 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= -golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -409,8 +409,8 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= -golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -430,8 +430,8 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -445,8 +445,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -464,8 +464,8 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -481,8 +481,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= @@ -497,8 +497,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= -golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= From f7c631f111535e21021e50f53294b533ca59eb19 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 31 Jul 2025 20:17:10 +0800 Subject: [PATCH 13/35] integrate with go-f3 --- f3-sidecar/api.go | 22 ++++++++++++ f3-sidecar/ffi_gen.go | 13 +++++++ f3-sidecar/ffi_impl.go | 7 ++++ f3-sidecar/import.go | 23 ++++++++++++ f3-sidecar/main.go | 8 +++++ f3-sidecar/run.go | 4 +-- f3-sidecar/utils.go | 11 +++++- src/chain/mod.rs | 6 ++-- src/f3/go_ffi.rs | 2 ++ src/rpc/methods/chain.rs | 23 +++++++++++- src/rpc/methods/f3.rs | 36 ++++++++++++++++++- src/rpc/mod.rs | 1 + .../api_cmd/test_snapshots_ignored.txt | 1 + 13 files changed, 148 insertions(+), 9 deletions(-) create mode 100644 f3-sidecar/import.go diff --git a/f3-sidecar/api.go b/f3-sidecar/api.go index 36c63fdcb0c6..b9cbea362dea 100644 --- a/f3-sidecar/api.go +++ b/f3-sidecar/api.go @@ -1,13 +1,16 @@ package main import ( + "bufio" "context" + "os" "github.com/filecoin-project/go-f3" "github.com/filecoin-project/go-f3/certs" "github.com/filecoin-project/go-f3/gpbft" "github.com/filecoin-project/go-f3/manifest" "github.com/filecoin-project/go-state-types/crypto" + "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" ) @@ -51,6 +54,25 @@ func (h *F3ServerHandler) F3GetF3PowerTable(ctx context.Context, tsk []byte) (gp return h.f3.GetPowerTable(ctx, tsk) } +func (h *F3ServerHandler) F3ExportLatestSnapshot(ctx context.Context, path string) (*cid.Cid, error) { + cs, err := h.f3.GetCertStore() + if err != nil { + return nil, err + } + + f, err := os.Create(path) + if err != nil { + return nil, err + } + + writer := bufio.NewWriter(f) + cid, _, err := cs.ExportLatestSnapshot(ctx, writer) + if err != nil { + return nil, err + } + return &cid, nil +} + // F3GetF3PowerTableByInstance retrieves the power table for a specific consensus instance. // It returns the power entries associated with the given instance number. // diff --git a/f3-sidecar/ffi_gen.go b/f3-sidecar/ffi_gen.go index edc7f5941af3..7d12ee09234e 100644 --- a/f3-sidecar/ffi_gen.go +++ b/f3-sidecar/ffi_gen.go @@ -30,6 +30,7 @@ var GoF3NodeImpl GoF3Node type GoF3Node interface { run(rpc_endpoint *string, jwt *string, f3_rpc_endpoint *string, initial_power_table *string, bootstrap_epoch *int64, finality *int64, f3_root *string) bool + import_snap(f3_root *string, snapshot *string) string } //export CGoF3Node_run @@ -49,6 +50,18 @@ func CGoF3Node_run(rpc_endpoint C.StringRef, jwt C.StringRef, f3_rpc_endpoint C. runtime.KeepAlive(buffer) } +//export CGoF3Node_import_snap +func CGoF3Node_import_snap(f3_root C.StringRef, snapshot C.StringRef, slot *C.void, cb *C.void) { + _new_f3_root := newString(f3_root) + _new_snapshot := newString(snapshot) + resp := GoF3NodeImpl.import_snap(&_new_f3_root, &_new_snapshot) + resp_ref, buffer := cvt_ref(cntString, refString)(&resp) + asmcall.CallFuncG0P2(unsafe.Pointer(cb), unsafe.Pointer(&resp_ref), unsafe.Pointer(slot)) + runtime.KeepAlive(resp_ref) + runtime.KeepAlive(resp) + runtime.KeepAlive(buffer) +} + func newString(s_ref C.StringRef) string { return unsafe.String((*byte)(unsafe.Pointer(s_ref.ptr)), s_ref.len) } diff --git a/f3-sidecar/ffi_impl.go b/f3-sidecar/ffi_impl.go index e58a1c173d98..04c05d4a4976 100644 --- a/f3-sidecar/ffi_impl.go +++ b/f3-sidecar/ffi_impl.go @@ -48,6 +48,13 @@ func (f3 *f3Impl) run(rpc_endpoint *string, jwt *string, f3_rpc_endpoint *string return err == nil } +func (f3 *f3Impl) import_snap(f3_root *string, snapshot *string) string { + if err := importSnap(f3.ctx, *f3_root, *snapshot); err != nil { + return err.Error() + } + return "" +} + func checkError(err error) { if err != nil { panic(err) diff --git a/f3-sidecar/import.go b/f3-sidecar/import.go new file mode 100644 index 000000000000..287ec58c7912 --- /dev/null +++ b/f3-sidecar/import.go @@ -0,0 +1,23 @@ +package main + +import ( + "bufio" + "context" + "os" + + "github.com/filecoin-project/go-f3/certstore" +) + +func importSnap(ctx context.Context, f3Root string, snapshot string) error { + logger.Infof("importing F3 snapshot at %s", snapshot) + ds, err := getDatastore(f3Root) + if err != nil { + return err + } + f, err := os.Open(snapshot) + if err != nil { + return err + } + certstore.ImportSnapshotToDatastore(ctx, bufio.NewReader(f), ds) + return nil +} diff --git a/f3-sidecar/main.go b/f3-sidecar/main.go index 3fa559e69221..5d8b19705503 100644 --- a/f3-sidecar/main.go +++ b/f3-sidecar/main.go @@ -41,11 +41,19 @@ func main() { flag.Int64Var(&finality, "finality", 900, "chain finality epochs") var root string flag.StringVar(&root, "root", "f3-data", "path to the f3 data directory") + var snapshot string + flag.StringVar(&snapshot, "snapshot", "", "path to the f3 snapshot file") flag.Parse() ctx := context.Background() + if len(snapshot) > 0 { + if err := importSnap(ctx, root, snapshot); err != nil { + panic(err) + } + } + err := run(ctx, rpcEndpoint, jwt, f3RpcEndpoint, initialPowerTable, bootstrapEpoch, finality, root) if err != nil { panic(err) diff --git a/f3-sidecar/run.go b/f3-sidecar/run.go index 64e6236da201..1f3659cdfdb5 100644 --- a/f3-sidecar/run.go +++ b/f3-sidecar/run.go @@ -6,7 +6,6 @@ import ( "fmt" "net" "net/http" - "path/filepath" "time" "github.com/filecoin-project/go-f3" @@ -15,7 +14,6 @@ import ( "github.com/filecoin-project/go-f3/manifest" "github.com/filecoin-project/go-jsonrpc" "github.com/ipfs/go-cid" - leveldb "github.com/ipfs/go-ds-leveldb" ) func run(ctx context.Context, rpcEndpoint string, jwt string, f3RpcEndpoint string, initialPowerTable string, bootstrapEpoch int64, finality int64, f3Root string) error { @@ -60,7 +58,7 @@ func run(ctx context.Context, rpcEndpoint string, jwt string, f3RpcEndpoint stri if err != nil { return err } - ds, err := leveldb.NewDatastore(filepath.Join(f3Root, "db"), nil) + ds, err := getDatastore(f3Root) if err != nil { return err } diff --git a/f3-sidecar/utils.go b/f3-sidecar/utils.go index 1309b75ff962..f4081498b08f 100644 --- a/f3-sidecar/utils.go +++ b/f3-sidecar/utils.go @@ -1,9 +1,18 @@ package main -import "github.com/ipfs/go-cid" +import ( + "path/filepath" + + "github.com/ipfs/go-cid" + leveldb "github.com/ipfs/go-ds-leveldb" +) var CID_UNDEF_RUST = cid.MustParse("baeaaaaa") func isCidDefined(c cid.Cid) bool { return c.Defined() && c != CID_UNDEF_RUST } + +func getDatastore(f3Root string) (*leveldb.Datastore, error) { + return leveldb.NewDatastore(filepath.Join(f3Root, "db"), nil) +} diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 6cc16b287b46..b13eb52d3485 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -69,7 +69,7 @@ pub async fn export( pub async fn export_v2( db: &Arc, - f3: Option<(Cid, &mut File)>, + f3: Option<(Cid, File)>, tipset: &Tipset, lookup_depth: ChainEpochDelta, writer: impl AsyncWrite + Unpin, @@ -97,10 +97,10 @@ pub async fn export_v2( )) }]; - if let Some((f3_cid, f3_data)) = f3 { + if let Some((f3_cid, mut f3_data)) = f3 { prefix_data_frames.push({ let mut encoder = forest::new_encoder(forest::DEFAULT_FOREST_CAR_COMPRESSION_LEVEL)?; - encoder.write_car_block(f3_cid, f3_data.metadata()?.len() as _, f3_data)?; + encoder.write_car_block(f3_cid, f3_data.metadata()?.len() as _, &mut f3_data)?; anyhow::Ok(( vec![f3_cid], finalize_frame(forest::DEFAULT_FOREST_CAR_COMPRESSION_LEVEL, &mut encoder)?, diff --git a/src/f3/go_ffi.rs b/src/f3/go_ffi.rs index 7048a3838f4f..c5663fdd719c 100644 --- a/src/f3/go_ffi.rs +++ b/src/f3/go_ffi.rs @@ -18,4 +18,6 @@ pub trait GoF3Node { finality: i64, f3_root: String, ) -> bool; + + fn import_snap(f3_root: String, snapshot: String) -> String; } diff --git a/src/rpc/methods/chain.rs b/src/rpc/methods/chain.rs index a2853b23cbbe..0f0a6ef0a93c 100644 --- a/src/rpc/methods/chain.rs +++ b/src/rpc/methods/chain.rs @@ -16,6 +16,7 @@ use crate::lotus_json::{HasLotusJson, LotusJson, lotus_json_with_self}; #[cfg(test)] use crate::lotus_json::{assert_all_snapshots, assert_unchanged_via_json}; use crate::message::{ChainMessage, SignedMessage}; +use crate::rpc::f3::F3ExportLatestSnapshot; use crate::rpc::types::{ApiTipsetKey, Event}; use crate::rpc::{ApiPaths, Ctx, EthEventHandler, Permission, RpcMethod, ServerError}; use crate::shim::clock::ChainEpoch; @@ -36,6 +37,7 @@ use num::BigInt; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sha2::Sha256; +use std::fs::File; use std::{ collections::VecDeque, path::PathBuf, @@ -284,9 +286,28 @@ impl RpcMethod<1> for ForestChainExport { .await } FilecoinSnapshotVersion::V2 => { + let f3_snap_tmp_path = { + let mut f3_snap_dir = output_path.clone(); + if f3_snap_dir.pop() { + tempfile::NamedTempFile::new_in(&f3_snap_dir) + } else { + tempfile::NamedTempFile::new_in(".") + }? + .into_temp_path() + }; + let f3_snap = { + match F3ExportLatestSnapshot::run(f3_snap_tmp_path.display().to_string()).await + { + Ok(cid) => Some((cid, File::open(&f3_snap_tmp_path)?)), + Err(e) => { + tracing::error!("Failed to export F3 snapshot: {e}"); + None + } + } + }; crate::chain::export_v2::( &ctx.store_owned(), - None, + f3_snap, &start_ts, recent_roots, writer, diff --git a/src/rpc/methods/f3.rs b/src/rpc/methods/f3.rs index c2c408a590c9..575e487186c9 100644 --- a/src/rpc/methods/f3.rs +++ b/src/rpc/methods/f3.rs @@ -32,7 +32,7 @@ use crate::{ LruBlockstoreReadCache, }, libp2p::{NetRPCMethods, NetworkMessage}, - lotus_json::HasLotusJson as _, + lotus_json::{HasLotusJson as _, LotusJson}, rpc::{ApiPaths, Ctx, Permission, RpcMethod, ServerError, types::ApiTipsetKey}, shim::{ address::{Address, Protocol}, @@ -43,6 +43,7 @@ use crate::{ }; use ahash::{HashMap, HashSet}; use anyhow::Context as _; +use cid::Cid; use enumflags2::BitFlags; use fvm_ipld_blockstore::Blockstore; use jsonrpsee::core::{client::ClientT as _, params::ArrayParams}; @@ -623,6 +624,39 @@ impl RpcMethod<2> for SignMessage { } } +pub enum F3ExportLatestSnapshot {} + +impl F3ExportLatestSnapshot { + pub async fn run(path: String) -> anyhow::Result { + let client = get_rpc_http_client()?; + let mut params = ArrayParams::new(); + params.insert(path)?; + let LotusJson(cid): LotusJson = client + .request("Filecoin.F3ExportLatestSnapshot", params) + .await?; + Ok(cid) + } +} + +impl RpcMethod<1> for F3ExportLatestSnapshot { + const NAME: &'static str = "F3.ExportLatestSnapshot"; + const PARAM_NAMES: [&'static str; 1] = ["path"]; + const API_PATHS: BitFlags = ApiPaths::all(); + const PERMISSION: Permission = Permission::Read; + const DESCRIPTION: Option<&'static str> = + Some("Gets the power table (committee) used to validate the specified instance"); + + type Params = (String,); + type Ok = Cid; + + async fn handle( + _ctx: Ctx, + (path,): Self::Params, + ) -> Result { + Ok(Self::run(path).await?) + } +} + /// returns a finality certificate at given instance number pub enum F3GetCertificate {} impl RpcMethod<1> for F3GetCertificate { diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index 00d836b41850..433deef4f063 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -280,6 +280,7 @@ macro_rules! for_each_rpc_method { $callback!($crate::rpc::f3::F3GetLatestCertificate); $callback!($crate::rpc::f3::F3GetOrRenewParticipationTicket); $callback!($crate::rpc::f3::F3Participate); + $callback!($crate::rpc::f3::F3ExportLatestSnapshot); $callback!($crate::rpc::f3::GetHead); $callback!($crate::rpc::f3::GetParent); $callback!($crate::rpc::f3::GetParticipatingMinerIDs); diff --git a/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt b/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt index 83b21d6206a7..3f141626363f 100644 --- a/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt +++ b/src/tool/subcommands/api_cmd/test_snapshots_ignored.txt @@ -1,3 +1,4 @@ +F3.ExportLatestSnapshot F3.Finalize F3.GetHead F3.GetParent From fa83aaae1de73565a69835c43ef2591ce8906208 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 1 Aug 2025 11:35:14 +0800 Subject: [PATCH 14/35] f3 snap import --- Cargo.lock | 1 - Cargo.toml | 1 - f3-sidecar/import.go | 2 ++ f3-sidecar/run.go | 1 + src/daemon/context.rs | 4 +++ src/daemon/db_util.rs | 29 +++++++++++++++++++- src/daemon/mod.rs | 54 ++++++++++++++++++-------------------- src/db/car/any.rs | 13 ++++++++- src/db/car/forest.rs | 21 +++++++++++++++ src/db/car/plain.rs | 11 ++++++++ src/f3/mod.rs | 19 ++++++++++++++ src/rpc/methods/chain.rs | 6 +++-- src/utils/db/car_stream.rs | 4 +-- 13 files changed, 130 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd073aaa2dd0..c643810e2d86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3107,7 +3107,6 @@ dependencies = [ "directories", "displaydoc", "educe", - "either", "enumflags2", "ethereum-types", "ez-jsonrpc-types", diff --git a/Cargo.toml b/Cargo.toml index ced6fa432dbb..9df5a7fcf4d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,6 @@ digest = "0.10" directories = "6" displaydoc = "0.2" educe = { version = "0.6.0", features = ["Debug"], default-features = false } -either = "1" enumflags2 = "0.7" ethereum-types = { version = "0.15", features = ["ethbloom"] } ez-jsonrpc-types = "0.5" diff --git a/f3-sidecar/import.go b/f3-sidecar/import.go index 287ec58c7912..188db9c414e9 100644 --- a/f3-sidecar/import.go +++ b/f3-sidecar/import.go @@ -14,10 +14,12 @@ func importSnap(ctx context.Context, f3Root string, snapshot string) error { if err != nil { return err } + defer ds.Close() f, err := os.Open(snapshot) if err != nil { return err } + defer f.Close() certstore.ImportSnapshotToDatastore(ctx, bufio.NewReader(f), ds) return nil } diff --git a/f3-sidecar/run.go b/f3-sidecar/run.go index 1f3659cdfdb5..de0f7b7f1735 100644 --- a/f3-sidecar/run.go +++ b/f3-sidecar/run.go @@ -62,6 +62,7 @@ func run(ctx context.Context, rpcEndpoint string, jwt string, f3RpcEndpoint stri if err != nil { return err } + defer ds.Close() verif := blssig.VerifierWithKeyOnG1() networkName := gpbft.NetworkName(rawNetwork) // Use "filecoin" as the network name on mainnet, otherwise use the network name. Yes, diff --git a/src/daemon/context.rs b/src/daemon/context.rs index 40d6b6f886d7..48f220a235b0 100644 --- a/src/daemon/context.rs +++ b/src/daemon/context.rs @@ -61,6 +61,10 @@ impl AppContext { snapshot_progress_tracker, }) } + + pub fn chain_config(&self) -> &Arc { + self.state_manager.chain_config() + } } fn get_chain_config_and_set_network(config: &Config) -> Arc { diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index 4f250b73a712..1cd4328116a9 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -18,6 +18,7 @@ use anyhow::{Context, bail}; use futures::TryStreamExt; use serde::{Deserialize, Serialize}; use std::ffi::OsStr; +use std::fs::File; use std::{ fs, path::{Path, PathBuf}, @@ -131,6 +132,8 @@ pub async fn import_chain_as_forest_car( from_path: &Path, forest_car_db_dir: &Path, import_mode: ImportMode, + f3_root: String, + is_sidecar_ffi_enabled: bool, snapshot_progress_tracker: &SnapshotProgressTracker, ) -> anyhow::Result<(PathBuf, Tipset)> { info!("Importing chain from snapshot at: {}", from_path.display()); @@ -230,7 +233,29 @@ pub async fn import_chain_as_forest_car( } }; - let ts = ForestCar::try_from(forest_car_db_path.as_path())?.heaviest_tipset()?; + let forest_car = ForestCar::try_from(forest_car_db_path.as_path())?; + + if !is_sidecar_ffi_enabled { + tracing::warn!("F3 sidecar is disabled, skip importing F3 snapshot"); + } else if let Some(f3_cid) = forest_car.metadata().as_ref().and_then(|m| m.f3_data) + && let Some(mut f3_data) = forest_car.get_reader(f3_cid)? + { + let temp_f3_snap_path = tempfile::Builder::new() + .suffix(".f3snap.bin") + .tempfile_in(forest_car_db_dir)? + .into_temp_path(); + { + let mut f = File::create(&temp_f3_snap_path)?; + std::io::copy(&mut f3_data, &mut f)?; + } + // #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] + { + tracing::info!("Importing F3 snapshot ..."); + crate::f3::import_f3_snapshot(f3_root, temp_f3_snap_path.display().to_string()); + } + } + + let ts = forest_car.heaviest_tipset()?; info!( "Imported snapshot in: {}s, heaviest tipset epoch: {}, key: {}", stopwatch.elapsed().as_secs(), @@ -495,6 +520,8 @@ mod test { file_path, temp_db_dir.path(), import_mode, + "".into(), + true, &SnapshotProgressTracker::default(), ) .await?; diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 49d4a2b1b472..c8f2890692e6 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -133,28 +133,30 @@ async fn maybe_import_snapshot( let snapshot_tracker = ctx.snapshot_progress_tracker.clone(); // Import chain if needed - if !opts.skip_load.unwrap_or_default() { - if let Some(path) = &config.client.snapshot_path { - let (car_db_path, ts) = import_chain_as_forest_car( - path, - &ctx.db_meta_data.get_forest_car_db_dir(), - config.client.import_mode, - &snapshot_tracker, - ) - .await?; - ctx.db - .read_only_files(std::iter::once(car_db_path.clone()))?; - let ts_epoch = ts.epoch(); - // Explicitly set heaviest tipset here in case HEAD_KEY has already been set - // in the current setting store - ctx.state_manager - .chain_store() - .set_heaviest_tipset(ts.into())?; - debug!( - "Loaded car DB at {} and set current head to epoch {ts_epoch}", - car_db_path.display(), - ); - } + if !opts.skip_load.unwrap_or_default() + && let Some(path) = &config.client.snapshot_path + { + let (car_db_path, ts) = import_chain_as_forest_car( + path, + &ctx.db_meta_data.get_forest_car_db_dir(), + config.client.import_mode, + crate::f3::get_f3_root(config), + crate::f3::is_sidecar_ffi_enabled(ctx.chain_config()), + &snapshot_tracker, + ) + .await?; + ctx.db + .read_only_files(std::iter::once(car_db_path.clone()))?; + let ts_epoch = ts.epoch(); + // Explicitly set heaviest tipset here in case HEAD_KEY has already been set + // in the current setting store + ctx.state_manager + .chain_store() + .set_heaviest_tipset(ts.into())?; + debug!( + "Loaded car DB at {} and set current head to epoch {ts_epoch}", + car_db_path.display(), + ); } // If the snapshot progress state is not completed, @@ -418,10 +420,7 @@ fn maybe_start_f3_service(opts: &CliOpts, config: &Config, ctx: &AppContext) { )) .expect("F3 lease manager should not have been initialized before"); let chain_config = state_manager.chain_config().clone(); - let default_f3_root = config - .client - .data_dir - .join(format!("f3/{}", config.chain())); + let f3_root = crate::f3::get_f3_root(config); let crate::f3::F3Options { chain_finality, bootstrap_epoch, @@ -438,8 +437,7 @@ fn maybe_start_f3_service(opts: &CliOpts, config: &Config, ctx: &AppContext) { .unwrap_or_default(), bootstrap_epoch, chain_finality, - std::env::var("FOREST_F3_ROOT") - .unwrap_or(default_f3_root.display().to_string()), + f3_root, ); } }); diff --git a/src/db/car/any.rs b/src/db/car/any.rs index fc271c6fe1d4..a247c901ff95 100644 --- a/src/db/car/any.rs +++ b/src/db/car/any.rs @@ -14,9 +14,10 @@ use crate::chain::FilecoinSnapshotMetadata; use crate::utils::io::EitherMmapOrRandomAccessFile; use cid::Cid; use fvm_ipld_blockstore::Blockstore; +use itertools::Either; use positioned_io::ReadAt; use std::borrow::Cow; -use std::io::{Error, ErrorKind, Result}; +use std::io::{Error, ErrorKind, Read, Result}; use std::path::Path; use std::sync::Arc; @@ -112,6 +113,16 @@ impl AnyCar { _ => None, } } + + #[allow(dead_code)] + /// Gets a reader of the block data by its `Cid` + pub fn get_reader(&self, k: Cid) -> anyhow::Result> { + match self { + Self::Forest(car) => Ok(car.get_reader(k)?.map(Either::Left)), + Self::Plain(car) => Ok(car.get_reader(k).map(|r| Either::Right(Either::Left(r)))), + Self::Memory(car) => Ok(car.get_reader(k).map(|r| Either::Right(Either::Right(r)))), + } + } } impl TryFrom<&'static [u8]> for AnyCar<&'static [u8]> { diff --git a/src/db/car/forest.rs b/src/db/car/forest.rs index 868f6dc1596b..c319960e7177 100644 --- a/src/db/car/forest.rs +++ b/src/db/car/forest.rs @@ -61,6 +61,7 @@ use cid::Cid; use futures::{Stream, TryStreamExt as _}; use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::CborStore as _; +use integer_encoding::VarIntReader; use nunny::Vec as NonEmpty; use positioned_io::{Cursor, ReadAt, ReadBytesAtExt, SizeCursor}; use std::io::{Seek, SeekFrom}; @@ -212,6 +213,26 @@ impl ForestCar { ..self } } + + /// Gets a reader of the block data by its `Cid` + pub fn get_reader(&self, k: Cid) -> anyhow::Result>> { + for position in self.indexed.get(k)? { + // Decode entire frame into memory, "position" arg is the frame start offset. + let entire_file = self.indexed.reader().get_ref(); // escape the positioned_io::Slice + let cursor = Cursor::new_pos(entire_file, position); + let mut decoder = zstd::Decoder::new(cursor)?.single_frame(); + while let Ok(frame_len) = decoder.read_varint::() { + let cid = Cid::read_bytes(&mut decoder)?; + let data_len = frame_len.saturating_sub(cid.encoded_len()) as u64; + if cid == k { + return Ok(Some(decoder.take(data_len))); + } + // Discard data bytes + io::copy(&mut decoder.by_ref().take(data_len), &mut io::sink())?; + } + } + Ok(None) + } } impl TryFrom<&Path> for ForestCar { diff --git a/src/db/car/plain.rs b/src/db/car/plain.rs index d67d2daefc79..467ba4f5181f 100644 --- a/src/db/car/plain.rs +++ b/src/db/car/plain.rs @@ -224,6 +224,17 @@ impl PlainCar { metadata: self.metadata, } } + + #[allow(dead_code)] + /// Gets a reader of the block data by its `Cid` + pub fn get_reader(&self, k: Cid) -> Option> { + self.index + .read() + .get(&k) + .map(|UncompressedBlockDataLocation { offset, length }| { + positioned_io::Cursor::new_pos(&self.reader, *offset).take(*length as u64) + }) + } } impl TryFrom<&'static [u8]> for PlainCar<&'static [u8]> { diff --git a/src/f3/mod.rs b/src/f3/mod.rs index 1775723c72f2..dc7d7feef3bc 100644 --- a/src/f3/mod.rs +++ b/src/f3/mod.rs @@ -19,6 +19,17 @@ pub struct F3Options { pub initial_power_table: Option, } +pub fn get_f3_root(config: &crate::Config) -> String { + std::env::var("FOREST_F3_ROOT").unwrap_or_else(|_| { + config + .client + .data_dir + .join(format!("f3/{}", config.chain())) + .display() + .to_string() + }) +} + pub fn get_f3_sidecar_params(chain_config: &ChainConfig) -> F3Options { let chain_finality = std::env::var("FOREST_F3_FINALITY") .ok() @@ -100,6 +111,14 @@ pub fn run_f3_sidecar_if_enabled( } } +pub fn import_f3_snapshot(_f3_root: String, _snapshot: String) { + #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] + { + tracing::info!("Importing F3 snapshot ..."); + GoF3NodeImpl::import_snap(_f3_root, _snapshot); + } +} + /// Whether F3 sidecar via FFI is enabled. pub fn is_sidecar_ffi_enabled(chain_config: &ChainConfig) -> bool { // Respect the environment variable when set, and fallback to chain config when not set. diff --git a/src/rpc/methods/chain.rs b/src/rpc/methods/chain.rs index 0f0a6ef0a93c..29e9b55f3d07 100644 --- a/src/rpc/methods/chain.rs +++ b/src/rpc/methods/chain.rs @@ -288,10 +288,12 @@ impl RpcMethod<1> for ForestChainExport { FilecoinSnapshotVersion::V2 => { let f3_snap_tmp_path = { let mut f3_snap_dir = output_path.clone(); + let mut builder = tempfile::Builder::new(); + let with_suffix = builder.suffix(".f3snap.bin"); if f3_snap_dir.pop() { - tempfile::NamedTempFile::new_in(&f3_snap_dir) + with_suffix.tempfile_in(&f3_snap_dir) } else { - tempfile::NamedTempFile::new_in(".") + with_suffix.tempfile_in(".") }? .into_temp_path() }; diff --git a/src/utils/db/car_stream.rs b/src/utils/db/car_stream.rs index e5bc41cd9e0c..99a5c296b448 100644 --- a/src/utils/db/car_stream.rs +++ b/src/utils/db/car_stream.rs @@ -231,9 +231,9 @@ impl CarStream { fn try_decode_header_v2_from_fill_buf(fill_buf: &[u8]) -> io::Result> { let is_compressed = is_zstd(fill_buf); let fill_buf_reader = if is_compressed { - either::Either::Right(zstd::Decoder::new(fill_buf)?) + itertools::Either::Right(zstd::Decoder::new(fill_buf)?) } else { - either::Either::Left(fill_buf) + itertools::Either::Left(fill_buf) }; read_v2_header(fill_buf_reader) } From fb63dfa9dd61635c03d19395315683d0213c7ee7 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 1 Aug 2025 13:40:12 +0800 Subject: [PATCH 15/35] integrity check of f3 data --- f3-sidecar/api.go | 2 ++ f3-sidecar/import.go | 3 +-- src/chain/mod.rs | 26 ++++++++++++++++++++------ src/daemon/db_util.rs | 6 +----- src/f3/mod.rs | 13 +++++++++++-- 5 files changed, 35 insertions(+), 15 deletions(-) diff --git a/f3-sidecar/api.go b/f3-sidecar/api.go index b9cbea362dea..91730822a70a 100644 --- a/f3-sidecar/api.go +++ b/f3-sidecar/api.go @@ -64,8 +64,10 @@ func (h *F3ServerHandler) F3ExportLatestSnapshot(ctx context.Context, path strin if err != nil { return nil, err } + defer f.Close() writer := bufio.NewWriter(f) + defer writer.Flush() cid, _, err := cs.ExportLatestSnapshot(ctx, writer) if err != nil { return nil, err diff --git a/f3-sidecar/import.go b/f3-sidecar/import.go index 188db9c414e9..dea2d504c819 100644 --- a/f3-sidecar/import.go +++ b/f3-sidecar/import.go @@ -20,6 +20,5 @@ func importSnap(ctx context.Context, f3Root string, snapshot string) error { return err } defer f.Close() - certstore.ImportSnapshotToDatastore(ctx, bufio.NewReader(f), ds) - return nil + return certstore.ImportSnapshotToDatastore(ctx, bufio.NewReader(f), ds) } diff --git a/src/chain/mod.rs b/src/chain/mod.rs index b13eb52d3485..6a448d6c9e7f 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -17,7 +17,7 @@ use cid::Cid; use digest::Digest; use futures::StreamExt as _; use fvm_ipld_blockstore::Blockstore; -use fvm_ipld_encoding::DAG_CBOR; +use fvm_ipld_encoding::{DAG_CBOR, IPLD_RAW}; use itertools::Itertools as _; use multihash_derive::MultihashDigest as _; use num::FromPrimitive as _; @@ -26,6 +26,7 @@ use nunny::Vec as NonEmpty; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::fs::File; +use std::io::{Seek as _, SeekFrom}; use std::sync::Arc; use tokio::io::{AsyncWrite, AsyncWriteExt, BufWriter}; @@ -37,7 +38,7 @@ pub async fn export_from_head( writer: impl AsyncWrite + Unpin, seen: CidHashSet, skip_checksum: bool, -) -> anyhow::Result<(Tipset, Option>), Error> { +) -> anyhow::Result<(Tipset, Option>)> { let head_key = SettingsStoreExt::read_obj::(db, crate::db::setting_keys::HEAD_KEY)? .context("chain head key not found")?; let head_ts = Tipset::load_required(&db, &head_key)?; @@ -52,7 +53,7 @@ pub async fn export( writer: impl AsyncWrite + Unpin, seen: CidHashSet, skip_checksum: bool, -) -> anyhow::Result>, Error> { +) -> anyhow::Result>> { let roots = tipset.key().to_cids(); export_to_forest_car::( roots, @@ -69,13 +70,26 @@ pub async fn export( pub async fn export_v2( db: &Arc, - f3: Option<(Cid, File)>, + mut f3: Option<(Cid, File)>, tipset: &Tipset, lookup_depth: ChainEpochDelta, writer: impl AsyncWrite + Unpin, seen: CidHashSet, skip_checksum: bool, -) -> anyhow::Result>, Error> { +) -> anyhow::Result>> { + // validate f3 data + if let Some((f3_cid, f3_data)) = &mut f3 { + let expected_cid = Cid::new_v1( + IPLD_RAW, + MultihashCode::Blake2b256.digest_byte_stream(f3_data)?, + ); + f3_data.seek(SeekFrom::Start(0))?; + anyhow::ensure!( + f3_cid == &expected_cid, + "f3 snapshot integrity check failed, actual cid: {f3_cid}, expected cid: {expected_cid}" + ); + } + let head = tipset.key().to_cids(); let f3_cid = f3.as_ref().map(|(cid, _)| *cid); let snap_meta = FilecoinSnapshotMetadata::new_v2(head, f3_cid); @@ -131,7 +145,7 @@ async fn export_to_forest_car( writer: impl AsyncWrite + Unpin, seen: CidHashSet, skip_checksum: bool, -) -> anyhow::Result>, Error> { +) -> anyhow::Result>> { let stateroot_lookup_limit = tipset.epoch() - lookup_depth; // Wrap writer in optional checksum calculator diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index 1cd4328116a9..9ffa5eeae1e8 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -248,11 +248,7 @@ pub async fn import_chain_as_forest_car( let mut f = File::create(&temp_f3_snap_path)?; std::io::copy(&mut f3_data, &mut f)?; } - // #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] - { - tracing::info!("Importing F3 snapshot ..."); - crate::f3::import_f3_snapshot(f3_root, temp_f3_snap_path.display().to_string()); - } + crate::f3::import_f3_snapshot(f3_root, temp_f3_snap_path.display().to_string())?; } let ts = forest_car.heaviest_tipset()?; diff --git a/src/f3/mod.rs b/src/f3/mod.rs index dc7d7feef3bc..05b9d2132aa9 100644 --- a/src/f3/mod.rs +++ b/src/f3/mod.rs @@ -111,12 +111,21 @@ pub fn run_f3_sidecar_if_enabled( } } -pub fn import_f3_snapshot(_f3_root: String, _snapshot: String) { +pub fn import_f3_snapshot(_f3_root: String, _snapshot: String) -> anyhow::Result<()> { #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] { + let sw = std::time::Instant::now(); tracing::info!("Importing F3 snapshot ..."); - GoF3NodeImpl::import_snap(_f3_root, _snapshot); + let err = GoF3NodeImpl::import_snap(_f3_root, _snapshot); + if !err.is_empty() { + anyhow::bail!("{err}"); + } + tracing::info!( + "Imported F3 snapshot, took {}", + humantime::format_duration(sw.elapsed()) + ); } + Ok(()) } /// Whether F3 sidecar via FFI is enabled. From 7713315637fce4df6cf107176f07c21f9fab5fa0 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 1 Aug 2025 17:20:29 +0800 Subject: [PATCH 16/35] fix f3 snap import --- f3-sidecar/ffi_gen.go | 7 +-- f3-sidecar/ffi_impl.go | 4 +- f3-sidecar/go.mod | 2 + f3-sidecar/go.sum | 4 +- f3-sidecar/import.go | 100 ++++++++++++++++++++++++++++++++++- f3-sidecar/main.go | 2 +- f3-sidecar/run.go | 19 +------ f3-sidecar/utils.go | 26 +++++++++ src/cli_shared/cli/client.rs | 4 ++ src/daemon/db_util.rs | 10 +++- src/daemon/mod.rs | 5 +- src/f3/go_ffi.rs | 2 +- src/f3/mod.rs | 8 ++- 13 files changed, 160 insertions(+), 33 deletions(-) diff --git a/f3-sidecar/ffi_gen.go b/f3-sidecar/ffi_gen.go index 7d12ee09234e..85f5ffb6f714 100644 --- a/f3-sidecar/ffi_gen.go +++ b/f3-sidecar/ffi_gen.go @@ -30,7 +30,7 @@ var GoF3NodeImpl GoF3Node type GoF3Node interface { run(rpc_endpoint *string, jwt *string, f3_rpc_endpoint *string, initial_power_table *string, bootstrap_epoch *int64, finality *int64, f3_root *string) bool - import_snap(f3_root *string, snapshot *string) string + import_snap(f3_rpc_endpoint *string, f3_root *string, snapshot *string) string } //export CGoF3Node_run @@ -51,10 +51,11 @@ func CGoF3Node_run(rpc_endpoint C.StringRef, jwt C.StringRef, f3_rpc_endpoint C. } //export CGoF3Node_import_snap -func CGoF3Node_import_snap(f3_root C.StringRef, snapshot C.StringRef, slot *C.void, cb *C.void) { +func CGoF3Node_import_snap(f3_rpc_endpoint C.StringRef, f3_root C.StringRef, snapshot C.StringRef, slot *C.void, cb *C.void) { + _new_f3_rpc_endpoint := newString(f3_rpc_endpoint) _new_f3_root := newString(f3_root) _new_snapshot := newString(snapshot) - resp := GoF3NodeImpl.import_snap(&_new_f3_root, &_new_snapshot) + resp := GoF3NodeImpl.import_snap(&_new_f3_rpc_endpoint, &_new_f3_root, &_new_snapshot) resp_ref, buffer := cvt_ref(cntString, refString)(&resp) asmcall.CallFuncG0P2(unsafe.Pointer(cb), unsafe.Pointer(&resp_ref), unsafe.Pointer(slot)) runtime.KeepAlive(resp_ref) diff --git a/f3-sidecar/ffi_impl.go b/f3-sidecar/ffi_impl.go index 04c05d4a4976..2c5392eb3692 100644 --- a/f3-sidecar/ffi_impl.go +++ b/f3-sidecar/ffi_impl.go @@ -48,8 +48,8 @@ func (f3 *f3Impl) run(rpc_endpoint *string, jwt *string, f3_rpc_endpoint *string return err == nil } -func (f3 *f3Impl) import_snap(f3_root *string, snapshot *string) string { - if err := importSnap(f3.ctx, *f3_root, *snapshot); err != nil { +func (f3 *f3Impl) import_snap(f3_rpc_endpoint *string, f3_root *string, snapshot *string) string { + if err := importSnap(f3.ctx, *f3_rpc_endpoint, *f3_root, *snapshot); err != nil { return err.Error() } return "" diff --git a/f3-sidecar/go.mod b/f3-sidecar/go.mod index 1150e3cec7ea..3b5a415a6ba6 100644 --- a/f3-sidecar/go.mod +++ b/f3-sidecar/go.mod @@ -2,6 +2,8 @@ module f3-sidecar/v2 go 1.24.5 +replace github.com/filecoin-project/go-f3 => github.com/hanabi1224/go-f3 v0.0.0-20250801054138-db8d8ac07ab1 + require ( github.com/filecoin-project/go-f3 v0.8.9 github.com/filecoin-project/go-jsonrpc v0.8.0 diff --git a/f3-sidecar/go.sum b/f3-sidecar/go.sum index e55c92e08444..543daf22a749 100644 --- a/f3-sidecar/go.sum +++ b/f3-sidecar/go.sum @@ -45,8 +45,6 @@ github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= -github.com/filecoin-project/go-f3 v0.8.9 h1:0SHqwWmcVAL02Or7uE4P7qG1feopyVBSlgrUxkHkQBM= -github.com/filecoin-project/go-f3 v0.8.9/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= github.com/filecoin-project/go-jsonrpc v0.8.0 h1:2yqlN3Vd8Gx5UtA3fib7tQu2aW1cSOJt253LEBWExo4= github.com/filecoin-project/go-jsonrpc v0.8.0/go.mod h1:p8WGOwQGYbFugSdK7qKIGhhb1VVcQ2rtBLdEiik1QWI= github.com/filecoin-project/go-state-types v0.16.0 h1:ajIREDzTGfq71ofIQ29iZR1WXxmkvd2nQNc6ApcP1wI= @@ -126,6 +124,8 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hanabi1224/go-f3 v0.0.0-20250801054138-db8d8ac07ab1 h1:+AuqFCtCCv9zPfBQzJD3Tfas4kqRoU6No6E2wVWgLYo= +github.com/hanabi1224/go-f3 v0.0.0-20250801054138-db8d8ac07ab1/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= diff --git a/f3-sidecar/import.go b/f3-sidecar/import.go index dea2d504c819..168c049e6a3a 100644 --- a/f3-sidecar/import.go +++ b/f3-sidecar/import.go @@ -6,19 +6,115 @@ import ( "os" "github.com/filecoin-project/go-f3/certstore" + "github.com/filecoin-project/go-f3/manifest" + "github.com/filecoin-project/go-jsonrpc" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + "github.com/ipfs/go-datastore/query" + leveldb "github.com/ipfs/go-ds-leveldb" ) -func importSnap(ctx context.Context, f3Root string, snapshot string) error { +func importSnap(ctx context.Context, rpcEndpoint string, f3Root string, snapshot string) error { logger.Infof("importing F3 snapshot at %s", snapshot) + + f3api := F3Api{} + closer, err := jsonrpc.NewClient(ctx, rpcEndpoint, "F3", &f3api, nil) + defer closer() + rawNetworkName := waitRawNetworkName(ctx, &f3api) + networkName := getNetworkName(rawNetworkName) + m := Network2PredefinedManifestMappings[networkName] + if m == nil { + m2 := manifest.LocalDevnetManifest() + m = &m2 + m.NetworkName = networkName + } + ds, err := getDatastore(f3Root) if err != nil { return err } defer ds.Close() + dsBatcher := LevelDBBatchWriter{ds: ds} + defer dsBatcher.Close() + dsWrapper := namespace.Wrap(&dsBatcher, m.DatastorePrefix()) + defer dsWrapper.Close() + f, err := os.Open(snapshot) if err != nil { return err } defer f.Close() - return certstore.ImportSnapshotToDatastore(ctx, bufio.NewReader(f), ds) + return certstore.ImportSnapshotToDatastore(ctx, bufio.NewReader(f), dsWrapper) +} + +type LevelDBBatchWriter struct { + ds *leveldb.Datastore + batch datastore.Batch + batchSize int +} + +func (w *LevelDBBatchWriter) Get(ctx context.Context, key datastore.Key) (value []byte, err error) { + return w.ds.Get(ctx, key) +} + +func (w *LevelDBBatchWriter) Has(ctx context.Context, key datastore.Key) (exists bool, err error) { + return w.ds.Has(ctx, key) +} + +func (w *LevelDBBatchWriter) GetSize(ctx context.Context, key datastore.Key) (size int, err error) { + return w.ds.GetSize(ctx, key) +} + +func (w *LevelDBBatchWriter) Query(ctx context.Context, q query.Query) (query.Results, error) { + return w.ds.Query(ctx, q) +} + +func (w *LevelDBBatchWriter) Close() error { + return w.ds.Close() +} + +func (w *LevelDBBatchWriter) Sync(ctx context.Context, prefix datastore.Key) error { + if err := w.Flush(ctx); err != nil { + return err + } + return w.ds.Sync(ctx, prefix) +} + +func (w *LevelDBBatchWriter) Put(ctx context.Context, key datastore.Key, value []byte) error { + if w.batch == nil { + batch, err := w.ds.Batch(ctx) + if err != nil { + return err + } + w.batch = batch + } + + if w.batchSize >= 1024 { + if err := w.Flush(ctx); err != nil { + return err + } + } + + w.batchSize += 1 + return w.batch.Put(ctx, key, value) + +} + +func (w *LevelDBBatchWriter) Delete(ctx context.Context, key datastore.Key) error { + return w.ds.Delete(ctx, key) +} + +func (w *LevelDBBatchWriter) Flush(ctx context.Context) error { + if w.batch != nil { + if err := w.batch.Commit(ctx); err != nil { + return err + } + batch, err := w.ds.Batch(ctx) + if err != nil { + return err + } + w.batch = batch + w.batchSize = 0 + } + return nil } diff --git a/f3-sidecar/main.go b/f3-sidecar/main.go index 5d8b19705503..97db86dfbf0d 100644 --- a/f3-sidecar/main.go +++ b/f3-sidecar/main.go @@ -49,7 +49,7 @@ func main() { ctx := context.Background() if len(snapshot) > 0 { - if err := importSnap(ctx, root, snapshot); err != nil { + if err := importSnap(ctx, rpcEndpoint, root, snapshot); err != nil { panic(err) } } diff --git a/f3-sidecar/run.go b/f3-sidecar/run.go index de0f7b7f1735..b6bb52fb9308 100644 --- a/f3-sidecar/run.go +++ b/f3-sidecar/run.go @@ -30,17 +30,7 @@ func run(ctx context.Context, rpcEndpoint string, jwt string, f3RpcEndpoint stri return err } - var rawNetwork string - for { - rawNetwork, err = ec.f3api.GetRawNetworkName(ctx) - if err == nil { - logger.Infoln("Forest RPC server is online") - break - } else { - logger.Warnln("waiting for Forest RPC server") - time.Sleep(5 * time.Second) - } - } + rawNetwork := waitRawNetworkName(ctx, &ec.f3api) listenAddrs, err := api.NetAddrsListen(ctx) if err != nil { return err @@ -64,12 +54,7 @@ func run(ctx context.Context, rpcEndpoint string, jwt string, f3RpcEndpoint stri } defer ds.Close() verif := blssig.VerifierWithKeyOnG1() - networkName := gpbft.NetworkName(rawNetwork) - // Use "filecoin" as the network name on mainnet, otherwise use the network name. Yes, - // mainnet is called testnetnet in state. - if networkName == "testnetnet" { - networkName = "filecoin" - } + networkName := getNetworkName(rawNetwork) m := Network2PredefinedManifestMappings[networkName] if m == nil { m2 := manifest.LocalDevnetManifest() diff --git a/f3-sidecar/utils.go b/f3-sidecar/utils.go index f4081498b08f..5746f70eb8f8 100644 --- a/f3-sidecar/utils.go +++ b/f3-sidecar/utils.go @@ -1,8 +1,11 @@ package main import ( + "context" "path/filepath" + "time" + "github.com/filecoin-project/go-f3/gpbft" "github.com/ipfs/go-cid" leveldb "github.com/ipfs/go-ds-leveldb" ) @@ -16,3 +19,26 @@ func isCidDefined(c cid.Cid) bool { func getDatastore(f3Root string) (*leveldb.Datastore, error) { return leveldb.NewDatastore(filepath.Join(f3Root, "db"), nil) } + +func waitRawNetworkName(ctx context.Context, f3api *F3Api) string { + for { + rawNetwork, err := f3api.GetRawNetworkName(ctx) + if err == nil { + logger.Infoln("Forest RPC server is online") + return rawNetwork + } else { + logger.Warnln("waiting for Forest RPC server") + time.Sleep(5 * time.Second) + } + } +} + +func getNetworkName(rawNetworkName string) gpbft.NetworkName { + networkName := gpbft.NetworkName(rawNetworkName) + // Use "filecoin" as the network name on mainnet, otherwise use the network name. Yes, + // mainnet is called testnetnet in state. + if networkName == "testnetnet" { + networkName = "filecoin" + } + return networkName +} diff --git a/src/cli_shared/cli/client.rs b/src/cli_shared/cli/client.rs index 6c3c2f8beb79..b856609d47ff 100644 --- a/src/cli_shared/cli/client.rs +++ b/src/cli_shared/cli/client.rs @@ -102,4 +102,8 @@ impl Client { pub fn default_rpc_token_path(&self) -> PathBuf { self.data_dir.join("token") } + + pub fn rpc_v1_endpoint(&self) -> String { + format!("http://{}/rpc/v1", self.rpc_address) + } } diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index 9ffa5eeae1e8..5422f0cebc45 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -132,6 +132,7 @@ pub async fn import_chain_as_forest_car( from_path: &Path, forest_car_db_dir: &Path, import_mode: ImportMode, + rpc_endpoint: String, f3_root: String, is_sidecar_ffi_enabled: bool, snapshot_progress_tracker: &SnapshotProgressTracker, @@ -248,7 +249,13 @@ pub async fn import_chain_as_forest_car( let mut f = File::create(&temp_f3_snap_path)?; std::io::copy(&mut f3_data, &mut f)?; } - crate::f3::import_f3_snapshot(f3_root, temp_f3_snap_path.display().to_string())?; + if let Err(e) = crate::f3::import_f3_snapshot( + rpc_endpoint, + f3_root, + temp_f3_snap_path.display().to_string(), + ) { + tracing::error!("Failed to import F3 snapshot: {e}"); + } } let ts = forest_car.heaviest_tipset()?; @@ -517,6 +524,7 @@ mod test { temp_db_dir.path(), import_mode, "".into(), + "".into(), true, &SnapshotProgressTracker::default(), ) diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index c8f2890692e6..4ad4e06d6655 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -140,6 +140,7 @@ async fn maybe_import_snapshot( path, &ctx.db_meta_data.get_forest_car_db_dir(), config.client.import_mode, + config.client.rpc_v1_endpoint(), crate::f3::get_f3_root(config), crate::f3::is_sidecar_ffi_enabled(ctx.chain_config()), &snapshot_tracker, @@ -408,7 +409,7 @@ fn maybe_start_f3_service(opts: &CliOpts, config: &Config, ctx: &AppContext) { } if !opts.halt_after_import && !opts.stateless { - let rpc_address = config.client.rpc_address; + let rpc_endpoint = config.client.rpc_v1_endpoint(); let state_manager = &ctx.state_manager; let p2p_peer_id = ctx.p2p_peer_id; let admin_jwt = ctx.admin_jwt.clone(); @@ -429,7 +430,7 @@ fn maybe_start_f3_service(opts: &CliOpts, config: &Config, ctx: &AppContext) { move || { crate::f3::run_f3_sidecar_if_enabled( &chain_config, - format!("http://{rpc_address}/rpc/v1"), + rpc_endpoint, admin_jwt, crate::rpc::f3::get_f3_rpc_endpoint().to_string(), initial_power_table diff --git a/src/f3/go_ffi.rs b/src/f3/go_ffi.rs index c5663fdd719c..008053ce44bf 100644 --- a/src/f3/go_ffi.rs +++ b/src/f3/go_ffi.rs @@ -19,5 +19,5 @@ pub trait GoF3Node { f3_root: String, ) -> bool; - fn import_snap(f3_root: String, snapshot: String) -> String; + fn import_snap(f3_rpc_endpoint: String, f3_root: String, snapshot: String) -> String; } diff --git a/src/f3/mod.rs b/src/f3/mod.rs index 05b9d2132aa9..e7ca1cee7ea6 100644 --- a/src/f3/mod.rs +++ b/src/f3/mod.rs @@ -111,12 +111,16 @@ pub fn run_f3_sidecar_if_enabled( } } -pub fn import_f3_snapshot(_f3_root: String, _snapshot: String) -> anyhow::Result<()> { +pub fn import_f3_snapshot( + _rpc_endpoint: String, + _f3_root: String, + _snapshot: String, +) -> anyhow::Result<()> { #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] { let sw = std::time::Instant::now(); tracing::info!("Importing F3 snapshot ..."); - let err = GoF3NodeImpl::import_snap(_f3_root, _snapshot); + let err = GoF3NodeImpl::import_snap(_rpc_endpoint, _f3_root, _snapshot); if !err.is_empty() { anyhow::bail!("{err}"); } From 93c4d7cf38f835ba941431ad4daef74b724deade Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 1 Aug 2025 18:36:07 +0800 Subject: [PATCH 17/35] bump go-f3 --- f3-sidecar/go.mod | 4 +-- f3-sidecar/go.sum | 4 +-- f3-sidecar/import.go | 79 +------------------------------------------- 3 files changed, 5 insertions(+), 82 deletions(-) diff --git a/f3-sidecar/go.mod b/f3-sidecar/go.mod index 3b5a415a6ba6..4bacaace92fd 100644 --- a/f3-sidecar/go.mod +++ b/f3-sidecar/go.mod @@ -2,7 +2,7 @@ module f3-sidecar/v2 go 1.24.5 -replace github.com/filecoin-project/go-f3 => github.com/hanabi1224/go-f3 v0.0.0-20250801054138-db8d8ac07ab1 +replace github.com/filecoin-project/go-f3 => github.com/hanabi1224/go-f3 v0.0.0-20250801102326-76a043056557 require ( github.com/filecoin-project/go-f3 v0.8.9 @@ -10,6 +10,7 @@ require ( github.com/filecoin-project/go-state-types v0.16.0 github.com/ihciah/rust2go v0.0.0-20250726175549-557d7a3a4e27 github.com/ipfs/go-cid v0.5.0 + github.com/ipfs/go-datastore v0.8.2 github.com/ipfs/go-ds-leveldb v0.5.2 github.com/ipfs/go-log/v2 v2.8.0 github.com/libp2p/go-libp2p v0.42.1 @@ -46,7 +47,6 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/ipfs/boxo v0.33.0 // indirect - github.com/ipfs/go-datastore v0.8.2 // indirect github.com/ipld/go-ipld-prime v0.21.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect diff --git a/f3-sidecar/go.sum b/f3-sidecar/go.sum index 543daf22a749..f4d6f2ce8fca 100644 --- a/f3-sidecar/go.sum +++ b/f3-sidecar/go.sum @@ -124,8 +124,8 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/hanabi1224/go-f3 v0.0.0-20250801054138-db8d8ac07ab1 h1:+AuqFCtCCv9zPfBQzJD3Tfas4kqRoU6No6E2wVWgLYo= -github.com/hanabi1224/go-f3 v0.0.0-20250801054138-db8d8ac07ab1/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= +github.com/hanabi1224/go-f3 v0.0.0-20250801102326-76a043056557 h1:yso7mVqpG7FX8MWJKZVa77cNXjJuSysyOzS4eVp5nKc= +github.com/hanabi1224/go-f3 v0.0.0-20250801102326-76a043056557/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= diff --git a/f3-sidecar/import.go b/f3-sidecar/import.go index 168c049e6a3a..f046e5f88e3d 100644 --- a/f3-sidecar/import.go +++ b/f3-sidecar/import.go @@ -8,10 +8,7 @@ import ( "github.com/filecoin-project/go-f3/certstore" "github.com/filecoin-project/go-f3/manifest" "github.com/filecoin-project/go-jsonrpc" - "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" - "github.com/ipfs/go-datastore/query" - leveldb "github.com/ipfs/go-ds-leveldb" ) func importSnap(ctx context.Context, rpcEndpoint string, f3Root string, snapshot string) error { @@ -34,9 +31,7 @@ func importSnap(ctx context.Context, rpcEndpoint string, f3Root string, snapshot return err } defer ds.Close() - dsBatcher := LevelDBBatchWriter{ds: ds} - defer dsBatcher.Close() - dsWrapper := namespace.Wrap(&dsBatcher, m.DatastorePrefix()) + dsWrapper := namespace.Wrap(ds, m.DatastorePrefix()) defer dsWrapper.Close() f, err := os.Open(snapshot) @@ -46,75 +41,3 @@ func importSnap(ctx context.Context, rpcEndpoint string, f3Root string, snapshot defer f.Close() return certstore.ImportSnapshotToDatastore(ctx, bufio.NewReader(f), dsWrapper) } - -type LevelDBBatchWriter struct { - ds *leveldb.Datastore - batch datastore.Batch - batchSize int -} - -func (w *LevelDBBatchWriter) Get(ctx context.Context, key datastore.Key) (value []byte, err error) { - return w.ds.Get(ctx, key) -} - -func (w *LevelDBBatchWriter) Has(ctx context.Context, key datastore.Key) (exists bool, err error) { - return w.ds.Has(ctx, key) -} - -func (w *LevelDBBatchWriter) GetSize(ctx context.Context, key datastore.Key) (size int, err error) { - return w.ds.GetSize(ctx, key) -} - -func (w *LevelDBBatchWriter) Query(ctx context.Context, q query.Query) (query.Results, error) { - return w.ds.Query(ctx, q) -} - -func (w *LevelDBBatchWriter) Close() error { - return w.ds.Close() -} - -func (w *LevelDBBatchWriter) Sync(ctx context.Context, prefix datastore.Key) error { - if err := w.Flush(ctx); err != nil { - return err - } - return w.ds.Sync(ctx, prefix) -} - -func (w *LevelDBBatchWriter) Put(ctx context.Context, key datastore.Key, value []byte) error { - if w.batch == nil { - batch, err := w.ds.Batch(ctx) - if err != nil { - return err - } - w.batch = batch - } - - if w.batchSize >= 1024 { - if err := w.Flush(ctx); err != nil { - return err - } - } - - w.batchSize += 1 - return w.batch.Put(ctx, key, value) - -} - -func (w *LevelDBBatchWriter) Delete(ctx context.Context, key datastore.Key) error { - return w.ds.Delete(ctx, key) -} - -func (w *LevelDBBatchWriter) Flush(ctx context.Context) error { - if w.batch != nil { - if err := w.batch.Commit(ctx); err != nil { - return err - } - batch, err := w.ds.Batch(ctx) - if err != nil { - return err - } - w.batch = batch - w.batchSize = 0 - } - return nil -} From 9ffd7c9b28287f2d86d72143770b491d18fddb7b Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 1 Aug 2025 20:53:51 +0800 Subject: [PATCH 18/35] bump go-f3 --- f3-sidecar/go.mod | 4 +--- f3-sidecar/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/f3-sidecar/go.mod b/f3-sidecar/go.mod index 4bacaace92fd..24180c972c5b 100644 --- a/f3-sidecar/go.mod +++ b/f3-sidecar/go.mod @@ -2,10 +2,8 @@ module f3-sidecar/v2 go 1.24.5 -replace github.com/filecoin-project/go-f3 => github.com/hanabi1224/go-f3 v0.0.0-20250801102326-76a043056557 - require ( - github.com/filecoin-project/go-f3 v0.8.9 + github.com/filecoin-project/go-f3 v0.8.10-0.20250801124500-9288fba86c47 github.com/filecoin-project/go-jsonrpc v0.8.0 github.com/filecoin-project/go-state-types v0.16.0 github.com/ihciah/rust2go v0.0.0-20250726175549-557d7a3a4e27 diff --git a/f3-sidecar/go.sum b/f3-sidecar/go.sum index f4d6f2ce8fca..6ce3301c4579 100644 --- a/f3-sidecar/go.sum +++ b/f3-sidecar/go.sum @@ -45,6 +45,8 @@ github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= +github.com/filecoin-project/go-f3 v0.8.10-0.20250801124500-9288fba86c47 h1:qf5j5kHyTfwivaXjXlySjS5Nax6wufrUiomE4elcW3s= +github.com/filecoin-project/go-f3 v0.8.10-0.20250801124500-9288fba86c47/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= github.com/filecoin-project/go-jsonrpc v0.8.0 h1:2yqlN3Vd8Gx5UtA3fib7tQu2aW1cSOJt253LEBWExo4= github.com/filecoin-project/go-jsonrpc v0.8.0/go.mod h1:p8WGOwQGYbFugSdK7qKIGhhb1VVcQ2rtBLdEiik1QWI= github.com/filecoin-project/go-state-types v0.16.0 h1:ajIREDzTGfq71ofIQ29iZR1WXxmkvd2nQNc6ApcP1wI= @@ -124,8 +126,6 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/hanabi1224/go-f3 v0.0.0-20250801102326-76a043056557 h1:yso7mVqpG7FX8MWJKZVa77cNXjJuSysyOzS4eVp5nKc= -github.com/hanabi1224/go-f3 v0.0.0-20250801102326-76a043056557/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= From d68fa837884cb445093f75f51ffc4625d19003b4 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 4 Aug 2025 19:37:54 +0800 Subject: [PATCH 19/35] bump go-f3 --- f3-sidecar/go.mod | 2 ++ f3-sidecar/go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/f3-sidecar/go.mod b/f3-sidecar/go.mod index 24180c972c5b..e2d2ae71c88d 100644 --- a/f3-sidecar/go.mod +++ b/f3-sidecar/go.mod @@ -2,6 +2,8 @@ module f3-sidecar/v2 go 1.24.5 +replace github.com/filecoin-project/go-f3 => github.com/hanabi1224/go-f3 v0.0.0-20250804095240-96be7655affc + require ( github.com/filecoin-project/go-f3 v0.8.10-0.20250801124500-9288fba86c47 github.com/filecoin-project/go-jsonrpc v0.8.0 diff --git a/f3-sidecar/go.sum b/f3-sidecar/go.sum index 6ce3301c4579..1a5fe00210bc 100644 --- a/f3-sidecar/go.sum +++ b/f3-sidecar/go.sum @@ -45,8 +45,6 @@ github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= -github.com/filecoin-project/go-f3 v0.8.10-0.20250801124500-9288fba86c47 h1:qf5j5kHyTfwivaXjXlySjS5Nax6wufrUiomE4elcW3s= -github.com/filecoin-project/go-f3 v0.8.10-0.20250801124500-9288fba86c47/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= github.com/filecoin-project/go-jsonrpc v0.8.0 h1:2yqlN3Vd8Gx5UtA3fib7tQu2aW1cSOJt253LEBWExo4= github.com/filecoin-project/go-jsonrpc v0.8.0/go.mod h1:p8WGOwQGYbFugSdK7qKIGhhb1VVcQ2rtBLdEiik1QWI= github.com/filecoin-project/go-state-types v0.16.0 h1:ajIREDzTGfq71ofIQ29iZR1WXxmkvd2nQNc6ApcP1wI= @@ -126,6 +124,8 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hanabi1224/go-f3 v0.0.0-20250804095240-96be7655affc h1:bkuCwP1YFgn7J12/9G3UyQt6oFQTtv5a9xPSHPtSBTA= +github.com/hanabi1224/go-f3 v0.0.0-20250804095240-96be7655affc/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= From 374c38d6c3d504498fefc611c439804b938450d7 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 6 Aug 2025 16:40:52 +0800 Subject: [PATCH 20/35] no go mod replace --- f3-sidecar/go.mod | 2 -- f3-sidecar/go.sum | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/f3-sidecar/go.mod b/f3-sidecar/go.mod index e2d2ae71c88d..24180c972c5b 100644 --- a/f3-sidecar/go.mod +++ b/f3-sidecar/go.mod @@ -2,8 +2,6 @@ module f3-sidecar/v2 go 1.24.5 -replace github.com/filecoin-project/go-f3 => github.com/hanabi1224/go-f3 v0.0.0-20250804095240-96be7655affc - require ( github.com/filecoin-project/go-f3 v0.8.10-0.20250801124500-9288fba86c47 github.com/filecoin-project/go-jsonrpc v0.8.0 diff --git a/f3-sidecar/go.sum b/f3-sidecar/go.sum index 1a5fe00210bc..6ce3301c4579 100644 --- a/f3-sidecar/go.sum +++ b/f3-sidecar/go.sum @@ -45,6 +45,8 @@ github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= +github.com/filecoin-project/go-f3 v0.8.10-0.20250801124500-9288fba86c47 h1:qf5j5kHyTfwivaXjXlySjS5Nax6wufrUiomE4elcW3s= +github.com/filecoin-project/go-f3 v0.8.10-0.20250801124500-9288fba86c47/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= github.com/filecoin-project/go-jsonrpc v0.8.0 h1:2yqlN3Vd8Gx5UtA3fib7tQu2aW1cSOJt253LEBWExo4= github.com/filecoin-project/go-jsonrpc v0.8.0/go.mod h1:p8WGOwQGYbFugSdK7qKIGhhb1VVcQ2rtBLdEiik1QWI= github.com/filecoin-project/go-state-types v0.16.0 h1:ajIREDzTGfq71ofIQ29iZR1WXxmkvd2nQNc6ApcP1wI= @@ -124,8 +126,6 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/hanabi1224/go-f3 v0.0.0-20250804095240-96be7655affc h1:bkuCwP1YFgn7J12/9G3UyQt6oFQTtv5a9xPSHPtSBTA= -github.com/hanabi1224/go-f3 v0.0.0-20250804095240-96be7655affc/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= From 0a1ca027ab2da150d4a0c9aa3a52cd85e60c1c53 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 6 Aug 2025 17:02:56 +0800 Subject: [PATCH 21/35] fix comments and small tweaks --- src/daemon/db_util.rs | 11 +++++------ src/daemon/mod.rs | 2 +- src/db/car/forest.rs | 8 +++++--- src/db/car/plain.rs | 2 +- src/f3/mod.rs | 27 ++++++++++++++++----------- 5 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index 5422f0cebc45..b8a1a46cfe12 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -7,7 +7,7 @@ use crate::db::car::forest::{ }; use crate::db::car::{ForestCar, ManyCar}; use crate::interpreter::VMTrace; -use crate::networks::Height; +use crate::networks::{ChainConfig, Height}; use crate::rpc::sync::SnapshotProgressTracker; use crate::shim::clock::ChainEpoch; use crate::state_manager::{NO_CALLBACK, StateManager}; @@ -134,7 +134,7 @@ pub async fn import_chain_as_forest_car( import_mode: ImportMode, rpc_endpoint: String, f3_root: String, - is_sidecar_ffi_enabled: bool, + chain_config: &ChainConfig, snapshot_progress_tracker: &SnapshotProgressTracker, ) -> anyhow::Result<(PathBuf, Tipset)> { info!("Importing chain from snapshot at: {}", from_path.display()); @@ -236,9 +236,7 @@ pub async fn import_chain_as_forest_car( let forest_car = ForestCar::try_from(forest_car_db_path.as_path())?; - if !is_sidecar_ffi_enabled { - tracing::warn!("F3 sidecar is disabled, skip importing F3 snapshot"); - } else if let Some(f3_cid) = forest_car.metadata().as_ref().and_then(|m| m.f3_data) + if let Some(f3_cid) = forest_car.metadata().as_ref().and_then(|m| m.f3_data) && let Some(mut f3_data) = forest_car.get_reader(f3_cid)? { let temp_f3_snap_path = tempfile::Builder::new() @@ -250,6 +248,7 @@ pub async fn import_chain_as_forest_car( std::io::copy(&mut f3_data, &mut f)?; } if let Err(e) = crate::f3::import_f3_snapshot( + chain_config, rpc_endpoint, f3_root, temp_f3_snap_path.display().to_string(), @@ -525,7 +524,7 @@ mod test { import_mode, "".into(), "".into(), - true, + &ChainConfig::devnet(), &SnapshotProgressTracker::default(), ) .await?; diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index 4ad4e06d6655..e9803154cf17 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -142,7 +142,7 @@ async fn maybe_import_snapshot( config.client.import_mode, config.client.rpc_v1_endpoint(), crate::f3::get_f3_root(config), - crate::f3::is_sidecar_ffi_enabled(ctx.chain_config()), + ctx.chain_config(), &snapshot_tracker, ) .await?; diff --git a/src/db/car/forest.rs b/src/db/car/forest.rs index fe208714cbf7..7b3d863ae20f 100644 --- a/src/db/car/forest.rs +++ b/src/db/car/forest.rs @@ -215,16 +215,18 @@ impl ForestCar { } /// Gets a reader of the block data by its `Cid` - pub fn get_reader(&self, k: Cid) -> anyhow::Result>> { + pub fn get_reader(&self, k: Cid) -> anyhow::Result> { for position in self.indexed.get(k)? { - // Decode entire frame into memory, "position" arg is the frame start offset. - let entire_file = self.indexed.reader().get_ref(); // escape the positioned_io::Slice + // escape the positioned_io::Slice + let entire_file = self.indexed.reader().get_ref(); + // `position` is the frame start offset. let cursor = Cursor::new_pos(entire_file, position); let mut decoder = zstd::Decoder::new(cursor)?.single_frame(); while let Ok(frame_len) = decoder.read_varint::() { let cid = Cid::read_bytes(&mut decoder)?; let data_len = frame_len.saturating_sub(cid.encoded_len()) as u64; if cid == k { + // return the reader instead of decoding the entire data block into memory return Ok(Some(decoder.take(data_len))); } // Discard data bytes diff --git a/src/db/car/plain.rs b/src/db/car/plain.rs index 455ed5182d4f..12a31dfa5de1 100644 --- a/src/db/car/plain.rs +++ b/src/db/car/plain.rs @@ -227,7 +227,7 @@ impl PlainCar { #[allow(dead_code)] /// Gets a reader of the block data by its `Cid` - pub fn get_reader(&self, k: Cid) -> Option> { + pub fn get_reader(&self, k: Cid) -> Option { self.index .read() .get(&k) diff --git a/src/f3/mod.rs b/src/f3/mod.rs index e7ca1cee7ea6..2d069032a11b 100644 --- a/src/f3/mod.rs +++ b/src/f3/mod.rs @@ -112,22 +112,27 @@ pub fn run_f3_sidecar_if_enabled( } pub fn import_f3_snapshot( + chain_config: &ChainConfig, _rpc_endpoint: String, _f3_root: String, _snapshot: String, ) -> anyhow::Result<()> { - #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] - { - let sw = std::time::Instant::now(); - tracing::info!("Importing F3 snapshot ..."); - let err = GoF3NodeImpl::import_snap(_rpc_endpoint, _f3_root, _snapshot); - if !err.is_empty() { - anyhow::bail!("{err}"); + if is_sidecar_ffi_enabled(chain_config) { + #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] + { + let sw = std::time::Instant::now(); + tracing::info!("Importing F3 snapshot ..."); + let err = GoF3NodeImpl::import_snap(_rpc_endpoint, _f3_root, _snapshot); + if !err.is_empty() { + anyhow::bail!("{err}"); + } + tracing::info!( + "Imported F3 snapshot, took {}", + humantime::format_duration(sw.elapsed()) + ); } - tracing::info!( - "Imported F3 snapshot, took {}", - humantime::format_duration(sw.elapsed()) - ); + } else { + tracing::warn!("F3 sidecar is disabled, skip importing the F3 snapshot"); } Ok(()) } From 725f233db777978e55b2c18404bd949120d3225e Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 6 Aug 2025 20:50:24 +0800 Subject: [PATCH 22/35] add tests --- .github/workflows/forest.yml | 4 +- scripts/tests/calibnet_eth_mapping_check.sh | 3 +- scripts/tests/calibnet_export_f3_check.sh | 36 ++++++ scripts/tests/harness.sh | 36 ++++++ src/beacon/beacon_entries.rs | 2 +- src/chain/mod.rs | 7 +- src/chain/snapshot_format.rs | 8 +- src/cli/subcommands/f3_cmd.rs | 1 + src/daemon/db_util.rs | 11 +- src/db/car/any.rs | 1 - src/db/car/forest.rs | 4 +- src/f3/mod.rs | 2 + src/f3/snapshot.rs | 46 ++++++++ src/f3/snapshot/tests.rs | 21 ++++ src/rpc/methods/f3.rs | 26 +++-- src/rpc/methods/f3/types.rs | 51 ++++++++- src/shim/actors/builtin/verifreg/mod.rs | 3 +- src/tool/subcommands/archive_cmd.rs | 119 +++++++++++++++++++- 18 files changed, 338 insertions(+), 43 deletions(-) create mode 100755 scripts/tests/calibnet_export_f3_check.sh create mode 100644 src/f3/snapshot.rs create mode 100644 src/f3/snapshot/tests.rs diff --git a/.github/workflows/forest.yml b/.github/workflows/forest.yml index 5c5ed1013e88..95a70077973d 100644 --- a/.github/workflows/forest.yml +++ b/.github/workflows/forest.yml @@ -287,7 +287,7 @@ jobs: calibnet-export-check-v2: needs: - build-ubuntu - name: Snapshot export checks v2 + name: Snapshot export checks v2 with F3 data runs-on: ubuntu-24.04 steps: - run: lscpu @@ -304,7 +304,7 @@ jobs: run: | chmod +x ~/.cargo/bin/forest* - name: Snapshot export check v2 - run: ./scripts/tests/calibnet_export_check.sh v2 + run: ./scripts/tests/calibnet_export_f3_check.sh timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }} calibnet-no-discovery-checks: needs: diff --git a/scripts/tests/calibnet_eth_mapping_check.sh b/scripts/tests/calibnet_eth_mapping_check.sh index 7a3e7168bd7b..8803edd8c35f 100755 --- a/scripts/tests/calibnet_eth_mapping_check.sh +++ b/scripts/tests/calibnet_eth_mapping_check.sh @@ -46,8 +46,7 @@ done echo "Done" -echo "Waiting to be ready for serving" -$FOREST_CLI_PATH healthcheck ready --wait +forest_wait_for_healthcheck_ready ERROR=0 echo "Testing Ethereum mappings" diff --git a/scripts/tests/calibnet_export_f3_check.sh b/scripts/tests/calibnet_export_f3_check.sh new file mode 100755 index 000000000000..5e174eece28a --- /dev/null +++ b/scripts/tests/calibnet_export_f3_check.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# This script is checking the correctness of +# the snapshot export feature. +# It requires both the `forest` and `forest-cli` binaries to be in the PATH. + +set -eu + +source "$(dirname "$0")/harness.sh" + +forest_init_with_f3 + +echo "Cleaning up the initial snapshot" +rm --force --verbose ./*.{car,car.zst,sha256sum} + +echo "Exporting zstd compressed snapshot in v2 format" +$FOREST_CLI_PATH snapshot export --format v2 + +$FOREST_CLI_PATH shutdown --force + +for f in *.car.zst; do + echo "Inspecting archive info $f" + $FOREST_TOOL_PATH archive info "$f" + echo "Inspecting archive metadata $f" + $FOREST_TOOL_PATH archive metadata "$f" +done + +echo "Testing snapshot validity" +zstd --test ./*.car.zst + +echo "Verifying snapshot checksum" +sha256sum --check ./*.sha256sum + +for f in *.car.zst; do + echo "Validating CAR file $f" + $FOREST_TOOL_PATH snapshot validate "$f" +done diff --git a/scripts/tests/harness.sh b/scripts/tests/harness.sh index dc153f27a54f..b6d6ead9e999 100644 --- a/scripts/tests/harness.sh +++ b/scripts/tests/harness.sh @@ -27,6 +27,18 @@ function forest_download_and_import_snapshot { $FOREST_PATH --chain calibnet --encrypt-keystore false --halt-after-import --height=-200 --auto-download-snapshot } +function forest_download_and_import_snapshot_with_f3 { + echo "Downloading v1 snapshot" + aria2c -x5 https://forest-archive.chainsafe.dev/latest/calibnet/ -o v1.forest.car.zst + echo "Downloading F3 snapshot" + aria2c -x5 https://forest-snapshots.fra1.cdn.digitaloceanspaces.com/f3/f3_snap_calibnet_552628.bin -o f3.bin + echo "Generating v2 snapshot" + $FOREST_TOOL_PATH archive merge-f3 --filecoin v1.forest.car.zst --f3 f3.bin --output v2.forest.car.zst + $FOREST_TOOL_PATH archive info v2.forest.car.zst + $FOREST_TOOL_PATH archive metadata v2.forest.car.zst + $FOREST_PATH --chain calibnet --encrypt-keystore false --halt-after-import --height=-200 --import-snapshot v2.forest.car.zst +} + function get_epoch_from_car_db { DB_PATH=$($FOREST_TOOL_PATH db stats --chain calibnet | grep "Database path:" | cut -d':' -f2- | xargs) SNAPSHOT=$(ls "$DB_PATH/car_db"/*.car.zst) @@ -97,6 +109,11 @@ function forest_wait_for_sync { timeout 30m $FOREST_CLI_PATH sync wait } +function forest_wait_for_healthcheck_ready { + echo "Waiting for healthcheck ready" + timeout 30m $FOREST_CLI_PATH healthcheck ready --wait +} + function forest_init { forest_download_and_import_snapshot @@ -117,6 +134,25 @@ function forest_init { forest_check_db_stats } +function forest_init_with_f3 { + forest_download_and_import_snapshot_with_f3 + + forest_check_db_stats + forest_run_node_detached + + forest_wait_api + + forest_wait_for_sync + forest_check_db_stats + + forest_wait_for_healthcheck_ready + + # print the latest F3 certificate + $FOREST_CLI_PATH f3 c get + echo "ensure F3 certificate at instance 550000 has been imported" + $FOREST_CLI_PATH f3 c get 550000 +} + function forest_init_stateless { forest_run_node_stateless_detached forest_wait_api diff --git a/src/beacon/beacon_entries.rs b/src/beacon/beacon_entries.rs index 5b020a8c4af3..bffb3aec21f8 100644 --- a/src/beacon/beacon_entries.rs +++ b/src/beacon/beacon_entries.rs @@ -5,7 +5,7 @@ use crate::utils::encoding::serde_byte_array; use byteorder::{BigEndian, ByteOrder as _}; use digest::Digest as _; use get_size2::GetSize; -use serde_tuple::{self, Deserialize_tuple, Serialize_tuple}; +use serde_tuple::{Deserialize_tuple, Serialize_tuple}; /// The result from getting an entry from `Drand`. /// The entry contains the round, or epoch as well as the BLS signature for that diff --git a/src/chain/mod.rs b/src/chain/mod.rs index 2cb48288f197..ee5c7a9a9591 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -23,7 +23,7 @@ use cid::Cid; use digest::Digest; use futures::StreamExt as _; use fvm_ipld_blockstore::Blockstore; -use fvm_ipld_encoding::{DAG_CBOR, IPLD_RAW}; +use fvm_ipld_encoding::DAG_CBOR; use multihash_derive::MultihashDigest as _; use nunny::Vec as NonEmpty; use std::fs::File; @@ -83,10 +83,7 @@ pub async fn export_v2( // validate f3 data if let Some((f3_cid, f3_data)) = &mut f3 { f3_data.seek(SeekFrom::Start(0))?; - let expected_cid = Cid::new_v1( - IPLD_RAW, - MultihashCode::Blake2b256.digest_byte_stream(f3_data)?, - ); + let expected_cid = crate::f3::snapshot::get_f3_snapshot_cid(f3_data)?; anyhow::ensure!( f3_cid == &expected_cid, "f3 snapshot integrity check failed, actual cid: {f3_cid}, expected cid: {expected_cid}" diff --git a/src/chain/snapshot_format.rs b/src/chain/snapshot_format.rs index e70fd3d9e4d3..b59c6aa54eff 100644 --- a/src/chain/snapshot_format.rs +++ b/src/chain/snapshot_format.rs @@ -74,16 +74,16 @@ impl FilecoinSnapshotMetadata { impl std::fmt::Display for FilecoinSnapshotMetadata { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - writeln!(f, "Snapshot version: {}", self.version as u64)?; + writeln!(f, "Snapshot version: {}", self.version as u64)?; let head_tipset_key_string = self .head_tipset_key .iter() .map(Cid::to_string) - .join("\n "); - writeln!(f, "Head Tipset: {head_tipset_key_string}")?; + .join("\n "); + writeln!(f, "Head Tipset: {head_tipset_key_string}")?; write!( f, - "F3 data: {}", + "F3 data: {}", self.f3_data .map(|c| c.to_string()) .unwrap_or_else(|| "not found".into()) diff --git a/src/cli/subcommands/f3_cmd.rs b/src/cli/subcommands/f3_cmd.rs index edf0dd91ae7b..c501e4a0c57c 100644 --- a/src/cli/subcommands/f3_cmd.rs +++ b/src/cli/subcommands/f3_cmd.rs @@ -476,6 +476,7 @@ pub struct F3PowerTableCliJson { #[serde(rename = "CID")] #[serde_as(as = "DisplayFromStr")] cid: Cid, + #[serde(with = "crate::lotus_json")] entries: Vec, #[serde(with = "crate::lotus_json::stringify")] total: num::BigInt, diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index b8a1a46cfe12..bdf3e1213576 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -236,9 +236,10 @@ pub async fn import_chain_as_forest_car( let forest_car = ForestCar::try_from(forest_car_db_path.as_path())?; - if let Some(f3_cid) = forest_car.metadata().as_ref().and_then(|m| m.f3_data) - && let Some(mut f3_data) = forest_car.get_reader(f3_cid)? - { + if let Some(f3_cid) = forest_car.metadata().as_ref().and_then(|m| m.f3_data) { + let mut f3_data = forest_car + .get_reader(f3_cid)? + .with_context(|| format!("f3 data not found, cid: {f3_cid}"))?; let temp_f3_snap_path = tempfile::Builder::new() .suffix(".f3snap.bin") .tempfile_in(forest_car_db_dir)? @@ -522,8 +523,8 @@ mod test { file_path, temp_db_dir.path(), import_mode, - "".into(), - "".into(), + "test".into(), + "test".into(), &ChainConfig::devnet(), &SnapshotProgressTracker::default(), ) diff --git a/src/db/car/any.rs b/src/db/car/any.rs index a247c901ff95..f393ce80027e 100644 --- a/src/db/car/any.rs +++ b/src/db/car/any.rs @@ -114,7 +114,6 @@ impl AnyCar { } } - #[allow(dead_code)] /// Gets a reader of the block data by its `Cid` pub fn get_reader(&self, k: Cid) -> anyhow::Result> { match self { diff --git a/src/db/car/forest.rs b/src/db/car/forest.rs index 7b3d863ae20f..9f83f239b015 100644 --- a/src/db/car/forest.rs +++ b/src/db/car/forest.rs @@ -222,9 +222,9 @@ impl ForestCar { // `position` is the frame start offset. let cursor = Cursor::new_pos(entire_file, position); let mut decoder = zstd::Decoder::new(cursor)?.single_frame(); - while let Ok(frame_len) = decoder.read_varint::() { + while let Ok(car_block_len) = decoder.read_varint::() { let cid = Cid::read_bytes(&mut decoder)?; - let data_len = frame_len.saturating_sub(cid.encoded_len()) as u64; + let data_len = car_block_len.saturating_sub(cid.encoded_len()) as u64; if cid == k { // return the reader instead of decoding the entire data block into memory return Ok(Some(decoder.take(data_len))); diff --git a/src/f3/mod.rs b/src/f3/mod.rs index 2d069032a11b..afc915158fa8 100644 --- a/src/f3/mod.rs +++ b/src/f3/mod.rs @@ -8,6 +8,8 @@ mod go_ffi; #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] use go_ffi::*; +pub mod snapshot; + use cid::Cid; use crate::{networks::ChainConfig, utils::misc::env::is_env_set_and_truthy}; diff --git a/src/f3/snapshot.rs b/src/f3/snapshot.rs new file mode 100644 index 000000000000..806b134ae7e4 --- /dev/null +++ b/src/f3/snapshot.rs @@ -0,0 +1,46 @@ +// Copyright 2019-2025 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +#[cfg(test)] +mod tests; + +use crate::{rpc::f3::F3PowerEntry, utils::multihash::MultihashCode}; +use cid::Cid; +use fvm_ipld_encoding::{ + IPLD_RAW, + tuple::{Deserialize_tuple, Serialize_tuple}, +}; +use integer_encoding::VarIntReader as _; +use std::io::Read; + +pub fn get_f3_snapshot_cid(f3_data: &mut impl Read) -> anyhow::Result { + Ok(Cid::new_v1( + IPLD_RAW, + MultihashCode::Blake2b256.digest_byte_stream(f3_data)?, + )) +} + +#[derive(Debug, Clone, Eq, PartialEq, Serialize_tuple, Deserialize_tuple)] +pub struct F3SnapshotHeader { + pub version: u64, + pub first_instance: u64, + pub latest_instance: u64, + pub initial_power_table: Vec, +} + +impl F3SnapshotHeader { + pub fn decode_from_snapshot(f3_snapshot: &mut impl Read) -> anyhow::Result { + let data_len = f3_snapshot.read_varint::()?; + let mut data_bytes = vec![0; data_len]; + f3_snapshot.read_exact(&mut data_bytes)?; + Ok(fvm_ipld_encoding::from_slice(&data_bytes)?) + } +} + +impl std::fmt::Display for F3SnapshotHeader { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f, "F3 snapshot version: {}", self.version)?; + writeln!(f, "F3 snapshot first instance: {}", self.first_instance)?; + write!(f, "F3 snapshot last instance: {}", self.latest_instance) + } +} diff --git a/src/f3/snapshot/tests.rs b/src/f3/snapshot/tests.rs new file mode 100644 index 000000000000..996acf1f05b6 --- /dev/null +++ b/src/f3/snapshot/tests.rs @@ -0,0 +1,21 @@ +// Copyright 2019-2025 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use super::*; + +const ENCODED_F3_SNAP_HEADER_HEX: &str = "8401001a00086e7d94831a00021b71480002fbe0000000005830aedb9e1f2f4e20a6a6327b9136b8472b16f1f56cec2b6d9341a12a97dc90a39e93f026baefeea09b7507e3bf1fc70981831a00022eff4800010ae80000000058308cc2fc55933c0413e763b63b10b131b3e53f57882e0c79231de0ada3227e01ed817d689e3150874170a4a2f6b590cd93831943eb48000103dcdf0000005830b2158a380b942e099ee4c7522921d638598ebb487f97c5d5ed06a29ba47f620513316f7a596b0a4d5aff7343275c212f83190fc84700faa80000000058308a9b00505fedc14edde52278ea229f4fe91e80890f5f8951200663e41f8034986f56ffd593f4e6a9a282a81ef8dfdc1c831a0002114a4700b2180000000058308c39f779f522b8de5836f7c421427f40556f934376e493a6e0fc0c72f8e35228fcf14e116fa73146ae4d88710c668e4083190ec64700948000000000583095748c4cd115988755c49578725996df98eb96c0447f2b4d5064d6562876745950d64ffb410b6db141b3a43e49a0d6908319ea7847003a90000000005830b5a56233aa1dcbffa5a2326e9b224826b3d540998e8dffea40ee238a7c7ed67e6cc9e21fad0e3e4af9dd00329063c9cd831945b047001fd0ec10000058308cddcf1d23cf25a361cd9a451eeed1d391aae84adc2220d3b2d6d9b6c929d487581ddbde3c53b6770b1b1b3dfe21e8bc831a0001c5b3470001980000000058308d1768b8b3c8fe31025ce52323dc1112bd2e3d1b39923cab98aa821283100fe3635eee56938b63eb60eef4e117c55013831a0001e00a470001600000000058308d1768b8b3c8fe31025ce52323dc1112bd2e3d1b39923cab98aa821283100fe3635eee56938b63eb60eef4e117c55013831a0001c2ad47000138000000005830a64817823ab8728463b34ee36b44761ffdab722762d675f76f631dcfc02affc95e9f54cd765b7bc98c1619991dfbe2f2831a0002307b4600b8000000005830962b2e0e92443cec526cac167a7f80f42bd907b68c17f2e93baff6232b8f4d396b43458810dd95aea41163b7cc047222831903f54600b0000000005830a37c98356d8fa32a1a8fcbd6a009395c4a34177aae9cd28c1c652963fbe445e95513e9f0c8281047ebcbf97cdd0aed0f831a00021051460090009000005830a57b3347936bc303bdf8fc232e701270db3817507562f0cd591114adeb5596c665f828510f52f678e213ad31f0eb6b3383190e7a460078000000005830b234c6533b5b40b7241345a9e9a06eea45877b510170ae9d221bfcdd577d482ead9eb1f74992f667e96459ff43b0e5218319049b460068000000005830a6c8a62aec6bff5b185c220aebe671af9cb24560fce4913c7edffd4368ba7af67a292ab8bfde1b326d6885b27976d59b831a0001bf50460040000000005830ad31d1d68bb36fc0e830ad5d452c7e676ff3277d3ad86a45bacbdfe15b5d89119434fa003031dd2f82907717aba10f9b8319048f46002800000000583097f447a28d7a7a3a489a1683d5a6eb7f0a96bac3b67d6dfeee43bb8ed3a8b223887d9defc6436b73e8b75b2ee2cf9b4f8319066b4600280000000058309561056d88ceb20291cb4770aab1613502bbf8bd226bb0b1ba132cf770bbd00120cc6c09ebf4f4c6ae0fab38d83c833d831a0002286b4600220000000058308018ef30920f5ef4420164b13a806958563fabd74a356537bd38796d0baf070d8c10f3ba57e34ee1ccbea1517bcff9c2"; + +#[test] +pub fn test_f3_snap_header_serde() { + let encoded_block_bytes = hex::decode(ENCODED_F3_SNAP_HEADER_HEX).unwrap(); + let F3SnapshotHeader { + version, + first_instance, + latest_instance, + initial_power_table, + } = fvm_ipld_encoding::from_slice(&encoded_block_bytes).unwrap(); + assert_eq!(version, 1); + assert_eq!(first_instance, 0); + assert_eq!(latest_instance, 552573); + assert_eq!(initial_power_table.len(), 20); +} diff --git a/src/rpc/methods/f3.rs b/src/rpc/methods/f3.rs index 575e487186c9..c9fd659978fa 100644 --- a/src/rpc/methods/f3.rs +++ b/src/rpc/methods/f3.rs @@ -675,8 +675,8 @@ impl RpcMethod<1> for F3GetCertificate { let client = get_rpc_http_client()?; let mut params = ArrayParams::new(); params.insert(instance)?; - let response = client.request(Self::NAME, params).await?; - Ok(response) + let response: LotusJson = client.request(Self::NAME, params).await?; + Ok(response.into_inner()) } } @@ -693,8 +693,8 @@ impl RpcMethod<0> for F3GetLatestCertificate { async fn handle(_: Ctx, _: Self::Params) -> Result { let client = get_rpc_http_client()?; - let response = client.request(Self::NAME, ArrayParams::new()).await?; - Ok(response) + let response: LotusJson = client.request(Self::NAME, ArrayParams::new()).await?; + Ok(response.into_inner()) } } @@ -737,8 +737,8 @@ impl RpcMethod<1> for F3GetF3PowerTable { let client = get_rpc_http_client()?; let mut params = ArrayParams::new(); params.insert(tsk.into_lotus_json())?; - let response = client.request(Self::NAME, params).await?; - Ok(response) + let response: LotusJson = client.request(Self::NAME, params).await?; + Ok(response.into_inner()) } } @@ -761,8 +761,8 @@ impl RpcMethod<1> for F3GetF3PowerTableByInstance { let client = get_rpc_http_client()?; let mut params = ArrayParams::new(); params.insert(instance)?; - let response = client.request(Self::NAME, params).await?; - Ok(response) + let response: LotusJson = client.request(Self::NAME, params).await?; + Ok(response.into_inner()) } } @@ -796,8 +796,9 @@ pub enum F3GetProgress {} impl F3GetProgress { async fn run() -> anyhow::Result { let client = get_rpc_http_client()?; - let response = client.request(Self::NAME, ArrayParams::new()).await?; - Ok(response) + let response: LotusJson = + client.request(Self::NAME, ArrayParams::new()).await?; + Ok(response.into_inner()) } } @@ -821,8 +822,9 @@ pub enum F3GetManifest {} impl F3GetManifest { async fn run() -> anyhow::Result { let client = get_rpc_http_client()?; - let response = client.request(Self::NAME, ArrayParams::new()).await?; - Ok(response) + let response: LotusJson = + client.request(Self::NAME, ArrayParams::new()).await?; + Ok(response.into_inner()) } } diff --git a/src/rpc/methods/f3/types.rs b/src/rpc/methods/f3/types.rs index e965da8ea485..268916e6a766 100644 --- a/src/rpc/methods/f3/types.rs +++ b/src/rpc/methods/f3/types.rs @@ -6,14 +6,14 @@ use crate::{ blocks::{Tipset, TipsetKey}, lotus_json::{HasLotusJson, LotusJson, base64_standard, lotus_json_with_self}, networks::NetworkChain, - shim::executor::Receipt, - utils::multihash::prelude::*, + shim::{executor::Receipt, fvm_shared_latest::bigint::bigint_ser}, + utils::{encoding::serde_byte_array, multihash::prelude::*}, }; use byteorder::ByteOrder as _; use cid::Cid; use fil_actors_shared::fvm_ipld_bitfield::BitField; use flate2::read::DeflateDecoder; -use fvm_ipld_encoding::tuple::{Deserialize_tuple, Serialize_tuple}; +use fvm_ipld_encoding::tuple::*; use fvm_shared4::ActorID; use itertools::Itertools as _; use libp2p::PeerId; @@ -134,9 +134,18 @@ pub struct ECTipSet { lotus_json_with_self!(ECTipSet); /// PowerEntry represents a single entry in the PowerTable, including ActorID and its StoragePower and PubKey. +#[derive(Debug, Clone, Serialize_tuple, Deserialize_tuple, Eq, PartialEq)] +pub struct F3PowerEntry { + pub id: ActorID, + #[serde(with = "bigint_ser")] + pub power: num::BigInt, + #[serde(with = "serde_byte_array")] + pub pub_key: Vec, +} + #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Eq, PartialEq)] #[serde(rename_all = "PascalCase")] -pub struct F3PowerEntry { +pub struct F3PowerEntryLotusJson { #[serde(rename = "ID")] pub id: ActorID, #[schemars(with = "String")] @@ -146,7 +155,39 @@ pub struct F3PowerEntry { #[serde(with = "base64_standard")] pub pub_key: Vec, } -lotus_json_with_self!(F3PowerEntry); + +impl HasLotusJson for F3PowerEntry { + type LotusJson = F3PowerEntryLotusJson; + + #[cfg(test)] + fn snapshots() -> Vec<(serde_json::Value, Self)> { + use base64::Engine; + use serde_json::json; + vec![( + json!({ + "ID": 143103, + "Power": "1233789485318144", + "PubKey": "jML8VZM8BBPnY7Y7ELExs+U/V4guDHkjHeCtoyJ+Ae2BfWieMVCHQXCkova1kM2T" + }), + Self { + id: 143103, + power: num::BigInt::from_str("1233789485318144").unwrap(), + pub_key: base64::prelude::BASE64_STANDARD + .decode("jML8VZM8BBPnY7Y7ELExs+U/V4guDHkjHeCtoyJ+Ae2BfWieMVCHQXCkova1kM2T") + .unwrap(), + }, + )] + } + + fn into_lotus_json(self) -> Self::LotusJson { + let Self { id, power, pub_key } = self; + F3PowerEntryLotusJson { id, power, pub_key } + } + + fn from_lotus_json(F3PowerEntryLotusJson { id, power, pub_key }: Self::LotusJson) -> Self { + Self { id, power, pub_key } + } +} /// Entries are sorted descending order of their power, where entries with equal power are /// sorted by ascending order of their ID. diff --git a/src/shim/actors/builtin/verifreg/mod.rs b/src/shim/actors/builtin/verifreg/mod.rs index b4dae1d9a0ea..ebe292fb7968 100644 --- a/src/shim/actors/builtin/verifreg/mod.rs +++ b/src/shim/actors/builtin/verifreg/mod.rs @@ -14,8 +14,7 @@ use fil_actor_verifreg_state::{ use fil_actors_shared::v8::{HAMT_BIT_WIDTH, make_map_with_root_and_bitwidth}; use fil_actors_shared::v9::Keyer; use fvm_ipld_blockstore::Blockstore; -use fvm_ipld_encoding::tuple::serde_tuple; -use fvm_ipld_encoding::tuple::{Deserialize_tuple, Serialize_tuple}; +use fvm_ipld_encoding::tuple::*; use fvm_shared2::address::{Address, Protocol}; use fvm_shared4::ActorID; use fvm_shared4::bigint::bigint_ser::BigIntDe; diff --git a/src/tool/subcommands/archive_cmd.rs b/src/tool/subcommands/archive_cmd.rs index 98e27bd5f9fd..216dec9dbc90 100644 --- a/src/tool/subcommands/archive_cmd.rs +++ b/src/tool/subcommands/archive_cmd.rs @@ -27,14 +27,16 @@ //! Additional reading: [`crate::db::car::plain`] use crate::blocks::Tipset; -use crate::chain::FilecoinSnapshotVersion; use crate::chain::{ ChainEpochDelta, index::{ChainIndex, ResolveNullTipset}, }; +use crate::chain::{FilecoinSnapshotMetadata, FilecoinSnapshotVersion}; use crate::cid_collections::CidHashSet; use crate::cli_shared::{snapshot, snapshot::TrustedVendor}; +use crate::db::car::forest::DEFAULT_FOREST_CAR_COMPRESSION_LEVEL; use crate::db::car::{AnyCar, ManyCar}; +use crate::f3::snapshot::F3SnapshotHeader; use crate::interpreter::VMTrace; use crate::ipld::{stream_graph, unordered_stream_graph}; use crate::networks::{ChainConfig, NetworkChain, butterflynet, calibnet, mainnet}; @@ -43,16 +45,22 @@ use crate::shim::clock::{ChainEpoch, EPOCH_DURATION_SECONDS, EPOCHS_IN_DAY}; use crate::shim::fvm_shared_latest::address::Network; use crate::shim::machine::GLOBAL_MULTI_ENGINE; use crate::state_manager::{NO_CALLBACK, StateOutput, apply_block_messages}; +use crate::utils::db::car_stream::{CarBlock, CarBlockWrite as _, CarStream}; +use crate::utils::multihash::MultihashCode; use anyhow::{Context as _, bail}; use chrono::DateTime; use cid::Cid; use clap::{Subcommand, ValueEnum}; use dialoguer::{Confirm, theme::ColorfulTheme}; -use futures::TryStreamExt; +use futures::{StreamExt as _, TryStreamExt as _}; use fvm_ipld_blockstore::Blockstore; +use fvm_ipld_encoding::DAG_CBOR; use indicatif::ProgressIterator; use itertools::Itertools; +use multihash_derive::MultihashDigest as _; use sha2::Sha256; +use std::fs::File; +use std::io::{Seek as _, SeekFrom}; use std::ops::Range; use std::path::PathBuf; use std::sync::Arc; @@ -138,6 +146,18 @@ pub enum ArchiveCommands { #[arg(long, default_value_t = false)] force: bool, }, + /// Merge a v1 Filecoin snapshot with an F3 snapshot into a v2 Filecoin snapshot in `.forest.car.zst` format + MergeF3 { + /// Path to the Filecoin snapshot + #[arg(long)] + filecoin: PathBuf, + /// Path to the F3 snapshot + #[arg(long)] + f3: PathBuf, + /// Snapshot output filename + #[arg(long)] + output: PathBuf, + }, /// Show the difference between the canonical and computed state of a /// tipset. Diff { @@ -198,6 +218,13 @@ impl ArchiveCommands { let store = AnyCar::try_from(snapshot.as_path())?; if let Some(metadata) = store.metadata() { println!("{metadata}"); + if let Some(f3_cid) = metadata.f3_data { + let mut f3_data = store + .get_reader(f3_cid)? + .with_context(|| format!("f3 data not found, cid: {f3_cid}"))?; + let f3_snap_header = F3SnapshotHeader::decode_from_snapshot(&mut f3_data)?; + println!("{f3_snap_header}"); + } } else { println!( "No metadata found (required by v2 snapshot) - this appears to be a v1 snapshot" @@ -236,6 +263,11 @@ impl ArchiveCommands { output_path, force, } => merge_snapshots(snapshot_files, output_path, force).await, + Self::MergeF3 { + filecoin, + f3, + output, + } => merge_f3_snapshot(filecoin, f3, output).await, Self::Diff { snapshot_files, epoch, @@ -607,6 +639,89 @@ async fn merge_snapshots( Ok(()) } +async fn merge_f3_snapshot(filecoin: PathBuf, f3: PathBuf, output: PathBuf) -> anyhow::Result<()> { + { + let store = AnyCar::try_from(filecoin.as_path())?; + anyhow::ensure!( + store.metadata().is_none(), + "The filecoin snapshot is not in v1 format" + ); + } + + let mut f3_data = File::open(f3)?; + let f3_cid = crate::f3::snapshot::get_f3_snapshot_cid(&mut f3_data)?; + + let car_stream = CarStream::new(tokio::io::BufReader::new( + tokio::fs::File::open(&filecoin).await?, + )) + .await?; + + let chain_head = car_stream.header_v1.roots.clone(); + + println!("f3 snapshot cid: {f3_cid}"); + println!( + "chain head: [{}]", + chain_head.iter().map(|c| c.to_string()).join(", ") + ); + + let snap_meta = FilecoinSnapshotMetadata::new_v2(chain_head, Some(f3_cid)); + let snap_meta_cbor_encoded = fvm_ipld_encoding::to_vec(&snap_meta)?; + let snap_meta_block = CarBlock { + cid: Cid::new_v1( + DAG_CBOR, + MultihashCode::Blake2b256.digest(&snap_meta_cbor_encoded), + ), + data: snap_meta_cbor_encoded, + }; + + let roots = nunny::vec![snap_meta_block.cid]; + let snap_meta_frame = { + let mut encoder = + crate::db::car::forest::new_encoder(DEFAULT_FOREST_CAR_COMPRESSION_LEVEL)?; + snap_meta_block.write(&mut encoder)?; + anyhow::Ok(( + vec![snap_meta_block.cid], + crate::db::car::forest::finalize_frame( + DEFAULT_FOREST_CAR_COMPRESSION_LEVEL, + &mut encoder, + )?, + )) + }; + let f3_frame = { + let mut encoder = + crate::db::car::forest::new_encoder(DEFAULT_FOREST_CAR_COMPRESSION_LEVEL)?; + f3_data.seek(SeekFrom::Start(0))?; + encoder.write_car_block(f3_cid, f3_data.metadata()?.len() as _, &mut f3_data)?; + anyhow::Ok(( + vec![f3_cid], + crate::db::car::forest::finalize_frame( + DEFAULT_FOREST_CAR_COMPRESSION_LEVEL, + &mut encoder, + )?, + )) + }; + + let block_frames = crate::db::car::forest::Encoder::compress_stream_default( + car_stream.map_err(anyhow::Error::from), + ); + let frames = futures::stream::iter([snap_meta_frame, f3_frame]).chain(block_frames); + + let temp_output = { + let mut dir = output.clone(); + if dir.pop() { + tempfile::NamedTempFile::new_in(dir)? + } else { + tempfile::NamedTempFile::new_in(".")? + } + }; + let mut writer = tokio::io::BufWriter::new(tokio::fs::File::create(&temp_output).await?); + crate::db::car::forest::Encoder::write(&mut writer, roots, frames).await?; + writer.shutdown().await?; + temp_output.persist(&output)?; + + Ok(()) +} + /// Compute the tree of actor states for a given epoch and compare it to the /// expected result (as encoded in the blockchain). Differences are printed /// using the diff format (red for the blockchain state, green for the computed From baf3d4232a94cd813cfea37f965541046141fdb4 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 6 Aug 2025 21:35:51 +0800 Subject: [PATCH 23/35] changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 277cfbe61615..12500a3ebc06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,10 +35,14 @@ - [#5859](https://github.com/ChainSafe/forest/pull/5859) Added size metrics for zstd frame cache and made max size configurable via `FOREST_ZSTD_FRAME_CACHE_DEFAULT_MAX_SIZE` environment variable. +- [#5886](https://github.com/ChainSafe/forest/issues/5886) Add `forest-tool archive merge-f3` subcommand for merging a v1 Filecoin snapshot and an F3 snapshot into a v2 Filecoin snapshot. + - [#4976](https://github.com/ChainSafe/forest/issues/4976) Add support for the `Filecoin.EthSubscribe` and `Filecoin.EthUnsubscribe` API methods to enable subscriptions to Ethereum event types: `heads` and `logs`. ### Changed +- [#5886](https://github.com/ChainSafe/forest/issues/5886) Updated `forest-tool archive metadata` to print F3 snapshot header info when applicable. + - [#5869](https://github.com/ChainSafe/forest/pull/5869) Updated `forest-cli snapshot export` to print average speed. ### Removed From 9181ead27f67ec081237e0ff653c5a64571ca0fa Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 6 Aug 2025 22:44:06 +0800 Subject: [PATCH 24/35] unit test coverage for get_reader & resolve AI comments --- src/chain/mod.rs | 3 ++- src/daemon/db_util.rs | 14 +++++++------- src/db/car/forest.rs | 10 +++++++++- src/db/car/plain.rs | 10 ++++++++-- src/f3/snapshot.rs | 7 +++++++ src/rpc/methods/f3.rs | 2 +- src/tool/subcommands/archive_cmd.rs | 3 ++- 7 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/chain/mod.rs b/src/chain/mod.rs index ee5c7a9a9591..9e5868b1e58f 100644 --- a/src/chain/mod.rs +++ b/src/chain/mod.rs @@ -112,10 +112,11 @@ pub async fn export_v2( }]; if let Some((f3_cid, mut f3_data)) = f3 { + let f3_data_len = f3_data.seek(SeekFrom::End(0))?; f3_data.seek(SeekFrom::Start(0))?; prefix_data_frames.push({ let mut encoder = forest::new_encoder(forest::DEFAULT_FOREST_CAR_COMPRESSION_LEVEL)?; - encoder.write_car_block(f3_cid, f3_data.metadata()?.len() as _, &mut f3_data)?; + encoder.write_car_block(f3_cid, f3_data_len as _, &mut f3_data)?; anyhow::Ok(( vec![f3_cid], finalize_frame(forest::DEFAULT_FOREST_CAR_COMPRESSION_LEVEL, &mut encoder)?, diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index bdf3e1213576..2ccc1999070e 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -18,7 +18,6 @@ use anyhow::{Context, bail}; use futures::TryStreamExt; use serde::{Deserialize, Serialize}; use std::ffi::OsStr; -use std::fs::File; use std::{ fs, path::{Path, PathBuf}, @@ -240,20 +239,21 @@ pub async fn import_chain_as_forest_car( let mut f3_data = forest_car .get_reader(f3_cid)? .with_context(|| format!("f3 data not found, cid: {f3_cid}"))?; - let temp_f3_snap_path = tempfile::Builder::new() + let mut temp_f3_snap = tempfile::Builder::new() .suffix(".f3snap.bin") - .tempfile_in(forest_car_db_dir)? - .into_temp_path(); + .tempfile_in(forest_car_db_dir)?; { - let mut f = File::create(&temp_f3_snap_path)?; - std::io::copy(&mut f3_data, &mut f)?; + let f = temp_f3_snap.as_file_mut(); + std::io::copy(&mut f3_data, f)?; + f.sync_all()?; } if let Err(e) = crate::f3::import_f3_snapshot( chain_config, rpc_endpoint, f3_root, - temp_f3_snap_path.display().to_string(), + temp_f3_snap.path().display().to_string(), ) { + // Do not make it a hard error if anything is wrong with F3 snapshot tracing::error!("Failed to import F3 snapshot: {e}"); } } diff --git a/src/db/car/forest.rs b/src/db/car/forest.rs index 9f83f239b015..dd4d7080b25f 100644 --- a/src/db/car/forest.rs +++ b/src/db/car/forest.rs @@ -511,7 +511,15 @@ mod tests { ForestCar::new(mk_encoded_car(1024 * 4, 3, roots.clone(), blocks.clone())).unwrap(); assert_eq!(forest_car.head_tipset_key(), &roots); for block in blocks { - assert_eq!(forest_car.get(&block.cid).unwrap(), Some(block.data)); + assert_eq!(forest_car.get(&block.cid).unwrap().unwrap(), block.data); + let mut buf = vec![]; + forest_car + .get_reader(block.cid) + .unwrap() + .unwrap() + .read_to_end(&mut buf) + .unwrap(); + assert_eq!(buf, block.data); } } diff --git a/src/db/car/plain.rs b/src/db/car/plain.rs index 12a31dfa5de1..4966cbfa73c3 100644 --- a/src/db/car/plain.rs +++ b/src/db/car/plain.rs @@ -225,7 +225,6 @@ impl PlainCar { } } - #[allow(dead_code)] /// Gets a reader of the block data by its `Cid` pub fn get_reader(&self, k: Cid) -> Option { self.index @@ -467,7 +466,7 @@ where #[cfg(test)] mod tests { - use super::PlainCar; + use super::*; use crate::utils::db::{ car_stream::{CarStream, CarV1Header}, car_util::load_car, @@ -494,10 +493,17 @@ mod tests { let expected = reference_car.get(&cid).unwrap().unwrap(); let expected2 = reference_car_zst.get(&cid).unwrap().unwrap(); let expected3 = reference_car_zst_unsafe.get(&cid).unwrap().unwrap(); + let mut expected4 = vec![]; + car_backed + .get_reader(cid) + .unwrap() + .read_to_end(&mut expected4) + .unwrap(); let actual = car_backed.get(&cid).unwrap().unwrap(); assert_eq!(expected, actual); assert_eq!(expected2, actual); assert_eq!(expected3, actual); + assert_eq!(expected4, actual); } } diff --git a/src/f3/snapshot.rs b/src/f3/snapshot.rs index 806b134ae7e4..764caac6d923 100644 --- a/src/f3/snapshot.rs +++ b/src/f3/snapshot.rs @@ -30,7 +30,14 @@ pub struct F3SnapshotHeader { impl F3SnapshotHeader { pub fn decode_from_snapshot(f3_snapshot: &mut impl Read) -> anyhow::Result { + // Reasonable upper bound for snapshot header size (100MiB) + const MAX_HEADER_SIZE: usize = 100 * 1024 * 1024; + let data_len = f3_snapshot.read_varint::()?; + anyhow::ensure!( + data_len <= MAX_HEADER_SIZE, + "F3 snapshot header size {data_len} exceeds maximum allowed size {MAX_HEADER_SIZE}" + ); let mut data_bytes = vec![0; data_len]; f3_snapshot.read_exact(&mut data_bytes)?; Ok(fvm_ipld_encoding::from_slice(&data_bytes)?) diff --git a/src/rpc/methods/f3.rs b/src/rpc/methods/f3.rs index c9fd659978fa..9e06896ea9bc 100644 --- a/src/rpc/methods/f3.rs +++ b/src/rpc/methods/f3.rs @@ -644,7 +644,7 @@ impl RpcMethod<1> for F3ExportLatestSnapshot { const API_PATHS: BitFlags = ApiPaths::all(); const PERMISSION: Permission = Permission::Read; const DESCRIPTION: Option<&'static str> = - Some("Gets the power table (committee) used to validate the specified instance"); + Some("Exports the latest F3 snapshot to the specified path and returns its CID"); type Params = (String,); type Ok = Cid; diff --git a/src/tool/subcommands/archive_cmd.rs b/src/tool/subcommands/archive_cmd.rs index 216dec9dbc90..988d2af535c0 100644 --- a/src/tool/subcommands/archive_cmd.rs +++ b/src/tool/subcommands/archive_cmd.rs @@ -690,8 +690,9 @@ async fn merge_f3_snapshot(filecoin: PathBuf, f3: PathBuf, output: PathBuf) -> a let f3_frame = { let mut encoder = crate::db::car::forest::new_encoder(DEFAULT_FOREST_CAR_COMPRESSION_LEVEL)?; + let f3_data_len = f3_data.seek(SeekFrom::End(0))?; f3_data.seek(SeekFrom::Start(0))?; - encoder.write_car_block(f3_cid, f3_data.metadata()?.len() as _, &mut f3_data)?; + encoder.write_car_block(f3_cid, f3_data_len as _, &mut f3_data)?; anyhow::Ok(( vec![f3_cid], crate::db::car::forest::finalize_frame( From e18e3516689d9ebddac29331adb74e4092d43117 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 7 Aug 2025 08:47:25 +0800 Subject: [PATCH 25/35] update doc gen script --- docs/docs/users/reference/cli.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/docs/users/reference/cli.sh b/docs/docs/users/reference/cli.sh index 4baa6c63b3e1..5da6f06810c6 100755 --- a/docs/docs/users/reference/cli.sh +++ b/docs/docs/users/reference/cli.sh @@ -119,7 +119,9 @@ generate_markdown_section "forest-tool" "archive" generate_markdown_section "forest-tool" "archive info" generate_markdown_section "forest-tool" "archive export" generate_markdown_section "forest-tool" "archive checkpoints" +generate_markdown_section "forest-tool" "archive metadata" generate_markdown_section "forest-tool" "archive merge" +generate_markdown_section "forest-tool" "archive merge-f3" generate_markdown_section "forest-tool" "archive diff" generate_markdown_section "forest-tool" "archive sync-bucket" From e9b6572b8b0f16704c69142ffacc9674ca25afbf Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 7 Aug 2025 19:12:57 +0800 Subject: [PATCH 26/35] print size and speed in `merge-f3` command output --- scripts/tests/harness.sh | 5 +++- src/tool/subcommands/archive_cmd.rs | 46 +++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/scripts/tests/harness.sh b/scripts/tests/harness.sh index b6d6ead9e999..132819a0b158 100644 --- a/scripts/tests/harness.sh +++ b/scripts/tests/harness.sh @@ -34,8 +34,11 @@ function forest_download_and_import_snapshot_with_f3 { aria2c -x5 https://forest-snapshots.fra1.cdn.digitaloceanspaces.com/f3/f3_snap_calibnet_552628.bin -o f3.bin echo "Generating v2 snapshot" $FOREST_TOOL_PATH archive merge-f3 --filecoin v1.forest.car.zst --f3 f3.bin --output v2.forest.car.zst + echo "Inspecting archive info" $FOREST_TOOL_PATH archive info v2.forest.car.zst + echo "Inspecting archive metadata" $FOREST_TOOL_PATH archive metadata v2.forest.car.zst + echo "Importing the v2 snapshot" $FOREST_PATH --chain calibnet --encrypt-keystore false --halt-after-import --height=-200 --import-snapshot v2.forest.car.zst } @@ -147,7 +150,7 @@ function forest_init_with_f3 { forest_wait_for_healthcheck_ready - # print the latest F3 certificate + echo "Print the latest F3 certificate" $FOREST_CLI_PATH f3 c get echo "ensure F3 certificate at instance 550000 has been imported" $FOREST_CLI_PATH f3 c get 550000 diff --git a/src/tool/subcommands/archive_cmd.rs b/src/tool/subcommands/archive_cmd.rs index 988d2af535c0..d2d6e60d260c 100644 --- a/src/tool/subcommands/archive_cmd.rs +++ b/src/tool/subcommands/archive_cmd.rs @@ -34,8 +34,7 @@ use crate::chain::{ use crate::chain::{FilecoinSnapshotMetadata, FilecoinSnapshotVersion}; use crate::cid_collections::CidHashSet; use crate::cli_shared::{snapshot, snapshot::TrustedVendor}; -use crate::db::car::forest::DEFAULT_FOREST_CAR_COMPRESSION_LEVEL; -use crate::db::car::{AnyCar, ManyCar}; +use crate::db::car::{AnyCar, ManyCar, forest::DEFAULT_FOREST_CAR_COMPRESSION_LEVEL}; use crate::f3::snapshot::F3SnapshotHeader; use crate::interpreter::VMTrace; use crate::ipld::{stream_graph, unordered_stream_graph}; @@ -55,15 +54,18 @@ use dialoguer::{Confirm, theme::ColorfulTheme}; use futures::{StreamExt as _, TryStreamExt as _}; use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::DAG_CBOR; +use human_repr::HumanCount as _; use indicatif::ProgressIterator; use itertools::Itertools; use multihash_derive::MultihashDigest as _; +use num::Zero as _; use sha2::Sha256; use std::fs::File; -use std::io::{Seek as _, SeekFrom}; +use std::io::{Seek as _, SeekFrom, Write as _}; use std::ops::Range; use std::path::PathBuf; use std::sync::Arc; +use std::time::Instant; use tokio::io::{AsyncWriteExt, BufWriter}; use tracing::info; @@ -716,10 +718,48 @@ async fn merge_f3_snapshot(filecoin: PathBuf, f3: PathBuf, output: PathBuf) -> a } }; let mut writer = tokio::io::BufWriter::new(tokio::fs::File::create(&temp_output).await?); + + let handle = tokio::spawn({ + let start = Instant::now(); + let temp_output = temp_output.path().to_owned(); + let output = output.clone(); + async move { + let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); + println!("Getting ready..."); + loop { + interval.tick().await; + let snapshot_size = std::fs::metadata(&temp_output) + .map(|meta| meta.len()) + .unwrap_or(0); + print!( + "{}{}", + anes::MoveCursorToPreviousLine(1), + anes::ClearLine::All + ); + let elapsed_secs = start.elapsed().as_secs_f64(); + println!( + "{}: {} ({}/s)", + output.display(), + snapshot_size.human_count_bytes(), + if elapsed_secs.is_zero() { + 0. + } else { + (snapshot_size as f64) / elapsed_secs + } + .human_count_bytes(), + ); + _ = std::io::stdout().flush(); + } + } + }); + crate::db::car::forest::Encoder::write(&mut writer, roots, frames).await?; writer.shutdown().await?; temp_output.persist(&output)?; + handle.abort(); + handle.await?; + Ok(()) } From 9b8395ba0ee8d3530174f853980c3cfd43e1f253 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 7 Aug 2025 19:57:31 +0800 Subject: [PATCH 27/35] chore(ci): add go linter --- .github/workflows/go-lint.yml | 27 +++ .golangci.yml | 3 + Makefile | 6 + f3-sidecar/ffi_impl.go | 3 +- interop-tests/src/tests/go_app/.gitignore | 1 - interop-tests/src/tests/go_app/common.go | 3 +- interop-tests/src/tests/go_app/gen.go | 237 ++++++++++++++++++++++ 7 files changed, 277 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/go-lint.yml create mode 100644 .golangci.yml delete mode 100644 interop-tests/src/tests/go_app/.gitignore create mode 100644 interop-tests/src/tests/go_app/gen.go diff --git a/.github/workflows/go-lint.yml b/.github/workflows/go-lint.yml new file mode 100644 index 000000000000..f450c4683359 --- /dev/null +++ b/.github/workflows/go-lint.yml @@ -0,0 +1,27 @@ +name: Go code linters + +# Cancel workflow if there is a new change to the branch. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +on: + workflow_dispatch: + merge_group: + pull_request: + branches: + - main + push: + branches: + - main + +jobs: + lint-go: + name: Go lint checks + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: "go.work" + - run: make lint-go diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000000..4bbb013f4e47 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,3 @@ +version: "2" +linters: + default: standard diff --git a/Makefile b/Makefile index 39b9dc0f671d..04a7669b317c 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +GOLANGCI_LINT_VERSION=v2.3.1 + install: cargo install --locked --path . --force @@ -77,6 +79,10 @@ DOCKERFILES=$(wildcard Dockerfile*) lint-docker: $(DOCKERFILES) docker run --rm -i hadolint/hadolint < $< +lint-go: + go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) + go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) run --timeout 10m --concurrency 4 ./f3-sidecar ./interop-tests/src/tests/go_app + # Formats Rust, TOML and Markdown files. fmt: cargo fmt --all diff --git a/f3-sidecar/ffi_impl.go b/f3-sidecar/ffi_impl.go index e58a1c173d98..ca4356449a01 100644 --- a/f3-sidecar/ffi_impl.go +++ b/f3-sidecar/ffi_impl.go @@ -57,5 +57,6 @@ func checkError(err error) { // To avoid potential panics // See func setGoDebugEnv() { - os.Setenv("GODEBUG", "invalidptr=0,cgocheck=0") + err := os.Setenv("GODEBUG", "invalidptr=0,cgocheck=0") + checkError(err) } diff --git a/interop-tests/src/tests/go_app/.gitignore b/interop-tests/src/tests/go_app/.gitignore deleted file mode 100644 index 40781048e6eb..000000000000 --- a/interop-tests/src/tests/go_app/.gitignore +++ /dev/null @@ -1 +0,0 @@ -gen.go diff --git a/interop-tests/src/tests/go_app/common.go b/interop-tests/src/tests/go_app/common.go index 045c3cf3d1c1..c06bb3bdfd29 100644 --- a/interop-tests/src/tests/go_app/common.go +++ b/interop-tests/src/tests/go_app/common.go @@ -13,5 +13,6 @@ func checkError(err error) { // To avoid potential panics // See func setGoDebugEnv() { - os.Setenv("GODEBUG", "invalidptr=0,cgocheck=0") + err := os.Setenv("GODEBUG", "invalidptr=0,cgocheck=0") + checkError(err) } diff --git a/interop-tests/src/tests/go_app/gen.go b/interop-tests/src/tests/go_app/gen.go new file mode 100644 index 000000000000..c9bce1458eca --- /dev/null +++ b/interop-tests/src/tests/go_app/gen.go @@ -0,0 +1,237 @@ +package main + +/* +// Generated by rust2go. Please DO NOT edit this C part manually. + +#include +#include +#include +#include + +typedef struct ListRef { + const void *ptr; + uintptr_t len; +} ListRef; + +typedef struct StringRef { + const uint8_t *ptr; + uintptr_t len; +} StringRef; +*/ +import "C" +import ( + "runtime" + "unsafe" + + "github.com/ihciah/rust2go/asmcall" +) + +var GoKadNodeImpl GoKadNode + +type GoKadNode interface { + run() + connect(multiaddr *string) + get_n_connected() uint +} + +//export CGoKadNode_run +func CGoKadNode_run() { + GoKadNodeImpl.run() +} + +//export CGoKadNode_connect +func CGoKadNode_connect(multiaddr C.StringRef) { + _new_multiaddr := newString(multiaddr) + GoKadNodeImpl.connect(&_new_multiaddr) +} + +//export CGoKadNode_get_n_connected +func CGoKadNode_get_n_connected(slot *C.void, cb *C.void) { + resp := GoKadNodeImpl.get_n_connected() + resp_ref, buffer := cvt_ref(cntC_uintptr_t, refC_uintptr_t)(&resp) + asmcall.CallFuncG0P2(unsafe.Pointer(cb), unsafe.Pointer(&resp_ref), unsafe.Pointer(slot)) + runtime.KeepAlive(resp_ref) + runtime.KeepAlive(resp) + runtime.KeepAlive(buffer) +} + +var GoBitswapNodeImpl GoBitswapNode + +type GoBitswapNode interface { + run() + connect(multiaddr *string) + get_block(cid *string) bool +} + +//export CGoBitswapNode_run +func CGoBitswapNode_run() { + GoBitswapNodeImpl.run() +} + +//export CGoBitswapNode_connect +func CGoBitswapNode_connect(multiaddr C.StringRef) { + _new_multiaddr := newString(multiaddr) + GoBitswapNodeImpl.connect(&_new_multiaddr) +} + +//export CGoBitswapNode_get_block +func CGoBitswapNode_get_block(cid C.StringRef, slot *C.void, cb *C.void) { + _new_cid := newString(cid) + resp := GoBitswapNodeImpl.get_block(&_new_cid) + resp_ref, buffer := cvt_ref(cntC_bool, refC_bool)(&resp) + asmcall.CallFuncG0P2(unsafe.Pointer(cb), unsafe.Pointer(&resp_ref), unsafe.Pointer(slot)) + runtime.KeepAlive(resp_ref) + runtime.KeepAlive(resp) + runtime.KeepAlive(buffer) +} + +func newString(s_ref C.StringRef) string { + return unsafe.String((*byte)(unsafe.Pointer(s_ref.ptr)), s_ref.len) +} +func refString(s *string, _ *[]byte) C.StringRef { + return C.StringRef{ + ptr: (*C.uint8_t)(unsafe.StringData(*s)), + len: C.uintptr_t(len(*s)), + } +} + +func ownString(s_ref C.StringRef) string { + return string(unsafe.Slice((*byte)(unsafe.Pointer(s_ref.ptr)), int(s_ref.len))) +} +func cntString(_ *string, _ *uint) [0]C.StringRef { return [0]C.StringRef{} } +func new_list_mapper[T1, T2 any](f func(T1) T2) func(C.ListRef) []T2 { + return func(x C.ListRef) []T2 { + input := unsafe.Slice((*T1)(unsafe.Pointer(x.ptr)), x.len) + output := make([]T2, len(input)) + for i, v := range input { + output[i] = f(v) + } + return output + } +} +func new_list_mapper_primitive[T1, T2 any](_ func(T1) T2) func(C.ListRef) []T2 { + return func(x C.ListRef) []T2 { + return unsafe.Slice((*T2)(unsafe.Pointer(x.ptr)), x.len) + } +} + +// only handle non-primitive type T +func cnt_list_mapper[T, R any](f func(s *T, cnt *uint) [0]R) func(s *[]T, cnt *uint) [0]C.ListRef { + return func(s *[]T, cnt *uint) [0]C.ListRef { + for _, v := range *s { + f(&v, cnt) + } + *cnt += uint(len(*s)) * size_of[R]() + return [0]C.ListRef{} + } +} + +// only handle primitive type T +func cnt_list_mapper_primitive[T, R any](_ func(s *T, cnt *uint) [0]R) func(s *[]T, cnt *uint) [0]C.ListRef { + return func(s *[]T, cnt *uint) [0]C.ListRef { return [0]C.ListRef{} } +} + +// only handle non-primitive type T +func ref_list_mapper[T, R any](f func(s *T, buffer *[]byte) R) func(s *[]T, buffer *[]byte) C.ListRef { + return func(s *[]T, buffer *[]byte) C.ListRef { + if len(*buffer) == 0 { + return C.ListRef{ + ptr: unsafe.Pointer(nil), + len: C.uintptr_t(len(*s)), + } + } + ret := C.ListRef{ + ptr: unsafe.Pointer(&(*buffer)[0]), + len: C.uintptr_t(len(*s)), + } + children_bytes := int(size_of[R]()) * len(*s) + children := (*buffer)[:children_bytes] + *buffer = (*buffer)[children_bytes:] + for _, v := range *s { + child := f(&v, buffer) + len := unsafe.Sizeof(child) + copy(children, unsafe.Slice((*byte)(unsafe.Pointer(&child)), len)) + children = children[len:] + } + return ret + } +} + +// only handle primitive type T +func ref_list_mapper_primitive[T, R any](_ func(s *T, buffer *[]byte) R) func(s *[]T, buffer *[]byte) C.ListRef { + return func(s *[]T, buffer *[]byte) C.ListRef { + if len(*s) == 0 { + return C.ListRef{ + ptr: unsafe.Pointer(nil), + len: C.uintptr_t(0), + } + } + return C.ListRef{ + ptr: unsafe.Pointer(&(*s)[0]), + len: C.uintptr_t(len(*s)), + } + } +} +func size_of[T any]() uint { + var t T + return uint(unsafe.Sizeof(t)) +} +func cvt_ref[R, CR any](cnt_f func(s *R, cnt *uint) [0]CR, ref_f func(p *R, buffer *[]byte) CR) func(p *R) (CR, []byte) { + return func(p *R) (CR, []byte) { + var cnt uint + cnt_f(p, &cnt) + buffer := make([]byte, cnt) + return ref_f(p, &buffer), buffer + } +} +func cvt_ref_cap[R, CR any](cnt_f func(s *R, cnt *uint) [0]CR, ref_f func(p *R, buffer *[]byte) CR, add_cap uint) func(p *R) (CR, []byte) { + return func(p *R) (CR, []byte) { + var cnt uint + cnt_f(p, &cnt) + buffer := make([]byte, cnt, cnt+add_cap) + return ref_f(p, &buffer), buffer + } +} + +func newC_uint8_t(n C.uint8_t) uint8 { return uint8(n) } +func newC_uint16_t(n C.uint16_t) uint16 { return uint16(n) } +func newC_uint32_t(n C.uint32_t) uint32 { return uint32(n) } +func newC_uint64_t(n C.uint64_t) uint64 { return uint64(n) } +func newC_int8_t(n C.int8_t) int8 { return int8(n) } +func newC_int16_t(n C.int16_t) int16 { return int16(n) } +func newC_int32_t(n C.int32_t) int32 { return int32(n) } +func newC_int64_t(n C.int64_t) int64 { return int64(n) } +func newC_bool(n C.bool) bool { return bool(n) } +func newC_uintptr_t(n C.uintptr_t) uint { return uint(n) } +func newC_intptr_t(n C.intptr_t) int { return int(n) } +func newC_float(n C.float) float32 { return float32(n) } +func newC_double(n C.double) float64 { return float64(n) } + +func cntC_uint8_t(_ *uint8, _ *uint) [0]C.uint8_t { return [0]C.uint8_t{} } +func cntC_uint16_t(_ *uint16, _ *uint) [0]C.uint16_t { return [0]C.uint16_t{} } +func cntC_uint32_t(_ *uint32, _ *uint) [0]C.uint32_t { return [0]C.uint32_t{} } +func cntC_uint64_t(_ *uint64, _ *uint) [0]C.uint64_t { return [0]C.uint64_t{} } +func cntC_int8_t(_ *int8, _ *uint) [0]C.int8_t { return [0]C.int8_t{} } +func cntC_int16_t(_ *int16, _ *uint) [0]C.int16_t { return [0]C.int16_t{} } +func cntC_int32_t(_ *int32, _ *uint) [0]C.int32_t { return [0]C.int32_t{} } +func cntC_int64_t(_ *int64, _ *uint) [0]C.int64_t { return [0]C.int64_t{} } +func cntC_bool(_ *bool, _ *uint) [0]C.bool { return [0]C.bool{} } +func cntC_uintptr_t(_ *uint, _ *uint) [0]C.uintptr_t { return [0]C.uintptr_t{} } +func cntC_intptr_t(_ *int, _ *uint) [0]C.intptr_t { return [0]C.intptr_t{} } +func cntC_float(_ *float32, _ *uint) [0]C.float { return [0]C.float{} } +func cntC_double(_ *float64, _ *uint) [0]C.double { return [0]C.double{} } + +func refC_uint8_t(p *uint8, _ *[]byte) C.uint8_t { return C.uint8_t(*p) } +func refC_uint16_t(p *uint16, _ *[]byte) C.uint16_t { return C.uint16_t(*p) } +func refC_uint32_t(p *uint32, _ *[]byte) C.uint32_t { return C.uint32_t(*p) } +func refC_uint64_t(p *uint64, _ *[]byte) C.uint64_t { return C.uint64_t(*p) } +func refC_int8_t(p *int8, _ *[]byte) C.int8_t { return C.int8_t(*p) } +func refC_int16_t(p *int16, _ *[]byte) C.int16_t { return C.int16_t(*p) } +func refC_int32_t(p *int32, _ *[]byte) C.int32_t { return C.int32_t(*p) } +func refC_int64_t(p *int64, _ *[]byte) C.int64_t { return C.int64_t(*p) } +func refC_bool(p *bool, _ *[]byte) C.bool { return C.bool(*p) } +func refC_uintptr_t(p *uint, _ *[]byte) C.uintptr_t { return C.uintptr_t(*p) } +func refC_intptr_t(p *int, _ *[]byte) C.intptr_t { return C.intptr_t(*p) } +func refC_float(p *float32, _ *[]byte) C.float { return C.float(*p) } +func refC_double(p *float64, _ *[]byte) C.double { return C.double(*p) } +func main() {} From 379633de9d2b4dd0cd828ccac62a2d4d26b56cd8 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 7 Aug 2025 20:09:41 +0800 Subject: [PATCH 28/35] Potential fix for code scanning alert no. 63: Workflow does not contain permissions Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/go-lint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/go-lint.yml b/.github/workflows/go-lint.yml index f450c4683359..570be5550ef3 100644 --- a/.github/workflows/go-lint.yml +++ b/.github/workflows/go-lint.yml @@ -1,4 +1,6 @@ name: Go code linters +permissions: + contents: read # Cancel workflow if there is a new change to the branch. concurrency: From b481cf891a43e28dfe94d5bbe355248624a0e51c Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 7 Aug 2025 20:25:22 +0800 Subject: [PATCH 29/35] fix go lint errors --- f3-sidecar/api.go | 15 ++++++++++++--- f3-sidecar/import.go | 24 ++++++++++++++++++++---- f3-sidecar/run.go | 8 ++++++-- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/f3-sidecar/api.go b/f3-sidecar/api.go index 91730822a70a..e11ac816e12a 100644 --- a/f3-sidecar/api.go +++ b/f3-sidecar/api.go @@ -3,6 +3,7 @@ package main import ( "bufio" "context" + "errors" "os" "github.com/filecoin-project/go-f3" @@ -54,7 +55,7 @@ func (h *F3ServerHandler) F3GetF3PowerTable(ctx context.Context, tsk []byte) (gp return h.f3.GetPowerTable(ctx, tsk) } -func (h *F3ServerHandler) F3ExportLatestSnapshot(ctx context.Context, path string) (*cid.Cid, error) { +func (h *F3ServerHandler) F3ExportLatestSnapshot(ctx context.Context, path string) (_ *cid.Cid, err error) { cs, err := h.f3.GetCertStore() if err != nil { return nil, err @@ -64,10 +65,18 @@ func (h *F3ServerHandler) F3ExportLatestSnapshot(ctx context.Context, path strin if err != nil { return nil, err } - defer f.Close() + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = errors.Join(err, closeErr) + } + }() writer := bufio.NewWriter(f) - defer writer.Flush() + defer func() { + if flushErr := writer.Flush(); flushErr != nil { + err = errors.Join(err, flushErr) + } + }() cid, _, err := cs.ExportLatestSnapshot(ctx, writer) if err != nil { return nil, err diff --git a/f3-sidecar/import.go b/f3-sidecar/import.go index f046e5f88e3d..700485cae699 100644 --- a/f3-sidecar/import.go +++ b/f3-sidecar/import.go @@ -3,6 +3,7 @@ package main import ( "bufio" "context" + "errors" "os" "github.com/filecoin-project/go-f3/certstore" @@ -11,11 +12,14 @@ import ( "github.com/ipfs/go-datastore/namespace" ) -func importSnap(ctx context.Context, rpcEndpoint string, f3Root string, snapshot string) error { +func importSnap(ctx context.Context, rpcEndpoint string, f3Root string, snapshot string) (err error) { logger.Infof("importing F3 snapshot at %s", snapshot) f3api := F3Api{} closer, err := jsonrpc.NewClient(ctx, rpcEndpoint, "F3", &f3api, nil) + if err != nil { + return err + } defer closer() rawNetworkName := waitRawNetworkName(ctx, &f3api) networkName := getNetworkName(rawNetworkName) @@ -30,14 +34,26 @@ func importSnap(ctx context.Context, rpcEndpoint string, f3Root string, snapshot if err != nil { return err } - defer ds.Close() + defer func() { + if closeErr := ds.Close(); closeErr != nil { + err = errors.Join(err, closeErr) + } + }() dsWrapper := namespace.Wrap(ds, m.DatastorePrefix()) - defer dsWrapper.Close() + defer func() { + if closeErr := dsWrapper.Close(); closeErr != nil { + err = errors.Join(err, closeErr) + } + }() f, err := os.Open(snapshot) if err != nil { return err } - defer f.Close() + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = errors.Join(err, closeErr) + } + }() return certstore.ImportSnapshotToDatastore(ctx, bufio.NewReader(f), dsWrapper) } diff --git a/f3-sidecar/run.go b/f3-sidecar/run.go index b6bb52fb9308..af7c8dae7f58 100644 --- a/f3-sidecar/run.go +++ b/f3-sidecar/run.go @@ -16,7 +16,7 @@ import ( "github.com/ipfs/go-cid" ) -func run(ctx context.Context, rpcEndpoint string, jwt string, f3RpcEndpoint string, initialPowerTable string, bootstrapEpoch int64, finality int64, f3Root string) error { +func run(ctx context.Context, rpcEndpoint string, jwt string, f3RpcEndpoint string, initialPowerTable string, bootstrapEpoch int64, finality int64, f3Root string) (err error) { api := FilecoinApi{} isJwtProvided := len(jwt) > 0 closer, err := jsonrpc.NewClient(ctx, rpcEndpoint, "Filecoin", &api, nil) @@ -52,7 +52,11 @@ func run(ctx context.Context, rpcEndpoint string, jwt string, f3RpcEndpoint stri if err != nil { return err } - defer ds.Close() + defer func() { + if closeErr := ds.Close(); closeErr != nil { + err = errors.Join(err, closeErr) + } + }() verif := blssig.VerifierWithKeyOnG1() networkName := getNetworkName(rawNetwork) m := Network2PredefinedManifestMappings[networkName] From 5d4130d1159f50dadaf0a3ebfa41052abe70c553 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 7 Aug 2025 20:51:11 +0800 Subject: [PATCH 30/35] use ProgressBar --- src/tool/subcommands/archive_cmd.rs | 60 +++++++---------------------- 1 file changed, 14 insertions(+), 46 deletions(-) diff --git a/src/tool/subcommands/archive_cmd.rs b/src/tool/subcommands/archive_cmd.rs index d2d6e60d260c..70690f43bd37 100644 --- a/src/tool/subcommands/archive_cmd.rs +++ b/src/tool/subcommands/archive_cmd.rs @@ -54,18 +54,15 @@ use dialoguer::{Confirm, theme::ColorfulTheme}; use futures::{StreamExt as _, TryStreamExt as _}; use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::DAG_CBOR; -use human_repr::HumanCount as _; -use indicatif::ProgressIterator; +use indicatif::{ProgressBar, ProgressIterator, ProgressStyle}; use itertools::Itertools; use multihash_derive::MultihashDigest as _; -use num::Zero as _; use sha2::Sha256; use std::fs::File; -use std::io::{Seek as _, SeekFrom, Write as _}; +use std::io::{Seek as _, SeekFrom}; use std::ops::Range; use std::path::PathBuf; use std::sync::Arc; -use std::time::Instant; use tokio::io::{AsyncWriteExt, BufWriter}; use tracing::info; @@ -575,8 +572,8 @@ async fn do_export( output_path.to_str().unwrap_or_default() ); - let pb = indicatif::ProgressBar::new_spinner().with_style( - indicatif::ProgressStyle::with_template( + let pb = ProgressBar::new_spinner().with_style( + ProgressStyle::with_template( "{spinner} exported {total_bytes} with {binary_bytes_per_sec} in {elapsed}", ) .expect("indicatif template must be valid"), @@ -717,48 +714,19 @@ async fn merge_f3_snapshot(filecoin: PathBuf, f3: PathBuf, output: PathBuf) -> a tempfile::NamedTempFile::new_in(".")? } }; - let mut writer = tokio::io::BufWriter::new(tokio::fs::File::create(&temp_output).await?); - - let handle = tokio::spawn({ - let start = Instant::now(); - let temp_output = temp_output.path().to_owned(); - let output = output.clone(); - async move { - let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); - println!("Getting ready..."); - loop { - interval.tick().await; - let snapshot_size = std::fs::metadata(&temp_output) - .map(|meta| meta.len()) - .unwrap_or(0); - print!( - "{}{}", - anes::MoveCursorToPreviousLine(1), - anes::ClearLine::All - ); - let elapsed_secs = start.elapsed().as_secs_f64(); - println!( - "{}: {} ({}/s)", - output.display(), - snapshot_size.human_count_bytes(), - if elapsed_secs.is_zero() { - 0. - } else { - (snapshot_size as f64) / elapsed_secs - } - .human_count_bytes(), - ); - _ = std::io::stdout().flush(); - } - } - }); - + let writer = tokio::io::BufWriter::new(tokio::fs::File::create(&temp_output).await?); + let pb = ProgressBar::new_spinner().with_style( + ProgressStyle::with_template( + "{spinner} {msg} {binary_total_bytes} written in {elapsed} ({binary_bytes_per_sec})", + ) + .expect("indicatif template must be valid"), + ).with_message(format!("Merging into {} ...", output.display())); + pb.enable_steady_tick(std::time::Duration::from_secs(1)); + let mut writer = pb.wrap_async_write(writer); crate::db::car::forest::Encoder::write(&mut writer, roots, frames).await?; writer.shutdown().await?; temp_output.persist(&output)?; - - handle.abort(); - handle.await?; + pb.finish(); Ok(()) } From 4e2b96dd3d3c9ae19707c4a9e024837ca38ca621 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 7 Aug 2025 22:27:28 +0800 Subject: [PATCH 31/35] resolve comments --- f3-sidecar/ffi_gen.go | 8 ++++---- f3-sidecar/ffi_impl.go | 4 ++-- f3-sidecar/import.go | 6 +++--- f3-sidecar/main.go | 14 +++++++------- f3-sidecar/utils.go | 10 ++++++---- f3-sidecar/utils_test.go | 6 ++++++ src/f3/go_ffi.rs | 2 +- 7 files changed, 29 insertions(+), 21 deletions(-) diff --git a/f3-sidecar/ffi_gen.go b/f3-sidecar/ffi_gen.go index 85f5ffb6f714..aea26d966715 100644 --- a/f3-sidecar/ffi_gen.go +++ b/f3-sidecar/ffi_gen.go @@ -30,7 +30,7 @@ var GoF3NodeImpl GoF3Node type GoF3Node interface { run(rpc_endpoint *string, jwt *string, f3_rpc_endpoint *string, initial_power_table *string, bootstrap_epoch *int64, finality *int64, f3_root *string) bool - import_snap(f3_rpc_endpoint *string, f3_root *string, snapshot *string) string + import_snap(f3_rpc_endpoint *string, f3_root *string, snapshot_path *string) string } //export CGoF3Node_run @@ -51,11 +51,11 @@ func CGoF3Node_run(rpc_endpoint C.StringRef, jwt C.StringRef, f3_rpc_endpoint C. } //export CGoF3Node_import_snap -func CGoF3Node_import_snap(f3_rpc_endpoint C.StringRef, f3_root C.StringRef, snapshot C.StringRef, slot *C.void, cb *C.void) { +func CGoF3Node_import_snap(f3_rpc_endpoint C.StringRef, f3_root C.StringRef, snapshot_path C.StringRef, slot *C.void, cb *C.void) { _new_f3_rpc_endpoint := newString(f3_rpc_endpoint) _new_f3_root := newString(f3_root) - _new_snapshot := newString(snapshot) - resp := GoF3NodeImpl.import_snap(&_new_f3_rpc_endpoint, &_new_f3_root, &_new_snapshot) + _new_snapshot_path := newString(snapshot_path) + resp := GoF3NodeImpl.import_snap(&_new_f3_rpc_endpoint, &_new_f3_root, &_new_snapshot_path) resp_ref, buffer := cvt_ref(cntString, refString)(&resp) asmcall.CallFuncG0P2(unsafe.Pointer(cb), unsafe.Pointer(&resp_ref), unsafe.Pointer(slot)) runtime.KeepAlive(resp_ref) diff --git a/f3-sidecar/ffi_impl.go b/f3-sidecar/ffi_impl.go index 5d93b95d9542..6bfb7f472e2f 100644 --- a/f3-sidecar/ffi_impl.go +++ b/f3-sidecar/ffi_impl.go @@ -48,8 +48,8 @@ func (f3 *f3Impl) run(rpc_endpoint *string, jwt *string, f3_rpc_endpoint *string return err == nil } -func (f3 *f3Impl) import_snap(f3_rpc_endpoint *string, f3_root *string, snapshot *string) string { - if err := importSnap(f3.ctx, *f3_rpc_endpoint, *f3_root, *snapshot); err != nil { +func (f3 *f3Impl) import_snap(f3_rpc_endpoint *string, f3_root *string, snapshot_path *string) string { + if err := importSnap(f3.ctx, *f3_rpc_endpoint, *f3_root, *snapshot_path); err != nil { return err.Error() } return "" diff --git a/f3-sidecar/import.go b/f3-sidecar/import.go index 700485cae699..7acbbe0fdbb0 100644 --- a/f3-sidecar/import.go +++ b/f3-sidecar/import.go @@ -12,8 +12,8 @@ import ( "github.com/ipfs/go-datastore/namespace" ) -func importSnap(ctx context.Context, rpcEndpoint string, f3Root string, snapshot string) (err error) { - logger.Infof("importing F3 snapshot at %s", snapshot) +func importSnap(ctx context.Context, rpcEndpoint string, f3Root string, snapshotPath string) (err error) { + logger.Infof("importing F3 snapshot at %s", snapshotPath) f3api := F3Api{} closer, err := jsonrpc.NewClient(ctx, rpcEndpoint, "F3", &f3api, nil) @@ -46,7 +46,7 @@ func importSnap(ctx context.Context, rpcEndpoint string, f3Root string, snapshot } }() - f, err := os.Open(snapshot) + f, err := os.Open(snapshotPath) if err != nil { return err } diff --git a/f3-sidecar/main.go b/f3-sidecar/main.go index 97db86dfbf0d..d94f71d50e70 100644 --- a/f3-sidecar/main.go +++ b/f3-sidecar/main.go @@ -39,22 +39,22 @@ func main() { flag.Int64Var(&bootstrapEpoch, "bootstrap", -1, "F3 bootstrap epoch") var finality int64 flag.Int64Var(&finality, "finality", 900, "chain finality epochs") - var root string - flag.StringVar(&root, "root", "f3-data", "path to the f3 data directory") - var snapshot string - flag.StringVar(&snapshot, "snapshot", "", "path to the f3 snapshot file") + var f3Root string + flag.StringVar(&f3Root, "root", "f3-data", "path to the f3 data directory") + var snapshotPath string + flag.StringVar(&snapshotPath, "snapshot", "", "path to the f3 snapshot file") flag.Parse() ctx := context.Background() - if len(snapshot) > 0 { - if err := importSnap(ctx, rpcEndpoint, root, snapshot); err != nil { + if len(snapshotPath) > 0 { + if err := importSnap(ctx, rpcEndpoint, f3Root, snapshotPath); err != nil { panic(err) } } - err := run(ctx, rpcEndpoint, jwt, f3RpcEndpoint, initialPowerTable, bootstrapEpoch, finality, root) + err := run(ctx, rpcEndpoint, jwt, f3RpcEndpoint, initialPowerTable, bootstrapEpoch, finality, f3Root) if err != nil { panic(err) } diff --git a/f3-sidecar/utils.go b/f3-sidecar/utils.go index 5746f70eb8f8..7d89126bd134 100644 --- a/f3-sidecar/utils.go +++ b/f3-sidecar/utils.go @@ -23,18 +23,20 @@ func getDatastore(f3Root string) (*leveldb.Datastore, error) { func waitRawNetworkName(ctx context.Context, f3api *F3Api) string { for { rawNetwork, err := f3api.GetRawNetworkName(ctx) - if err == nil { - logger.Infoln("Forest RPC server is online") - return rawNetwork - } else { + if err != nil { logger.Warnln("waiting for Forest RPC server") time.Sleep(5 * time.Second) + continue } + + logger.Infoln("Forest RPC server is online") + return rawNetwork } } func getNetworkName(rawNetworkName string) gpbft.NetworkName { networkName := gpbft.NetworkName(rawNetworkName) + // See // Use "filecoin" as the network name on mainnet, otherwise use the network name. Yes, // mainnet is called testnetnet in state. if networkName == "testnetnet" { diff --git a/f3-sidecar/utils_test.go b/f3-sidecar/utils_test.go index cfd31fb0fd15..bb218dcf9c82 100644 --- a/f3-sidecar/utils_test.go +++ b/f3-sidecar/utils_test.go @@ -1,6 +1,7 @@ package main import ( + "os" "testing" "github.com/ipfs/go-cid" @@ -13,3 +14,8 @@ func TestIsCidDefined(t *testing.T) { require.False(t, isCidDefined(CID_UNDEF_RUST)) require.True(t, isCidDefined(cid.MustParse("bafy2bzaceac6jbaeolcsbh7rawcshcvh2cokvxrgsh4sxg5yu34i5xllbfpw4"))) } + +func TestOsOpen(t *testing.T) { + _, err := os.Open("/var/tmp/dummy") + require.NoError(t, err) +} diff --git a/src/f3/go_ffi.rs b/src/f3/go_ffi.rs index 008053ce44bf..144e162816e8 100644 --- a/src/f3/go_ffi.rs +++ b/src/f3/go_ffi.rs @@ -19,5 +19,5 @@ pub trait GoF3Node { f3_root: String, ) -> bool; - fn import_snap(f3_rpc_endpoint: String, f3_root: String, snapshot: String) -> String; + fn import_snap(f3_rpc_endpoint: String, f3_root: String, snapshot_path: String) -> String; } From fefc3f297e8e3282b8a7ce982f3131351085d927 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 8 Aug 2025 00:02:04 +0800 Subject: [PATCH 32/35] update CI umbrella job --- .github/workflows/forest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/forest.yml b/.github/workflows/forest.yml index 95a70077973d..fb8f4a18a6e6 100644 --- a/.github/workflows/forest.yml +++ b/.github/workflows/forest.yml @@ -576,6 +576,7 @@ jobs: - state-migrations-check - calibnet-wallet-check - calibnet-export-check + - calibnet-export-check-v2 - calibnet-no-discovery-checks - calibnet-kademlia-checks - calibnet-eth-mapping-check From 86bc8558e387f6c1fbffc3d075e92a36ecbc312e Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 11 Aug 2025 18:36:35 +0800 Subject: [PATCH 33/35] Update src/tool/subcommands/archive_cmd.rs Co-authored-by: Hubert --- src/tool/subcommands/archive_cmd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tool/subcommands/archive_cmd.rs b/src/tool/subcommands/archive_cmd.rs index 70690f43bd37..a50d9b7c2932 100644 --- a/src/tool/subcommands/archive_cmd.rs +++ b/src/tool/subcommands/archive_cmd.rs @@ -153,7 +153,7 @@ pub enum ArchiveCommands { /// Path to the F3 snapshot #[arg(long)] f3: PathBuf, - /// Snapshot output filename + /// Snapshot output file path #[arg(long)] output: PathBuf, }, From 52433837d56327f305f38a84b79e6223142a5dd9 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 11 Aug 2025 19:31:08 +0800 Subject: [PATCH 34/35] resolve comments --- scripts/tests/harness.sh | 2 +- src/cli_shared/cli/client.rs | 4 +- src/daemon/db_util.rs | 12 +++--- src/daemon/mod.rs | 20 +++++----- src/f3/mod.rs | 58 +++++++++++++++-------------- src/f3/snapshot.rs | 1 + src/tool/subcommands/archive_cmd.rs | 12 +++--- 7 files changed, 59 insertions(+), 50 deletions(-) diff --git a/scripts/tests/harness.sh b/scripts/tests/harness.sh index 132819a0b158..f861daecfc89 100644 --- a/scripts/tests/harness.sh +++ b/scripts/tests/harness.sh @@ -33,7 +33,7 @@ function forest_download_and_import_snapshot_with_f3 { echo "Downloading F3 snapshot" aria2c -x5 https://forest-snapshots.fra1.cdn.digitaloceanspaces.com/f3/f3_snap_calibnet_552628.bin -o f3.bin echo "Generating v2 snapshot" - $FOREST_TOOL_PATH archive merge-f3 --filecoin v1.forest.car.zst --f3 f3.bin --output v2.forest.car.zst + $FOREST_TOOL_PATH archive merge-f3 --v1 v1.forest.car.zst --f3 f3.bin --output v2.forest.car.zst echo "Inspecting archive info" $FOREST_TOOL_PATH archive info v2.forest.car.zst echo "Inspecting archive metadata" diff --git a/src/cli_shared/cli/client.rs b/src/cli_shared/cli/client.rs index b856609d47ff..a00f50287396 100644 --- a/src/cli_shared/cli/client.rs +++ b/src/cli_shared/cli/client.rs @@ -103,7 +103,9 @@ impl Client { self.data_dir.join("token") } - pub fn rpc_v1_endpoint(&self) -> String { + pub fn rpc_v1_endpoint(&self) -> Result { format!("http://{}/rpc/v1", self.rpc_address) + .as_str() + .parse() } } diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index d1c3a72c16d3..f51f4805d732 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -131,8 +131,8 @@ pub async fn import_chain_as_forest_car( from_path: &Path, forest_car_db_dir: &Path, import_mode: ImportMode, - rpc_endpoint: String, - f3_root: String, + rpc_endpoint: Url, + f3_root: &Path, chain_config: &ChainConfig, snapshot_progress_tracker: &SnapshotProgressTracker, ) -> anyhow::Result<(PathBuf, Tipset)> { @@ -249,8 +249,8 @@ pub async fn import_chain_as_forest_car( } if let Err(e) = crate::f3::import_f3_snapshot( chain_config, - rpc_endpoint, - f3_root, + rpc_endpoint.to_string(), + f3_root.display().to_string(), temp_f3_snap.path().display().to_string(), ) { // Do not make it a hard error if anything is wrong with F3 snapshot @@ -485,8 +485,8 @@ mod test { file_path, temp_db_dir.path(), import_mode, - "test".into(), - "test".into(), + "test".parse().unwrap(), + Path::new("test"), &ChainConfig::devnet(), &SnapshotProgressTracker::default(), ) diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index e9803154cf17..7d279dc400a3 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -140,8 +140,8 @@ async fn maybe_import_snapshot( path, &ctx.db_meta_data.get_forest_car_db_dir(), config.client.import_mode, - config.client.rpc_v1_endpoint(), - crate::f3::get_f3_root(config), + config.client.rpc_v1_endpoint()?, + &crate::f3::get_f3_root(config), ctx.chain_config(), &snapshot_tracker, ) @@ -395,21 +395,21 @@ fn maybe_start_rpc_service( Ok(()) } -fn maybe_start_f3_service(opts: &CliOpts, config: &Config, ctx: &AppContext) { +fn maybe_start_f3_service(opts: &CliOpts, config: &Config, ctx: &AppContext) -> anyhow::Result<()> { // already running if crate::rpc::f3::F3_LEASE_MANAGER.get().is_some() { - return; + return Ok(()); } if !config.client.enable_rpc { if crate::f3::is_sidecar_ffi_enabled(ctx.state_manager.chain_config()) { tracing::warn!("F3 sidecar is enabled but not run because RPC is disabled. ") } - return; + return Ok(()); } if !opts.halt_after_import && !opts.stateless { - let rpc_endpoint = config.client.rpc_v1_endpoint(); + let rpc_endpoint = config.client.rpc_v1_endpoint()?; let state_manager = &ctx.state_manager; let p2p_peer_id = ctx.p2p_peer_id; let admin_jwt = ctx.admin_jwt.clone(); @@ -430,7 +430,7 @@ fn maybe_start_f3_service(opts: &CliOpts, config: &Config, ctx: &AppContext) { move || { crate::f3::run_f3_sidecar_if_enabled( &chain_config, - rpc_endpoint, + rpc_endpoint.to_string(), admin_jwt, crate::rpc::f3::get_f3_rpc_endpoint().to_string(), initial_power_table @@ -438,11 +438,13 @@ fn maybe_start_f3_service(opts: &CliOpts, config: &Config, ctx: &AppContext) { .unwrap_or_default(), bootstrap_epoch, chain_finality, - f3_root, + f3_root.display().to_string(), ); } }); } + + Ok(()) } fn maybe_start_indexer_service( @@ -576,7 +578,7 @@ pub(super) async fn start_services( on_app_context_and_db_initialized(&ctx); ctx.state_manager.populate_cache(); maybe_start_metrics_service(&mut services, &config, &ctx).await?; - maybe_start_f3_service(opts, &config, &ctx); + maybe_start_f3_service(opts, &config, &ctx)?; maybe_start_health_check_service(&mut services, &config, &p2p_service, &chain_follower, &ctx) .await?; maybe_start_indexer_service(&mut services, opts, &config, &ctx); diff --git a/src/f3/mod.rs b/src/f3/mod.rs index afc915158fa8..003fbb21016d 100644 --- a/src/f3/mod.rs +++ b/src/f3/mod.rs @@ -5,6 +5,8 @@ #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] mod go_ffi; +use std::path::{Path, PathBuf}; + #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] use go_ffi::*; @@ -21,15 +23,15 @@ pub struct F3Options { pub initial_power_table: Option, } -pub fn get_f3_root(config: &crate::Config) -> String { - std::env::var("FOREST_F3_ROOT").unwrap_or_else(|_| { - config - .client - .data_dir - .join(format!("f3/{}", config.chain())) - .display() - .to_string() - }) +pub fn get_f3_root(config: &crate::Config) -> PathBuf { + std::env::var("FOREST_F3_ROOT") + .map(|p| Path::new(&p).to_path_buf()) + .unwrap_or_else(|_| { + config + .client + .data_dir + .join(format!("f3/{}", config.chain())) + }) } pub fn get_f3_sidecar_params(chain_config: &ChainConfig) -> F3Options { @@ -86,45 +88,47 @@ pub fn get_f3_sidecar_params(chain_config: &ChainConfig) -> F3Options { } } +#[allow(unused_variables)] pub fn run_f3_sidecar_if_enabled( chain_config: &ChainConfig, - _rpc_endpoint: String, - _jwt: String, - _f3_rpc_endpoint: String, - _initial_power_table: String, - _bootstrap_epoch: i64, - _finality: i64, - _f3_root: String, + rpc_endpoint: String, + jwt: String, + f3_rpc_endpoint: String, + initial_power_table: String, + bootstrap_epoch: i64, + finality: i64, + f3_root: String, ) { if is_sidecar_ffi_enabled(chain_config) { #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] { tracing::info!("Starting F3 sidecar service ..."); GoF3NodeImpl::run( - _rpc_endpoint, - _jwt, - _f3_rpc_endpoint, - _initial_power_table, - _bootstrap_epoch, - _finality, - _f3_root, + rpc_endpoint, + jwt, + f3_rpc_endpoint, + initial_power_table, + bootstrap_epoch, + finality, + f3_root, ); } } } +#[allow(unused_variables)] pub fn import_f3_snapshot( chain_config: &ChainConfig, - _rpc_endpoint: String, - _f3_root: String, - _snapshot: String, + rpc_endpoint: String, + f3_root: String, + snapshot: String, ) -> anyhow::Result<()> { if is_sidecar_ffi_enabled(chain_config) { #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] { let sw = std::time::Instant::now(); tracing::info!("Importing F3 snapshot ..."); - let err = GoF3NodeImpl::import_snap(_rpc_endpoint, _f3_root, _snapshot); + let err = GoF3NodeImpl::import_snap(rpc_endpoint, f3_root, snapshot); if !err.is_empty() { anyhow::bail!("{err}"); } diff --git a/src/f3/snapshot.rs b/src/f3/snapshot.rs index 37aaaec4760e..2b8a6d597500 100644 --- a/src/f3/snapshot.rs +++ b/src/f3/snapshot.rs @@ -17,6 +17,7 @@ pub fn get_f3_snapshot_cid(f3_data: &mut impl Read) -> anyhow::Result { )) } +/// Defined in #[derive(Debug, Clone, Eq, PartialEq, Serialize_tuple, Deserialize_tuple)] pub struct F3SnapshotHeader { pub version: u64, diff --git a/src/tool/subcommands/archive_cmd.rs b/src/tool/subcommands/archive_cmd.rs index a50d9b7c2932..0da18f47cf95 100644 --- a/src/tool/subcommands/archive_cmd.rs +++ b/src/tool/subcommands/archive_cmd.rs @@ -147,13 +147,13 @@ pub enum ArchiveCommands { }, /// Merge a v1 Filecoin snapshot with an F3 snapshot into a v2 Filecoin snapshot in `.forest.car.zst` format MergeF3 { - /// Path to the Filecoin snapshot - #[arg(long)] - filecoin: PathBuf, + /// Path to the v1 Filecoin snapshot + #[arg(long = "v1")] + filecoin_v1: PathBuf, /// Path to the F3 snapshot #[arg(long)] f3: PathBuf, - /// Snapshot output file path + /// Path to the snapshot output file in `.forest.car.zst` format #[arg(long)] output: PathBuf, }, @@ -263,10 +263,10 @@ impl ArchiveCommands { force, } => merge_snapshots(snapshot_files, output_path, force).await, Self::MergeF3 { - filecoin, + filecoin_v1, f3, output, - } => merge_f3_snapshot(filecoin, f3, output).await, + } => merge_f3_snapshot(filecoin_v1, f3, output).await, Self::Diff { snapshot_files, epoch, From 196c88461bd1a3c4795cac3ae3fd86d8b20c87a7 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Mon, 11 Aug 2025 19:49:14 +0800 Subject: [PATCH 35/35] fix rpc endpoint in tests --- src/daemon/db_util.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daemon/db_util.rs b/src/daemon/db_util.rs index f51f4805d732..2a8553d1fc98 100644 --- a/src/daemon/db_util.rs +++ b/src/daemon/db_util.rs @@ -485,7 +485,7 @@ mod test { file_path, temp_db_dir.path(), import_mode, - "test".parse().unwrap(), + "http://127.0.0.1:2345/rpc/v1".parse().unwrap(), Path::new("test"), &ChainConfig::devnet(), &SnapshotProgressTracker::default(),