feat: integrate with go-f3 and implement F3 data export and import

.github/workflows/forest.yml

@@ -287,7 +287,7 @@ jobs:
   calibnet-export-check-v2:
     needs:
       - build-ubuntu
-    name: Snapshot export checks v2
+    name: Snapshot export checks v2 with F3 data
     runs-on: ubuntu-24.04
     steps:
       - run: lscpu
@@ -304,7 +304,7 @@ jobs:
         run: |
           chmod +x ~/.cargo/bin/forest*
       - name: Snapshot export check v2
-        run: ./scripts/tests/calibnet_export_check.sh v2
+        run: ./scripts/tests/calibnet_export_f3_check.sh
         timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}
   calibnet-no-discovery-checks:
     needs:
@@ -576,6 +576,7 @@ jobs:
       - state-migrations-check
       - calibnet-wallet-check
       - calibnet-export-check
+      - calibnet-export-check-v2
       - calibnet-no-discovery-checks
       - calibnet-kademlia-checks
       - calibnet-eth-mapping-check

.github/workflows/go-lint.yml

@@ -0,0 +1,29 @@
+name: Go code linters
+permissions:
+  contents: read
+
+# Cancel workflow if there is a new change to the branch.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+on:
+  workflow_dispatch:
+  merge_group:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  lint-go:
+    name: Go lint checks
+    runs-on: ubuntu-24.04-arm
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: "go.work"
+      - run: make lint-go

.golangci.yml

@@ -0,0 +1,3 @@
+version: "2"
+linters:
+  default: standard

CHANGELOG.md

@@ -35,10 +35,14 @@
 
 - [#5859](https://github.com/ChainSafe/forest/pull/5859) Added size metrics for zstd frame cache and made max size configurable via `FOREST_ZSTD_FRAME_CACHE_DEFAULT_MAX_SIZE` environment variable.
 
+- [#5886](https://github.com/ChainSafe/forest/issues/5886) Add `forest-tool archive merge-f3` subcommand for merging a v1 Filecoin snapshot and an F3 snapshot into a v2 Filecoin snapshot.
+
 - [#4976](https://github.com/ChainSafe/forest/issues/4976) Add support for the `Filecoin.EthSubscribe` and `Filecoin.EthUnsubscribe` API methods to enable subscriptions to Ethereum event types: `heads` and `logs`.
 
 ### Changed
 
+- [#5886](https://github.com/ChainSafe/forest/issues/5886) Updated `forest-tool archive metadata` to print F3 snapshot header info when applicable.
+
 - [#5869](https://github.com/ChainSafe/forest/pull/5869) Updated `forest-cli snapshot export` to print average speed.
 
 ### Removed

Makefile

@@ -1,3 +1,5 @@
+GOLANGCI_LINT_VERSION=v2.3.1
+
 install:
 	cargo install --locked --path . --force
 
@@ -77,6 +79,9 @@ DOCKERFILES=$(wildcard Dockerfile*)
 lint-docker: $(DOCKERFILES)
 	docker run --rm -i hadolint/hadolint < $<
 
+lint-go:
+	go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) run --timeout 10m --concurrency 4 ./f3-sidecar ./interop-tests/src/tests/go_app
+
 # Formats Rust, TOML and Markdown files.
 fmt:
 	cargo fmt --all

docs/docs/users/reference/cli.sh

@@ -119,7 +119,9 @@ generate_markdown_section "forest-tool" "archive"
 generate_markdown_section "forest-tool" "archive info"
 generate_markdown_section "forest-tool" "archive export"
 generate_markdown_section "forest-tool" "archive checkpoints"
+generate_markdown_section "forest-tool" "archive metadata"
 generate_markdown_section "forest-tool" "archive merge"
+generate_markdown_section "forest-tool" "archive merge-f3"
 generate_markdown_section "forest-tool" "archive diff"
 generate_markdown_section "forest-tool" "archive sync-bucket"
 

f3-sidecar/api.go

@@ -1,13 +1,17 @@
 package main
 
 import (
+	"bufio"
 	"context"
+	"errors"
+	"os"
 
 	"github.com/filecoin-project/go-f3"
 	"github.com/filecoin-project/go-f3/certs"
 	"github.com/filecoin-project/go-f3/gpbft"
 	"github.com/filecoin-project/go-f3/manifest"
 	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/ipfs/go-cid"
 	"github.com/libp2p/go-libp2p/core/peer"
 )
 
@@ -51,6 +55,35 @@ func (h *F3ServerHandler) F3GetF3PowerTable(ctx context.Context, tsk []byte) (gp
 	return h.f3.GetPowerTable(ctx, tsk)
 }
 
+func (h *F3ServerHandler) F3ExportLatestSnapshot(ctx context.Context, path string) (_ *cid.Cid, err error) {
+	cs, err := h.f3.GetCertStore()
+	if err != nil {
+		return nil, err
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if closeErr := f.Close(); closeErr != nil {
+			err = errors.Join(err, closeErr)
+		}
+	}()
+
+	writer := bufio.NewWriter(f)
+	defer func() {
+		if flushErr := writer.Flush(); flushErr != nil {
+			err = errors.Join(err, flushErr)
+		}
+	}()
+	cid, _, err := cs.ExportLatestSnapshot(ctx, writer)
+	if err != nil {
+		return nil, err
+	}
+	return &cid, nil
+}
+
 // F3GetF3PowerTableByInstance retrieves the power table for a specific consensus instance.
 // It returns the power entries associated with the given instance number.
 //

f3-sidecar/ffi_gen.go

@@ -30,6 +30,7 @@ var GoF3NodeImpl GoF3Node
 
 type GoF3Node interface {
 	run(rpc_endpoint *string, jwt *string, f3_rpc_endpoint *string, initial_power_table *string, bootstrap_epoch *int64, finality *int64, f3_root *string) bool
+	import_snap(f3_rpc_endpoint *string, f3_root *string, snapshot_path *string) string
 }
 
 //export CGoF3Node_run
@@ -49,6 +50,19 @@ func CGoF3Node_run(rpc_endpoint C.StringRef, jwt C.StringRef, f3_rpc_endpoint C.
 	runtime.KeepAlive(buffer)
 }
 
+//export CGoF3Node_import_snap
+func CGoF3Node_import_snap(f3_rpc_endpoint C.StringRef, f3_root C.StringRef, snapshot_path C.StringRef, slot *C.void, cb *C.void) {
+	_new_f3_rpc_endpoint := newString(f3_rpc_endpoint)
+	_new_f3_root := newString(f3_root)
+	_new_snapshot_path := newString(snapshot_path)
+	resp := GoF3NodeImpl.import_snap(&_new_f3_rpc_endpoint, &_new_f3_root, &_new_snapshot_path)
+	resp_ref, buffer := cvt_ref(cntString, refString)(&resp)
+	asmcall.CallFuncG0P2(unsafe.Pointer(cb), unsafe.Pointer(&resp_ref), unsafe.Pointer(slot))
+	runtime.KeepAlive(resp_ref)
+	runtime.KeepAlive(resp)
+	runtime.KeepAlive(buffer)
+}
+
 func newString(s_ref C.StringRef) string {
 	return unsafe.String((*byte)(unsafe.Pointer(s_ref.ptr)), s_ref.len)
 }

f3-sidecar/ffi_impl.go

@@ -48,6 +48,13 @@ func (f3 *f3Impl) run(rpc_endpoint *string, jwt *string, f3_rpc_endpoint *string
 	return err == nil
 }
 
+func (f3 *f3Impl) import_snap(f3_rpc_endpoint *string, f3_root *string, snapshot_path *string) string {
+	if err := importSnap(f3.ctx, *f3_rpc_endpoint, *f3_root, *snapshot_path); err != nil {
+		return err.Error()
+	}
+	return ""
+}
+
 func checkError(err error) {
 	if err != nil {
 		panic(err)
@@ -57,5 +64,6 @@ func checkError(err error) {
 // To avoid potential panics
 // See <https://github.com/ChainSafe/forest/pull/4636#issuecomment-2306500753>
 func setGoDebugEnv() {
-	os.Setenv("GODEBUG", "invalidptr=0,cgocheck=0")
+	err := os.Setenv("GODEBUG", "invalidptr=0,cgocheck=0")
+	checkError(err)
 }

f3-sidecar/go.mod

@@ -3,11 +3,12 @@ module f3-sidecar/v2
 go 1.24.5
 
 require (
-	github.com/filecoin-project/go-f3 v0.8.9
+	github.com/filecoin-project/go-f3 v0.8.10-0.20250801124500-9288fba86c47
 	github.com/filecoin-project/go-jsonrpc v0.8.0
 	github.com/filecoin-project/go-state-types v0.16.0
 	github.com/ihciah/rust2go v0.0.0-20250726175549-557d7a3a4e27
 	github.com/ipfs/go-cid v0.5.0
+	github.com/ipfs/go-datastore v0.8.2
 	github.com/ipfs/go-ds-leveldb v0.5.2
 	github.com/ipfs/go-log/v2 v2.8.0
 	github.com/libp2p/go-libp2p v0.42.1
@@ -44,7 +45,6 @@ require (
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/huin/goupnp v1.3.0 // indirect
 	github.com/ipfs/boxo v0.33.0 // indirect
-	github.com/ipfs/go-datastore v0.8.2 // indirect
 	github.com/ipld/go-ipld-prime v0.21.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect

f3-sidecar/go.sum

@@ -45,8 +45,8 @@ github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW
 github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
 github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU=
 github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs=
-github.com/filecoin-project/go-f3 v0.8.9 h1:0SHqwWmcVAL02Or7uE4P7qG1feopyVBSlgrUxkHkQBM=
-github.com/filecoin-project/go-f3 v0.8.9/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI=
+github.com/filecoin-project/go-f3 v0.8.10-0.20250801124500-9288fba86c47 h1:qf5j5kHyTfwivaXjXlySjS5Nax6wufrUiomE4elcW3s=
+github.com/filecoin-project/go-f3 v0.8.10-0.20250801124500-9288fba86c47/go.mod h1:hFvb2CMxHDmlJAVzfiIL/V8zCtNMQqfSnhP5TyM6CHI=
 github.com/filecoin-project/go-jsonrpc v0.8.0 h1:2yqlN3Vd8Gx5UtA3fib7tQu2aW1cSOJt253LEBWExo4=
 github.com/filecoin-project/go-jsonrpc v0.8.0/go.mod h1:p8WGOwQGYbFugSdK7qKIGhhb1VVcQ2rtBLdEiik1QWI=
 github.com/filecoin-project/go-state-types v0.16.0 h1:ajIREDzTGfq71ofIQ29iZR1WXxmkvd2nQNc6ApcP1wI=

f3-sidecar/import.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"os"
+
+	"github.com/filecoin-project/go-f3/certstore"
+	"github.com/filecoin-project/go-f3/manifest"
+	"github.com/filecoin-project/go-jsonrpc"
+	"github.com/ipfs/go-datastore/namespace"
+)
+
+func importSnap(ctx context.Context, rpcEndpoint string, f3Root string, snapshotPath string) (err error) {
+	logger.Infof("importing F3 snapshot at %s", snapshotPath)
+
+	f3api := F3Api{}
+	closer, err := jsonrpc.NewClient(ctx, rpcEndpoint, "F3", &f3api, nil)
+	if err != nil {
+		return err
+	}
+	defer closer()
+	rawNetworkName := waitRawNetworkName(ctx, &f3api)
+	networkName := getNetworkName(rawNetworkName)
+	m := Network2PredefinedManifestMappings[networkName]
+	if m == nil {
+		m2 := manifest.LocalDevnetManifest()
+		m = &m2
+		m.NetworkName = networkName
+	}
+
+	ds, err := getDatastore(f3Root)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if closeErr := ds.Close(); closeErr != nil {
+			err = errors.Join(err, closeErr)
+		}
+	}()
+	dsWrapper := namespace.Wrap(ds, m.DatastorePrefix())
+	defer func() {
+		if closeErr := dsWrapper.Close(); closeErr != nil {
+			err = errors.Join(err, closeErr)
+		}
+	}()
+
+	f, err := os.Open(snapshotPath)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if closeErr := f.Close(); closeErr != nil {
+			err = errors.Join(err, closeErr)
+		}
+	}()
+	return certstore.ImportSnapshotToDatastore(ctx, bufio.NewReader(f), dsWrapper)
+}

f3-sidecar/main.go

@@ -39,14 +39,22 @@ func main() {
 	flag.Int64Var(&bootstrapEpoch, "bootstrap", -1, "F3 bootstrap epoch")
 	var finality int64
 	flag.Int64Var(&finality, "finality", 900, "chain finality epochs")
-	var root string
-	flag.StringVar(&root, "root", "f3-data", "path to the f3 data directory")
+	var f3Root string
+	flag.StringVar(&f3Root, "root", "f3-data", "path to the f3 data directory")
+	var snapshotPath string
+	flag.StringVar(&snapshotPath, "snapshot", "", "path to the f3 snapshot file")
 
 	flag.Parse()
 
 	ctx := context.Background()
 
-	err := run(ctx, rpcEndpoint, jwt, f3RpcEndpoint, initialPowerTable, bootstrapEpoch, finality, root)
+	if len(snapshotPath) > 0 {
+		if err := importSnap(ctx, rpcEndpoint, f3Root, snapshotPath); err != nil {
+			panic(err)
+		}
+	}
+
+	err := run(ctx, rpcEndpoint, jwt, f3RpcEndpoint, initialPowerTable, bootstrapEpoch, finality, f3Root)
 	if err != nil {
 		panic(err)
 	}

f3-sidecar/run.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"net"
 	"net/http"
-	"path/filepath"
 	"time"
 
 	"github.com/filecoin-project/go-f3"
@@ -15,10 +14,9 @@ import (
 	"github.com/filecoin-project/go-f3/manifest"
 	"github.com/filecoin-project/go-jsonrpc"
 	"github.com/ipfs/go-cid"
-	leveldb "github.com/ipfs/go-ds-leveldb"
 )
 
-func run(ctx context.Context, rpcEndpoint string, jwt string, f3RpcEndpoint string, initialPowerTable string, bootstrapEpoch int64, finality int64, f3Root string) error {
+func run(ctx context.Context, rpcEndpoint string, jwt string, f3RpcEndpoint string, initialPowerTable string, bootstrapEpoch int64, finality int64, f3Root string) (err error) {
 	api := FilecoinApi{}
 	isJwtProvided := len(jwt) > 0
 	closer, err := jsonrpc.NewClient(ctx, rpcEndpoint, "Filecoin", &api, nil)
@@ -32,17 +30,7 @@ func run(ctx context.Context, rpcEndpoint string, jwt string, f3RpcEndpoint stri
 		return err
 	}
 
-	var rawNetwork string
-	for {
-		rawNetwork, err = ec.f3api.GetRawNetworkName(ctx)
-		if err == nil {
-			logger.Infoln("Forest RPC server is online")
-			break
-		} else {
-			logger.Warnln("waiting for Forest RPC server")
-			time.Sleep(5 * time.Second)
-		}
-	}
+	rawNetwork := waitRawNetworkName(ctx, &ec.f3api)
 	listenAddrs, err := api.NetAddrsListen(ctx)
 	if err != nil {
 		return err
@@ -60,17 +48,17 @@ func run(ctx context.Context, rpcEndpoint string, jwt string, f3RpcEndpoint stri
 	if err != nil {
 		return err
 	}
-	ds, err := leveldb.NewDatastore(filepath.Join(f3Root, "db"), nil)
+	ds, err := getDatastore(f3Root)
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if closeErr := ds.Close(); closeErr != nil {
+			err = errors.Join(err, closeErr)
+		}
+	}()
 	verif := blssig.VerifierWithKeyOnG1()
-	networkName := gpbft.NetworkName(rawNetwork)
-	// Use "filecoin" as the network name on mainnet, otherwise use the network name. Yes,
-	// mainnet is called testnetnet in state.
-	if networkName == "testnetnet" {
-		networkName = "filecoin"
-	}
+	networkName := getNetworkName(rawNetwork)
 	m := Network2PredefinedManifestMappings[networkName]
 	if m == nil {
 		m2 := manifest.LocalDevnetManifest()