Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion core/node/groups.go
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,9 @@ func Storage(bcfg *BuildCfg, cfg *config.Config) fx.Option {

finalBstore := fx.Provide(GcBlockstoreCtor)
if cfg.Experimental.FilestoreEnabled || cfg.Experimental.UrlstoreEnabled {
finalBstore = fx.Provide(FilestoreBlockstoreCtor)
finalBstore = fx.Provide(FilestoreBlockstoreCtor(
cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy),
))
}

return fx.Options(
Expand Down
28 changes: 20 additions & 8 deletions core/node/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"go.uber.org/fx"

"github.com/ipfs/boxo/filestore"
"github.com/ipfs/boxo/provider"
"github.com/ipfs/kubo/core/node/helpers"
"github.com/ipfs/kubo/repo"
"github.com/ipfs/kubo/thirdparty/verifbs"
Expand Down Expand Up @@ -77,14 +78,25 @@ func GcBlockstoreCtor(bb BaseBlocks) (gclocker blockstore.GCLocker, gcbs blockst
}

// FilestoreBlockstoreCtor wraps GcBlockstore and adds Filestore support
func FilestoreBlockstoreCtor(repo repo.Repo, bb BaseBlocks, prov DHTProvider) (gclocker blockstore.GCLocker, gcbs blockstore.GCBlockstore, bs blockstore.Blockstore, fstore *filestore.Filestore) {
gclocker = blockstore.NewGCLocker()
func FilestoreBlockstoreCtor(
providingStrategy string,
) func(repo repo.Repo, bb BaseBlocks, prov DHTProvider) (gclocker blockstore.GCLocker, gcbs blockstore.GCBlockstore, bs blockstore.Blockstore, fstore *filestore.Filestore) {
return func(repo repo.Repo, bb BaseBlocks, prov DHTProvider) (gclocker blockstore.GCLocker, gcbs blockstore.GCBlockstore, bs blockstore.Blockstore, fstore *filestore.Filestore) {
gclocker = blockstore.NewGCLocker()

// hash security
fstore = filestore.NewFilestore(bb, repo.FileManager(), prov)
gcbs = blockstore.NewGCBlockstore(fstore, gclocker)
gcbs = &verifbs.VerifBSGC{GCBlockstore: gcbs}
var fstoreProv provider.MultihashProvider
strategyFlag := config.MustParseProvideStrategy(providingStrategy)
if strategyFlag&config.ProvideStrategyAll != 0 {
fstoreProv = prov
}

bs = gcbs
return
fstore = filestore.NewFilestore(bb, repo.FileManager(), fstoreProv)

// hash security
gcbs = blockstore.NewGCBlockstore(fstore, gclocker)
gcbs = &verifbs.VerifBSGC{GCBlockstore: gcbs}

bs = gcbs
return
}
}
5 changes: 5 additions & 0 deletions docs/changelogs/v0.41.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team.
- [📌 `pin add` and `pin update` now fast-provide root CID](#-pin-add-and-pin-update-now-fast-provide-root-cid)
- [🌳 New `--fast-provide-dag` flag for fine-tuned provide control](#-new---fast-provide-dag-flag-for-fine-tuned-provide-control)
- [🛡️ Hardened `Provide.Strategy` parsing](#-hardened-providestrategy-parsing)
- [🔧 Filestore now respects `Provide.Strategy`](#-filestore-now-respects-providestrategy)
- [🛡️ `ipfs object patch` validates UnixFS node types](#-ipfs-object-patch-validates-unixfs-node-types)
- [🔗 MFS: fixed CidBuilder preservation](#-mfs-fixed-cidbuilder-preservation)
- [📂 FUSE Mount Improvements](#-fuse-mount-improvements)
Expand Down Expand Up @@ -120,6 +121,10 @@ Pass `--fast-provide-dag=true` (or set [`Import.FastProvideDAG`](https://github.

Unknown strategy tokens (e.g. typo `"uniuqe"`), malformed delimiters (`"pinned+"`), and invalid combinations (`"all+pinned"`) now produce a clear error at startup instead of being silently ignored.

#### 🔧 Filestore now respects `Provide.Strategy`

Blocks added via the [filestore](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-filestore) or [urlstore](https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-urlstore) (`ipfs add --nocopy`) used to ignore [`Provide.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providestrategy) and were always announced at write time. The filestore is now gated on the strategy the same way the regular blockstore is, so selective strategies get the same [fast-provide knobs](#-new---fast-provide-dag-flag-for-fine-tuned-provide-control) for filestore-backed content that they already had for regular `ipfs add`.

#### 🛡️ `ipfs object patch` validates UnixFS node types

As part of the ongoing deprecation of the legacy `ipfs object` API (which
Expand Down
97 changes: 97 additions & 0 deletions test/cli/provider_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,103 @@ func runProviderSuite(t *testing.T, sweep bool, apply cfgApplier, awaitReprovide
expectNoProviders(t, cidChunk, peers...)
})

// addLargeFilestoreFile writes a 2 MiB file to the publisher's
// node directory and adds it via --nocopy, returning the root CID
// and a chunk CID from the file's DAG links. With the configured
// 1 MiB chunker the file produces multiple leaf blocks so we can
// distinguish root-level from chunk-level provide behavior.
addLargeFilestoreFile := func(t *testing.T, publisher *harness.Node, addArgs ...string) (cidRoot, cidChunk string) {
t.Helper()
filePath := filepath.Join(publisher.Dir, "filestore-"+strconv.FormatInt(time.Now().UnixNano(), 10)+".bin")
require.NoError(t, os.WriteFile(filePath, random.Bytes(2*1024*1024), 0o644))

args := append([]string{"add", "-q", "--nocopy"}, addArgs...)
args = append(args, filePath)
cidRoot = strings.TrimSpace(publisher.IPFS(args...).Stdout.String())

dagOut := publisher.IPFS("dag", "get", cidRoot)
var dagNode struct {
Links []struct {
Hash map[string]string `json:"Hash"`
} `json:"Links"`
}
require.NoError(t, json.Unmarshal(dagOut.Stdout.Bytes(), &dagNode))
require.Greater(t, len(dagNode.Links), 1, "filestore file should have multiple chunks")
cidChunk = dagNode.Links[0].Hash["/"]
require.NotEmpty(t, cidChunk)
return cidRoot, cidChunk
}

t.Run("Filestore --nocopy with 'all' strategy provides every block", func(t *testing.T) {
t.Parallel()

nodes := initNodes(t, 2, func(n *harness.Node) {
n.SetIPFSConfig("Experimental.FilestoreEnabled", true)
n.SetIPFSConfig("Provide.Strategy", "all")
n.SetIPFSConfig("Import.UnixFSChunker", "size-1048576") // 1 MiB chunks
})
defer nodes.StopDaemons()
publisher, peers := nodes[0], nodes[1:]

// Positive control: with the default 'all' strategy the
// filestore Put path provides every block as it is written,
// including non-root chunks.
cidRoot, cidChunk := addLargeFilestoreFile(t, publisher)

pid := publisher.PeerID().String()
expectProviders(t, cidRoot, pid, peers...)
expectProviders(t, cidChunk, pid, peers...)
})

t.Run("Filestore --nocopy with selective strategy skips write-time provide", func(t *testing.T) {
t.Parallel()

nodes := initNodes(t, 2, func(n *harness.Node) {
n.SetIPFSConfig("Experimental.FilestoreEnabled", true)
n.SetIPFSConfig("Provide.Strategy", "pinned")
n.SetIPFSConfig("Import.UnixFSChunker", "size-1048576") // 1 MiB chunks
})
defer nodes.StopDaemons()
publisher, peers := nodes[0], nodes[1:]

// With a selective strategy the filestore must not eagerly
// announce blocks at write time. --pin=false skips the pin
// (so fast-provide-root has nothing to do) and
// --fast-provide-root=false disables it explicitly, isolating
// the assertion to the filestore's internal provide path.
cidRoot, cidChunk := addLargeFilestoreFile(t, publisher,
"--pin=false", "--fast-provide-root=false")

expectNoProviders(t, cidRoot, peers...)
expectNoProviders(t, cidChunk, peers...)
})

t.Run("Filestore --nocopy + selective strategy + --fast-provide-dag walks DAG", func(t *testing.T) {
t.Parallel()

nodes := initNodes(t, 2, func(n *harness.Node) {
n.SetIPFSConfig("Experimental.FilestoreEnabled", true)
n.SetIPFSConfig("Provide.Strategy", "pinned")
n.SetIPFSConfig("Import.UnixFSChunker", "size-1048576") // 1 MiB chunks
})
defer nodes.StopDaemons()
publisher, peers := nodes[0], nodes[1:]

// The selective-strategy gate skips the filestore's write-time
// provide, but the post-add ExecuteFastProvideDAG walk reads
// blocks through the wrapping blockstore (which transparently
// serves filestore-backed content) and announces each block,
// honoring the active strategy. This is the integration test
// behind the changelog claim that filestore content now plays
// well with the fast-provide-dag flag.
cidRoot, cidChunk := addLargeFilestoreFile(t, publisher,
"--fast-provide-dag", "--fast-provide-wait")

pid := publisher.PeerID().String()
expectProviders(t, cidRoot, pid, peers...)
expectProviders(t, cidChunk, pid, peers...)
})

t.Run("Provide with 'roots' strategy", func(t *testing.T) {
t.Parallel()

Expand Down
Loading