Skip to content

Commit 996bcf3

Browse files
hsanjuanlidel
andauthored
feat: partial DAG provides with Reprovider.Strategy=mfs|pinned+mfs (#10754)
Co-authored-by: Marcin Rataj <lidel@lidel.org>
1 parent 19b591d commit 996bcf3

File tree

11 files changed

+91
-41
lines changed

11 files changed

+91
-41
lines changed

core/node/core.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ func FetcherConfig(bs blockservice.BlockService) FetchersOut {
115115
// path resolution should not fetch new blocks via exchange.
116116
offlineBs := blockservice.New(bs.Blockstore(), offline.Exchange(bs.Blockstore()))
117117
offlineIpldFetcher := bsfetcher.NewFetcherConfig(offlineBs)
118+
offlineIpldFetcher.SkipNotFound = true // carries onto the UnixFSFetcher below
118119
offlineIpldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
119120
offlineUnixFSFetcher := offlineIpldFetcher.WithReifier(unixfsnode.Reify)
120121

core/node/provider.go

Lines changed: 60 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@ import (
77

88
"github.com/ipfs/boxo/blockstore"
99
"github.com/ipfs/boxo/fetcher"
10+
"github.com/ipfs/boxo/mfs"
1011
pin "github.com/ipfs/boxo/pinning/pinner"
1112
provider "github.com/ipfs/boxo/provider"
13+
"github.com/ipfs/go-cid"
1214
"github.com/ipfs/kubo/repo"
1315
irouting "github.com/ipfs/kubo/routing"
1416
"go.uber.org/fx"
@@ -136,14 +138,8 @@ func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, repro
136138

137139
var keyProvider fx.Option
138140
switch reprovideStrategy {
139-
case "all", "":
140-
keyProvider = fx.Provide(newProvidingStrategy(false, false))
141-
case "roots":
142-
keyProvider = fx.Provide(newProvidingStrategy(true, true))
143-
case "pinned":
144-
keyProvider = fx.Provide(newProvidingStrategy(true, false))
145-
case "flat":
146-
keyProvider = fx.Provide(provider.NewBlockstoreProvider)
141+
case "all", "", "roots", "pinned", "mfs", "pinned+mfs", "flat":
142+
keyProvider = fx.Provide(newProvidingStrategy(reprovideStrategy))
147143
default:
148144
return fx.Error(fmt.Errorf("unknown reprovider strategy %q", reprovideStrategy))
149145
}
@@ -159,32 +155,68 @@ func OfflineProviders() fx.Option {
159155
return fx.Provide(provider.NewNoopProvider)
160156
}
161157

162-
func newProvidingStrategy(onlyPinned, onlyRoots bool) interface{} {
158+
func mfsProvider(mfsRoot *mfs.Root, fetcher fetcher.Factory) provider.KeyChanFunc {
159+
return func(ctx context.Context) (<-chan cid.Cid, error) {
160+
err := mfsRoot.FlushMemFree(ctx)
161+
if err != nil {
162+
return nil, fmt.Errorf("error flushing mfs, cannot provide MFS: %w", err)
163+
}
164+
rootNode, err := mfsRoot.GetDirectory().GetNode()
165+
if err != nil {
166+
return nil, fmt.Errorf("error loading mfs root, cannot provide MFS: %w", err)
167+
}
168+
169+
kcf := provider.NewDAGProvider(rootNode.Cid(), fetcher)
170+
return kcf(ctx)
171+
}
172+
173+
}
174+
175+
func mfsRootProvider(mfsRoot *mfs.Root) provider.KeyChanFunc {
176+
return func(ctx context.Context) (<-chan cid.Cid, error) {
177+
rootNode, err := mfsRoot.GetDirectory().GetNode()
178+
if err != nil {
179+
return nil, fmt.Errorf("error loading mfs root, cannot provide MFS: %w", err)
180+
}
181+
ch := make(chan cid.Cid, 1)
182+
ch <- rootNode.Cid()
183+
close(ch)
184+
return ch, nil
185+
}
186+
}
187+
188+
func newProvidingStrategy(strategy string) interface{} {
163189
type input struct {
164190
fx.In
165-
Pinner pin.Pinner
166-
Blockstore blockstore.Blockstore
167-
IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
191+
Pinner pin.Pinner
192+
Blockstore blockstore.Blockstore
193+
OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"`
194+
OfflineUnixFSFetcher fetcher.Factory `name:"offlineUnixfsFetcher"`
195+
MFSRoot *mfs.Root
168196
}
169197
return func(in input) provider.KeyChanFunc {
170-
// Pinner-related CIDs will be buffered in memory to avoid
171-
// deadlocking the pinner when the providing process is slow.
172-
173-
if onlyRoots {
174-
return provider.NewBufferedProvider(
175-
provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher),
198+
switch strategy {
199+
case "roots":
200+
return provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.OfflineIPLDFetcher))
201+
case "pinned":
202+
return provider.NewBufferedProvider(provider.NewPinnedProvider(false, in.Pinner, in.OfflineIPLDFetcher))
203+
case "pinned+mfs":
204+
return provider.NewPrioritizedProvider(
205+
provider.NewBufferedProvider(provider.NewPinnedProvider(false, in.Pinner, in.OfflineIPLDFetcher)),
206+
mfsProvider(in.MFSRoot, in.OfflineUnixFSFetcher),
176207
)
177-
}
178-
179-
if onlyPinned {
180-
return provider.NewBufferedProvider(
181-
provider.NewPinnedProvider(false, in.Pinner, in.IPLDFetcher),
208+
case "mfs":
209+
return mfsProvider(in.MFSRoot, in.OfflineUnixFSFetcher)
210+
case "flat":
211+
return provider.NewBlockstoreProvider(in.Blockstore)
212+
default: // "all", ""
213+
return provider.NewPrioritizedProvider(
214+
provider.NewPrioritizedProvider(
215+
provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.OfflineIPLDFetcher)),
216+
mfsRootProvider(in.MFSRoot),
217+
),
218+
provider.NewBlockstoreProvider(in.Blockstore),
182219
)
183220
}
184-
185-
return provider.NewPrioritizedProvider(
186-
provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher)),
187-
provider.NewBlockstoreProvider(in.Blockstore),
188-
)
189221
}
190222
}

docs/changelogs/v0.35.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team.
1010

1111
- [Overview](#overview)
1212
- [🔦 Highlights](#-highlights)
13+
- [Dedicated `Reprovider.Strategy` for MFS](#dedicated-reproviderstrategy-for-mfs)
1314
- [📦️ Important dependency updates](#-important-dependency-updates)
1415
- [📝 Changelog](#-changelog)
1516
- [👨‍👩‍👧‍👦 Contributors](#-contributors)
@@ -18,6 +19,16 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team.
1819

1920
### 🔦 Highlights
2021

22+
#### Dedicated `Reprovider.Strategy` for MFS
23+
24+
The [Mutable File System (MFS)](https://docs.ipfs.tech/concepts/glossary/#mfs) in Kubo is a UnixFS filesystem managed with [`ipfs files`](https://docs.ipfs.tech/reference/kubo/cli/#ipfs-files) commands. It supports familiar file operations like cp and mv within a folder-tree structure, automatically updating a MerkleDAG and a "root CID" that reflects the current MFS state. Files in MFS are protected from garbage collection, offering a simpler alternative to `ipfs pin`. This makes it a popular choice for tools like [IPFS Desktop](https://docs.ipfs.tech/install/ipfs-desktop/) and the [WebUI](https://github.com/ipfs/ipfs-webui/#readme).
25+
26+
Previously, the `pinned` reprovider strategy required manual pin management: each dataset update meant pinning the new version and unpinning the old one. Now, new strategies—`mfs` and `pinned+mfs`—let users limit announcements to data explicitly placed in MFS. This simplifies updating datasets and announcing only the latest version to the Amino DHT.
27+
28+
Users relying on the `pinned` strategy can switch to `pinned+mfs` and use MFS alone to manage updates and announcements, eliminating the need for manual pinning and unpinning. We hope this makes it easier to publish just the data that matters to you.
29+
30+
See [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy) for more details.
31+
2132
##### `Routing.IgnoreProviders`
2233

2334
This new option allows ignoring specific peer IDs when returned by the content

docs/config.md

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1587,21 +1587,26 @@ Type: `optionalDuration` (unset for the default)
15871587
Tells reprovider what should be announced. Valid strategies are:
15881588

15891589
- `"all"` - announce all CIDs of stored blocks
1590-
- Order: root blocks of direct and recursive pins are announced first, then the rest of blockstore
1591-
- `"pinned"` - only announce pinned CIDs recursively (both roots and child blocks)
1590+
- Order: root blocks of direct and recursive pins and MFS root are announced first, then the rest of blockstore
1591+
- `"pinned"` - only announce recursively pinned CIDs (`ipfs pin add -r`, both roots and child blocks)
15921592
- Order: root blocks of direct and recursive pins are announced first, then the child blocks of recursive pins
1593-
- `"roots"` - only announce the root block of explicitly pinned CIDs
1593+
- `"roots"` - only announce the root block of explicitly pinned CIDs (`ipfs pin add`)
15941594
- **⚠️ BE CAREFUL:** node with `roots` strategy will not announce child blocks.
15951595
It makes sense only for use cases where the entire DAG is fetched in full,
15961596
and a graceful resume does not have to be guaranteed: the lack of child
15971597
announcements means an interrupted retrieval won't be able to find
15981598
providers for the missing block in the middle of a file, unless the peer
15991599
happens to already be connected to a provider and ask for child CID over
16001600
bitswap.
1601+
- `"mfs"` - announce only the local CIDs that are part of the MFS (`ipfs files`)
1602+
- Note: MFS is lazy-loaded. Only the MFS blocks present in local datastore are announced.
1603+
- `"pinned+mfs"` - a combination of the `pinned` and `mfs` strategies.
1604+
- **ℹ️ NOTE:** This is the suggested strategy for users who run without GC and don't want to provide everything in cache.
1605+
- Order: first `pinned` and then the locally available part of `mfs`.
16011606
- `"flat"` - same as `all`, announce all CIDs of stored blocks, but without prioritizing anything.
16021607

16031608
> [!IMPORTANT]
1604-
> Reproviding larger pinsets using the `all`, `pinned`, or `roots` strategies requires additional memory, with an estimated ~1 GiB of RAM per 20 million items for reproviding to the Amino DHT.
1609+
> Reproviding larger pinsets using the `all`, `mfs`, `pinned`, `pinned+mfs` or `roots` strategies requires additional memory, with an estimated ~1 GiB of RAM per 20 million items for reproviding to the Amino DHT.
16051610
> This is due to the use of a buffered provider, which avoids holding a lock on the entire pinset during the reprovide cycle.
16061611
> The `flat` strategy can be used to lower memory requirements, but only recommended if memory utilization is too high, prioritization of pins is not necessary, and it is acceptable to announce every block cached in the local repository.
16071612

docs/examples/kubo-as-a-library/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ go 1.24
77
replace github.com/ipfs/kubo => ./../../..
88

99
require (
10-
github.com/ipfs/boxo v0.29.1
10+
github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb
1111
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
1212
github.com/libp2p/go-libp2p v0.41.1
1313
github.com/multiformats/go-multiaddr v0.15.0

docs/examples/kubo-as-a-library/go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
298298
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
299299
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
300300
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
301-
github.com/ipfs/boxo v0.29.1 h1:z61ZT4YDfTHLjXTsu/+3wvJ8aJlExthDSOCpx6Nh8xc=
302-
github.com/ipfs/boxo v0.29.1/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
301+
github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb h1:kA7c3CF6/d8tUwGJR/SwIfaRz7Xk7Fbyoh2ePZAFMlw=
302+
github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
303303
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
304304
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
305305
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ require (
2121
github.com/hashicorp/go-version v1.7.0
2222
github.com/ipfs-shipyard/nopfs v0.0.14
2323
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0
24-
github.com/ipfs/boxo v0.29.1
24+
github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb
2525
github.com/ipfs/go-block-format v0.2.0
2626
github.com/ipfs/go-cid v0.5.0
2727
github.com/ipfs/go-cidutil v0.1.0

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
362362
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
363363
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
364364
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
365-
github.com/ipfs/boxo v0.29.1 h1:z61ZT4YDfTHLjXTsu/+3wvJ8aJlExthDSOCpx6Nh8xc=
366-
github.com/ipfs/boxo v0.29.1/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
365+
github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb h1:kA7c3CF6/d8tUwGJR/SwIfaRz7Xk7Fbyoh2ePZAFMlw=
366+
github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
367367
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
368368
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
369369
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=

test/dependencies/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ require (
116116
github.com/huin/goupnp v1.3.0 // indirect
117117
github.com/inconshreveable/mousetrap v1.1.0 // indirect
118118
github.com/ipfs/bbloom v0.0.4 // indirect
119-
github.com/ipfs/boxo v0.29.1 // indirect
119+
github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb // indirect
120120
github.com/ipfs/go-block-format v0.2.0 // indirect
121121
github.com/ipfs/go-cid v0.5.0 // indirect
122122
github.com/ipfs/go-datastore v0.8.2 // indirect

test/dependencies/go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
294294
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
295295
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
296296
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
297-
github.com/ipfs/boxo v0.29.1 h1:z61ZT4YDfTHLjXTsu/+3wvJ8aJlExthDSOCpx6Nh8xc=
298-
github.com/ipfs/boxo v0.29.1/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
297+
github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb h1:kA7c3CF6/d8tUwGJR/SwIfaRz7Xk7Fbyoh2ePZAFMlw=
298+
github.com/ipfs/boxo v0.29.2-0.20250409154342-bbaf2e146dfb/go.mod h1:omQZmLS7LegSpBy3m4CrAB9/SO7Fq3pfv+5y1FOd+gI=
299299
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
300300
github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
301301
github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg=

0 commit comments

Comments
 (0)