Skip to content
Merged
49 changes: 49 additions & 0 deletions config/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,12 @@ import (
"strings"

chunk "github.com/ipfs/boxo/chunker"
merkledag "github.com/ipfs/boxo/ipld/merkledag"
"github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
uio "github.com/ipfs/boxo/ipld/unixfs/io"
"github.com/ipfs/boxo/mfs"
"github.com/ipfs/boxo/verifcid"
cid "github.com/ipfs/go-cid"
mh "github.com/multiformats/go-multihash"
)

Expand Down Expand Up @@ -259,3 +262,49 @@ func (i *Import) UnixFSSplitterFunc() chunk.SplitterGen {
return s
}
}

// MFSRootOptions returns all MFS root options derived from Import config.
func (i *Import) MFSRootOptions() ([]mfs.Option, error) {
cidBuilder, err := i.UnixFSCidBuilder()
if err != nil {
return nil, err
}
sizeEstimationMode := i.HAMTSizeEstimationMode()
return []mfs.Option{
mfs.WithCidBuilder(cidBuilder),
mfs.WithChunker(i.UnixFSSplitterFunc()),
mfs.WithMaxLinks(int(i.UnixFSDirectoryMaxLinks.WithDefault(DefaultUnixFSDirectoryMaxLinks))),
mfs.WithMaxHAMTFanout(int(i.UnixFSHAMTDirectoryMaxFanout.WithDefault(DefaultUnixFSHAMTDirectoryMaxFanout))),
mfs.WithHAMTShardingSize(int(i.UnixFSHAMTDirectorySizeThreshold.WithDefault(DefaultUnixFSHAMTDirectorySizeThreshold))),
mfs.WithSizeEstimationMode(sizeEstimationMode),
}, nil
}

// UnixFSCidBuilder returns a cid.Builder based on Import.CidVersion and
// Import.HashFunction. Returns nil when both are at defaults (no override).
func (i *Import) UnixFSCidBuilder() (cid.Builder, error) {
cidVer := int(i.CidVersion.WithDefault(DefaultCidVersion))
hashFunc := i.HashFunction.WithDefault(DefaultHashFunction)

if cidVer == DefaultCidVersion && hashFunc == DefaultHashFunction {
return nil, nil
}

if hashFunc != DefaultHashFunction && cidVer == 0 {
cidVer = 1
}

prefix, err := merkledag.PrefixForCidVersion(cidVer)
if err != nil {
return nil, err
}

hashCode, ok := mh.Names[strings.ToLower(hashFunc)]
if !ok {
return nil, fmt.Errorf("Import.HashFunction unrecognized: %q", hashFunc)
}
prefix.MhType = hashCode
prefix.MhLength = -1

return &prefix, nil
}
130 changes: 43 additions & 87 deletions core/commands/files.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ import (
offline "github.com/ipfs/boxo/exchange/offline"
dag "github.com/ipfs/boxo/ipld/merkledag"
ft "github.com/ipfs/boxo/ipld/unixfs"
uio "github.com/ipfs/boxo/ipld/unixfs/io"
mfs "github.com/ipfs/boxo/mfs"
"github.com/ipfs/boxo/path"
cid "github.com/ipfs/go-cid"
Expand Down Expand Up @@ -505,7 +504,7 @@ being GC'ed.
return err
}

prefix, err := getPrefixNew(req, &cfg.Import)
prefix, err := getPrefix(req, &cfg.Import)
if err != nil {
return err
}
Expand Down Expand Up @@ -558,7 +557,11 @@ being GC'ed.
if mkParents {
maxDirLinks := int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks))
sizeEstimationMode := cfg.Import.HAMTSizeEstimationMode()
err := ensureContainingDirectoryExists(nd.FilesRoot, dst, prefix, maxDirLinks, &sizeEstimationMode)
err := ensureContainingDirectoryExists(nd.FilesRoot, dst,
mfs.WithCidBuilder(prefix),
mfs.WithMaxLinks(maxDirLinks),
mfs.WithSizeEstimationMode(sizeEstimationMode),
)
if err != nil {
return err
}
Expand Down Expand Up @@ -1060,7 +1063,7 @@ See '--to-files' in 'ipfs add --help' for more information.
rawLeaves = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
}

prefix, err := getPrefixNew(req, &cfg.Import)
prefix, err := getPrefix(req, &cfg.Import)
if err != nil {
return err
}
Expand All @@ -1073,7 +1076,11 @@ See '--to-files' in 'ipfs add --help' for more information.
if mkParents {
maxDirLinks := int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks))
sizeEstimationMode := cfg.Import.HAMTSizeEstimationMode()
err := ensureContainingDirectoryExists(nd.FilesRoot, path, prefix, maxDirLinks, &sizeEstimationMode)
err := ensureContainingDirectoryExists(nd.FilesRoot, path,
mfs.WithCidBuilder(prefix),
mfs.WithMaxLinks(maxDirLinks),
mfs.WithSizeEstimationMode(sizeEstimationMode),
)
if err != nil {
return err
}
Expand Down Expand Up @@ -1203,13 +1210,11 @@ Examples:
maxDirLinks := int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks))
sizeEstimationMode := cfg.Import.HAMTSizeEstimationMode()

err = mfs.Mkdir(root, dirtomake, mfs.MkdirOpts{
Mkparents: dashp,
Flush: flush,
CidBuilder: prefix,
MaxLinks: maxDirLinks,
SizeEstimationMode: &sizeEstimationMode,
})
err = mfs.Mkdir(root, dirtomake, mfs.MkdirOpts{Mkparents: dashp, Flush: flush},
mfs.WithCidBuilder(prefix),
mfs.WithMaxLinks(maxDirLinks),
mfs.WithSizeEstimationMode(sizeEstimationMode),
)

return err
},
Expand Down Expand Up @@ -1446,97 +1451,48 @@ func removePath(filesRoot *mfs.Root, path string, force bool, dashr bool) error
return pdir.Flush()
}

func getPrefixNew(req *cmds.Request, importCfg *config.Import) (cid.Builder, error) {
cidVer, cidVerSet := req.Options[filesCidVersionOptionName].(int)
hashFunStr, hashFunSet := req.Options[filesHashOptionName].(string)

// Fall back to Import config if CLI options not set
if !cidVerSet && importCfg != nil && !importCfg.CidVersion.IsDefault() {
cidVer = int(importCfg.CidVersion.WithDefault(config.DefaultCidVersion))
cidVerSet = true
}
if !hashFunSet && importCfg != nil && !importCfg.HashFunction.IsDefault() {
hashFunStr = importCfg.HashFunction.WithDefault(config.DefaultHashFunction)
hashFunSet = true
}

if !cidVerSet && !hashFunSet {
return nil, nil
}

if hashFunSet && cidVer == 0 {
cidVer = 1
}

prefix, err := dag.PrefixForCidVersion(cidVer)
if err != nil {
return nil, err
}

if hashFunSet {
hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)]
if !ok {
return nil, fmt.Errorf("unrecognized hash function: %s", strings.ToLower(hashFunStr))
}
prefix.MhType = hashFunCode
prefix.MhLength = -1
}

return &prefix, nil
}

// getPrefix builds a cid.Builder from CLI flags, falling back to importCfg
// when provided. Returns (nil, nil) when neither CLI nor config set a value.
func getPrefix(req *cmds.Request, importCfg *config.Import) (cid.Builder, error) {
cidVer, cidVerSet := req.Options[filesCidVersionOptionName].(int)
hashFunStr, hashFunSet := req.Options[filesHashOptionName].(string)

// Fall back to Import config if CLI options not set
if !cidVerSet && importCfg != nil && !importCfg.CidVersion.IsDefault() {
cidVer = int(importCfg.CidVersion.WithDefault(config.DefaultCidVersion))
cidVerSet = true
}
if !hashFunSet && importCfg != nil && !importCfg.HashFunction.IsDefault() {
hashFunStr = importCfg.HashFunction.WithDefault(config.DefaultHashFunction)
hashFunSet = true
}

if !cidVerSet && !hashFunSet {
return nil, nil
}

if hashFunSet && cidVer == 0 {
cidVer = 1
}

prefix, err := dag.PrefixForCidVersion(cidVer)
if err != nil {
return nil, err
if cidVerSet || hashFunSet {
// CLI flags take precedence: build prefix from them directly.
if hashFunSet && cidVer == 0 {
cidVer = 1
}
prefix, err := dag.PrefixForCidVersion(cidVer)
if err != nil {
return nil, err
}
if hashFunSet {
hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)]
if !ok {
return nil, fmt.Errorf("unrecognized hash function: %q", hashFunStr)
}
prefix.MhType = hashFunCode
prefix.MhLength = -1
}
return &prefix, nil
}

if hashFunSet {
hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)]
if !ok {
return nil, fmt.Errorf("unrecognized hash function: %s", strings.ToLower(hashFunStr))
}
prefix.MhType = hashFunCode
prefix.MhLength = -1
// No CLI flags: fall back to Import config.
if importCfg != nil {
return importCfg.UnixFSCidBuilder()
}

return &prefix, nil
return nil, nil
}

func ensureContainingDirectoryExists(r *mfs.Root, path string, builder cid.Builder, maxLinks int, sizeEstimationMode *uio.SizeEstimationMode) error {
func ensureContainingDirectoryExists(r *mfs.Root, path string, opts ...mfs.Option) error {
dirtomake := gopath.Dir(path)

if dirtomake == "/" {
return nil
}

return mfs.Mkdir(r, dirtomake, mfs.MkdirOpts{
Mkparents: true,
CidBuilder: builder,
MaxLinks: maxLinks,
SizeEstimationMode: sizeEstimationMode,
})
return mfs.Mkdir(r, dirtomake, mfs.MkdirOpts{Mkparents: true}, opts...)
}

func getFileHandle(r *mfs.Root, path string, create bool, builder cid.Builder) (*mfs.File, error) {
Expand Down
55 changes: 21 additions & 34 deletions core/coreunix/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,7 @@ func (adder *Adder) mfsRoot() (*mfs.Root, error) {
}

// Note, this adds it to DAGService already.
mr, err := mfs.NewEmptyRoot(adder.ctx, adder.dagService, nil, nil, mfs.MkdirOpts{
CidBuilder: adder.CidBuilder,
MaxLinks: adder.MaxDirectoryLinks,
MaxHAMTFanout: adder.MaxHAMTFanout,
SizeEstimationMode: adder.SizeEstimationMode,
})
mr, err := mfs.NewEmptyRoot(adder.ctx, adder.dagService, nil, nil, adder.mkdirOpts()...)
if err != nil {
return nil, err
}
Expand All @@ -125,6 +120,20 @@ func (adder *Adder) SetMfsRoot(r *mfs.Root) {
adder.mroot = r
}

// mkdirOpts returns MFS options derived from the adder's config,
// with any additional options appended.
func (adder *Adder) mkdirOpts(extra ...mfs.Option) []mfs.Option {
opts := []mfs.Option{
mfs.WithCidBuilder(adder.CidBuilder),
mfs.WithMaxLinks(adder.MaxDirectoryLinks),
mfs.WithMaxHAMTFanout(adder.MaxHAMTFanout),
}
if adder.SizeEstimationMode != nil {
opts = append(opts, mfs.WithSizeEstimationMode(*adder.SizeEstimationMode))
}
return append(opts, extra...)
}

// Constructs a node from reader's data, and adds it. Doesn't pin.
func (adder *Adder) add(reader io.Reader) (ipld.Node, error) {
chnk, err := chunker.FromString(reader, adder.Chunker)
Expand Down Expand Up @@ -274,15 +283,8 @@ func (adder *Adder) addNode(node ipld.Node, path string) error {

dir := gopath.Dir(path)
if dir != "." {
opts := mfs.MkdirOpts{
Mkparents: true,
Flush: false,
CidBuilder: adder.CidBuilder,
MaxLinks: adder.MaxDirectoryLinks,
MaxHAMTFanout: adder.MaxHAMTFanout,
SizeEstimationMode: adder.SizeEstimationMode,
}
if err := mfs.Mkdir(mr, dir, opts); err != nil {
mkdirOpts := adder.mkdirOpts()
if err := mfs.Mkdir(mr, dir, mfs.MkdirOpts{Mkparents: true, Flush: false}, mkdirOpts...); err != nil {
return err
}
}
Expand Down Expand Up @@ -506,15 +508,8 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory

// if we need to store mode or modification time then create a new root which includes that data
if toplevel && (adder.FileMode != 0 || !adder.FileMtime.IsZero()) {
mr, err := mfs.NewEmptyRoot(ctx, adder.dagService, nil, nil,
mfs.MkdirOpts{
CidBuilder: adder.CidBuilder,
MaxLinks: adder.MaxDirectoryLinks,
MaxHAMTFanout: adder.MaxHAMTFanout,
ModTime: adder.FileMtime,
Mode: adder.FileMode,
SizeEstimationMode: adder.SizeEstimationMode,
})
opts := adder.mkdirOpts(mfs.WithMode(adder.FileMode), mfs.WithModTime(adder.FileMtime))
mr, err := mfs.NewEmptyRoot(ctx, adder.dagService, nil, nil, opts...)
if err != nil {
return err
}
Expand All @@ -526,16 +521,8 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory
if err != nil {
return err
}
err = mfs.Mkdir(mr, path, mfs.MkdirOpts{
Mkparents: true,
Flush: false,
CidBuilder: adder.CidBuilder,
Mode: adder.FileMode,
ModTime: adder.FileMtime,
MaxLinks: adder.MaxDirectoryLinks,
MaxHAMTFanout: adder.MaxHAMTFanout,
SizeEstimationMode: adder.SizeEstimationMode,
})
mkdirOpts := adder.mkdirOpts(mfs.WithMode(adder.FileMode), mfs.WithModTime(adder.FileMtime))
err = mfs.Mkdir(mr, path, mfs.MkdirOpts{Mkparents: true, Flush: false}, mkdirOpts...)
if err != nil {
return err
}
Expand Down
19 changes: 6 additions & 13 deletions core/node/core.go
Original file line number Diff line number Diff line change
Expand Up @@ -248,19 +248,12 @@ func Files(strategy string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo
if err != nil {
return nil, fmt.Errorf("failed to get config: %w", err)
}
chunkerGen := cfg.Import.UnixFSSplitterFunc()
maxDirLinks := int(cfg.Import.UnixFSDirectoryMaxLinks.WithDefault(config.DefaultUnixFSDirectoryMaxLinks))
maxHAMTFanout := int(cfg.Import.UnixFSHAMTDirectoryMaxFanout.WithDefault(config.DefaultUnixFSHAMTDirectoryMaxFanout))
hamtShardingSize := int(cfg.Import.UnixFSHAMTDirectorySizeThreshold.WithDefault(config.DefaultUnixFSHAMTDirectorySizeThreshold))
sizeEstimationMode := cfg.Import.HAMTSizeEstimationMode()

root, err := mfs.NewRoot(ctx, dag, nd, pf, prov,
mfs.WithChunker(chunkerGen),
mfs.WithMaxLinks(maxDirLinks),
mfs.WithMaxHAMTFanout(maxHAMTFanout),
mfs.WithHAMTShardingSize(hamtShardingSize),
mfs.WithSizeEstimationMode(sizeEstimationMode),
)
mfsOpts, err := cfg.Import.MFSRootOptions()
if err != nil {
return nil, fmt.Errorf("failed to build MFS options from Import config: %w", err)
}

root, err := mfs.NewRoot(ctx, dag, nd, pf, prov, mfsOpts...)
if err != nil {
return nil, fmt.Errorf("failed to initialize MFS root from %s stored at %s: %w. "+
"If corrupted, use 'ipfs files chroot' to reset (see --help)", nd.Cid(), FilesRootDatastoreKey, err)
Expand Down
Loading
Loading