feat: Add pathSplit configuration to filestore:local

Splitting path into sub-directories is crucial to
keep the number of entries within a single directory
within sane limits.

Author: byo

filestore/config.go

@@ -18,6 +18,8 @@ type Config struct {
 type LocalConfig struct {
 	// BasePath is the filesystem directory where files are stored.
 	BasePath string
+	// PathSplit determines how the car name is split into subdirectories
+	PathSplit []int
 }
 
 type S3Config struct {
@@ -38,7 +40,9 @@ type S3Config struct {
 func MakeFilestore(cfg Config) (Interface, error) {
 	switch cfg.Type {
 	case "local":
-		return NewLocal(cfg.Local.BasePath)
+		return NewLocal(cfg.Local.BasePath,
+			WithPathSplit(cfg.Local.PathSplit...),
+		)
 	case "s3":
 		return NewS3(cfg.S3.BucketName,
 			WithEndpoint(cfg.S3.Endpoint),

filestore/filestore_test.go

@@ -106,6 +106,46 @@ func TestLocal(t *testing.T) {
 	})
 }
 
+func TestLocalWithPathSplit(t *testing.T) {
+	carDir := t.TempDir()
+
+	fileStore, err := filestore.NewLocal(carDir, filestore.WithPathSplit(2, 1))
+	require.NoError(t, err)
+	require.Equal(t, "local", fileStore.Type())
+
+	t.Run("test-Local-Put", func(t *testing.T) {
+		testPut(t, fileStore)
+	})
+
+	require.FileExists(t, filepath.Join(carDir, fileName[:2], fileName[2:3], fileName))
+
+	t.Run("test-Local-Head", func(t *testing.T) {
+		testHead(t, fileStore)
+	})
+
+	t.Run("test-Local-Get", func(t *testing.T) {
+		testGet(t, fileStore)
+	})
+
+	// Some extra bogus filesystem entries that should be skipped
+	for data, fName := range map[string]string{
+		"no-path-prefix":              filepath.Join(carDir, fileName),
+		"wrong-path-prefix":           filepath.Join(carDir, fileName[:2], fileName),
+		"prefix-not-on-path-boundary": filepath.Join(carDir, "corner-"+fileName[:2], fileName[2:3], fileName),
+	} {
+		require.NoError(t, os.MkdirAll(filepath.Dir(fName), 0700))
+		require.NoError(t, os.WriteFile(fName, []byte(data), 0600))
+	}
+
+	t.Run("test-Local-List", func(t *testing.T) {
+		testList(t, fileStore)
+	})
+
+	t.Run("test-Local-Delete", func(t *testing.T) {
+		testDelete(t, fileStore)
+	})
+}
+
 func TestMakeFilestore(t *testing.T) {
 	cfg := filestore.Config{
 		Type: "none",
@@ -130,6 +170,15 @@ func TestMakeFilestore(t *testing.T) {
 	fs, err = filestore.MakeFilestore(cfg)
 	require.NoError(t, err)
 	require.NotNil(t, fs)
+
+	cfg.Local.PathSplit = []int{0, -1}
+	fs, err = filestore.MakeFilestore(cfg)
+	require.ErrorContains(t, err, "invalid path split")
+
+	cfg.Local.PathSplit = []int{7, 5}
+	fs, err = filestore.MakeFilestore(cfg)
+	require.NoError(t, err)
+	require.NotNil(t, fs)
 }
 
 func testPut(t *testing.T, fileStore filestore.Interface) {

filestore/local.go

@@ -7,40 +7,82 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/ipni/storetheindex/fsutil"
 )
 
 // Local is a file store that stores files in the local file system.
 type Local struct {
-	basePath string
+	basePath  string
+	pathSplit []int
 }
 
-func NewLocal(basePath string) (*Local, error) {
+func NewLocal(basePath string, options ...LocalOption) (*Local, error) {
 	if !filepath.IsAbs(basePath) {
 		return nil, errors.New("base path must be absolute")
 	}
+
 	err := fsutil.DirWritable(basePath)
 	if err != nil {
 		return nil, err
 	}
+
+	opts, err := getLocalOpts(options)
+	if err != nil {
+		return nil, err
+	}
+
 	return &Local{
-		basePath: basePath,
+		basePath:  basePath,
+		pathSplit: opts.pathSplit,
 	}, nil
 }
 
+// fsPath returns the filesystem path of a given object path
+// based on a given basePath with path splitting criteria applied.
+func (l *Local) fsPath(basePath, relPath string) string {
+	fsDir, fsName := filepath.Split(filepath.FromSlash(relPath))
+
+	pathSegments := make([]string, 0, len(l.pathSplit)+3)
+	pathSegments = append(pathSegments, basePath, fsDir)
+
+	pathSegments = l.appendFnamePathSegments(fsName, pathSegments)
+
+	return filepath.Join(pathSegments...)
+}
+
+func (l *Local) appendFnamePathSegments(fileName string, pathSegments []string) []string {
+	fsNameNoExt := fileName
+	if dotPos := strings.IndexByte(fileName, '.'); dotPos > 0 {
+		fsNameNoExt = fileName[:dotPos]
+	}
+
+	splitPos := 0
+	for _, split := range l.pathSplit {
+		splitPos += split
+		if splitPos > len(fsNameNoExt) {
+			break
+		}
+
+		pathSegments = append(pathSegments, fsNameNoExt[splitPos-split:splitPos])
+	}
+
+	pathSegments = append(pathSegments, fileName)
+
+	return pathSegments
+}
+
 func (l *Local) Delete(ctx context.Context, relPath string) error {
-	err := os.Remove(filepath.Join(l.basePath, filepath.FromSlash(relPath)))
+	err := os.Remove(l.fsPath(l.basePath, relPath))
 	if err != nil && !errors.Is(err, os.ErrNotExist) {
 		return err
 	}
 	return nil
 }
 
 func (l *Local) Get(ctx context.Context, relPath string) (*File, io.ReadCloser, error) {
-	absPath := filepath.Join(l.basePath, filepath.FromSlash(relPath))
-
-	f, err := os.Open(absPath)
+	f, err := os.Open(l.fsPath(l.basePath, relPath))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, nil, fs.ErrNotExist
@@ -67,8 +109,7 @@ func (l *Local) Get(ctx context.Context, relPath string) (*File, io.ReadCloser,
 }
 
 func (l *Local) Head(ctx context.Context, relPath string) (*File, error) {
-	absPath := filepath.Join(l.basePath, filepath.FromSlash(relPath))
-	fi, err := os.Stat(absPath)
+	fi, err := os.Stat(l.fsPath(l.basePath, relPath))
 	if err != nil {
 		if errors.Is(err, os.ErrNotExist) {
 			return nil, fs.ErrNotExist
@@ -95,7 +136,23 @@ func (l *Local) List(ctx context.Context, relPath string, recursive bool) (<-cha
 		defer close(e)
 		defer close(c)
 
-		absPath := filepath.Join(l.basePath, filepath.FromSlash(relPath))
+		// The relPath may either be a path to a file or a path prefix,
+		// those cases must be handled separately due to pathSplit that only
+		// considers the filename as a splittable segment.
+		if stat, err := os.Stat(l.fsPath(l.basePath, relPath)); err == nil && stat.Mode().IsRegular() {
+			// relPath points to a file - emit that one and exit
+			c <- &File{
+				Modified: stat.ModTime(),
+				Path:     relPath,
+				Size:     stat.Size(),
+			}
+
+			return
+		}
+
+		// relPath must be treated as a directory prefix
+		absPath := filepath.Join(l.basePath, relPath)
+
 		e <- filepath.WalkDir(absPath, func(path string, d fs.DirEntry, err error) error {
 			if err != nil {
 				if errors.Is(err, os.ErrNotExist) {
@@ -106,9 +163,18 @@ func (l *Local) List(ctx context.Context, relPath string, recursive bool) (<-cha
 			}
 
 			if d.IsDir() {
-				if !recursive && path != absPath {
+				if recursive || len(l.pathSplit) > 0 {
+					// For both recursive scan and filepath split, keep descending
+					// to find files in sub-directories
+					return nil
+				}
+
+				if path != absPath {
+					// Without path split only a flat structure allowed,
+					// skip any sub-directories for faster iteration
 					return fs.SkipDir
 				}
+
 				return nil
 			}
 
@@ -122,15 +188,43 @@ func (l *Local) List(ctx context.Context, relPath string, recursive bool) (<-cha
 				return err
 			}
 
-			relFilePath, err := filepath.Rel(l.basePath, path)
-			if err != nil {
-				return err
-			}
-
 			f := &File{
 				Modified: fi.ModTime(),
-				Path:     filepath.ToSlash(relFilePath),
-				Size:     fi.Size(),
+				// Path:     filepath.ToSlash(relFilePath),
+				Size: fi.Size(),
+			}
+
+			if len(l.pathSplit) > 0 {
+				// Before emitting file entry, verify that the path is correct according
+				// to path split rules
+				fileName := filepath.Base(path)
+				expectedFileSubPath := l.fsPath("", fileName)
+
+				relAbsPath, err := filepath.Rel(absPath, path)
+				if err != nil {
+					return err
+				}
+
+				if prefix, found := strings.CutSuffix(relAbsPath, expectedFileSubPath); !found {
+					// Skip the file, path structure does not match
+					return nil
+				} else if !recursive && prefix != "" {
+					// Structure matches but is in sub-folder and not recursive mode, skip that one
+					return nil
+				} else if prefix != "" && prefix[len(prefix)-1] != filepath.Separator {
+					// Corner case - the prefix path must align with the path separator boundary
+					return nil
+				} else {
+					// All looks good, flatten adjust the final path by removing sub-folder structure
+					f.Path = filepath.ToSlash(filepath.Join(relPath, prefix, fileName))
+				}
+			} else {
+				relFilePath, err := filepath.Rel(l.basePath, path)
+				if err != nil {
+					return err
+				}
+
+				f.Path = filepath.ToSlash(relFilePath)
 			}
 
 			select {
@@ -146,11 +240,10 @@ func (l *Local) List(ctx context.Context, relPath string, recursive bool) (<-cha
 }
 
 func (l *Local) Put(ctx context.Context, relPath string, r io.Reader) (*File, error) {
-	absPath := filepath.Join(l.basePath, filepath.FromSlash(relPath))
+	absPath := l.fsPath(l.basePath, relPath)
 
-	dir, _ := filepath.Split(relPath)
-	if dir != "" {
-		err := os.MkdirAll(filepath.Dir(absPath), 0755)
+	if dir := filepath.Dir(absPath); dir != "" {
+		err := os.MkdirAll(dir, 0755)
 		if err != nil {
 			return nil, err
 		}

filestore/option.go

@@ -13,11 +13,11 @@ type s3Config struct {
 
 type S3Option func(*s3Config) error
 
-func getOpts(opts []S3Option) (s3Config, error) {
+func getS3Opts(opts []S3Option) (s3Config, error) {
 	var cfg s3Config
 	for i, opt := range opts {
 		if err := opt(&cfg); err != nil {
-			return s3Config{}, fmt.Errorf("option %d error: %s", i, err)
+			return s3Config{}, fmt.Errorf("option %d error: %w", i, err)
 		}
 	}
 	return cfg, nil
@@ -44,3 +44,38 @@ func WithKeys(accessKey, secretKey string) S3Option {
 		return nil
 	}
 }
+
+type localConfig struct {
+	basePath  string
+	pathSplit []int
+}
+
+type LocalOption func(*localConfig) error
+
+func getLocalOpts(opts []LocalOption) (localConfig, error) {
+	var cfg localConfig
+	for i, opt := range opts {
+		if err := opt(&cfg); err != nil {
+			return localConfig{}, fmt.Errorf("option %d error: %w", i, err)
+		}
+	}
+	return cfg, nil
+}
+
+func WithPathSplit(pathSplit ...int) LocalOption {
+	for i, splitSegment := range pathSplit {
+		if splitSegment <= 0 {
+			return func(lc *localConfig) error {
+				return fmt.Errorf(
+					"invalid path split config, segment %d size %d must be a positive integer",
+					i, splitSegment,
+				)
+			}
+		}
+	}
+
+	return func(lc *localConfig) error {
+		lc.pathSplit = pathSplit
+		return nil
+	}
+}

filestore/s3.go

@@ -36,7 +36,7 @@ func NewS3(bucketName string, options ...S3Option) (*S3, error) {
 		return nil, errors.New("s3 filestore requires bucket name")
 	}
 
-	opts, err := getOpts(options)
+	opts, err := getS3Opts(options)
 	if err != nil {
 		return nil, err
 	}