Ignore hidden files in batch (#617)

* Ignore hidden files in batch

* Allow for hidden directories

* Clarify comment.

* Remove hidden files when batch is in transferDir

* Add unit tests

* Improve uni test

Author: DanielCosme

internal/batch/workflow.go

@@ -89,6 +89,10 @@ func (a *BatchActivity) Execute(ctx context.Context, params BatchWorkflowInput)
 			return nil // Keep walking.
 		}
 
+		if strings.HasPrefix(filepath.Base(path), ".") && !entry.IsDir() {
+			return nil // Don't process hidden files as SIPs
+		}
+
 		req := collection.ProcessingWorkflowRequest{
 			BatchDir:           filepath.Dir(path),
 			Key:                entry.Name(),

internal/workflow/activities/bundle.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -61,6 +62,11 @@ func (a *BundleActivity) Execute(ctx context.Context, params *BundleActivityPara
 			res.FullPath = filepath.Join(params.BatchDir, params.Key)
 			// This makes the workflow not to delete the original content in the transfer directory
 			res.FullPathBeforeStrip = ""
+			if params.ExcludeHiddenFiles {
+				if err := removeHiddenFiles(res.FullPath); err != nil {
+					return nil, temporal.NewNonRetryableError(fmt.Errorf("failed to remove hidden files: %w", err))
+				}
+			}
 		} else {
 			src := filepath.Join(params.BatchDir, params.Key)
 			dst := params.TransferDir
@@ -312,3 +318,19 @@ func unbag(path string) error {
 
 	return nil
 }
+
+func removeHiddenFiles(path string) error {
+	return filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+		if strings.HasPrefix(info.Name(), ".") {
+			return os.Remove(path)
+		}
+		return nil
+	})
+}

internal/workflow/activities/bundle_test.go

@@ -61,6 +61,51 @@ func TestBundleActivity(t *testing.T) {
 			),
 		)
 	})
+
+	t.Run("Remove hidden files when BatchDir is a subfolder of the TransferDir", func(t *testing.T) {
+		activity := NewBundleActivity()
+		ts := &temporalsdk_testsuite.WorkflowTestSuite{}
+		env := ts.NewTestActivityEnvironment()
+		env.RegisterActivity(activity.Execute)
+
+		transferDir := fs.NewDir(t, "enduro",
+			fs.WithDir("batch-folder",
+				fs.WithDir(
+					"sip",
+					fs.WithFile("foobar.txt", "Hello world!\n"),
+					fs.WithFile(".hidden", ""),
+				),
+			),
+		)
+		batchDir := transferDir.Join("batch-folder")
+		sipSourceDir := transferDir.Join("batch-folder", "sip")
+
+		fut, err := env.ExecuteActivity(activity.Execute, &BundleActivityParams{
+			ExcludeHiddenFiles: true,
+			IsDir:              true,
+			TransferDir:        transferDir.Path(),
+			BatchDir:           batchDir,
+			Key:                "sip",
+		})
+		assert.NilError(t, err)
+
+		res := BundleActivityResult{}
+		assert.NilError(t, fut.Get(&res))
+		assert.Assert(t,
+			fs.Equal(
+				sipSourceDir,
+				fs.Expected(t,
+					// .hidden is not expected because ExcludeHiddenFiles is enabled.
+					fs.WithFile("foobar.txt", "Hello world!\n"),
+					fs.MatchAnyFileMode,
+				),
+			),
+		)
+		assert.DeepEqual(t, res.FullPath, sipSourceDir)
+		rePath, err := filepath.Rel(transferDir.Path(), sipSourceDir)
+		assert.NilError(t, err)
+		assert.DeepEqual(t, res.RelPath, rePath)
+	})
 }
 
 func TestUnbag(t *testing.T) {