diff --git a/cmd/mount.go b/cmd/mount.go index e59dd4fef9..f57dddcf5f 100644 --- a/cmd/mount.go +++ b/cmd/mount.go @@ -106,6 +106,7 @@ be interacting with the file system.`) FinalizeFileForRapid: newConfig.Write.FinalizeFileForRapid, DisableListAccessCheck: newConfig.DisableListAccessCheck, DummyIOCfg: newConfig.DummyIo, + IsTypeCacheDeprecated: newConfig.EnableTypeCacheDeprecation, } bm := gcsx.NewBucketManager(bucketCfg, storageHandle) diff --git a/internal/fs/inode/dir.go b/internal/fs/inode/dir.go index c9349d57e7..45964f3161 100644 --- a/internal/fs/inode/dir.go +++ b/internal/fs/inode/dir.go @@ -27,6 +27,7 @@ import ( "github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx" "github.com/googlecloudplatform/gcsfuse/v3/internal/locker" "github.com/googlecloudplatform/gcsfuse/v3/internal/logger" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching" "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" "github.com/jacobsa/fuse/fuseops" @@ -251,6 +252,8 @@ type dirInode struct { // Represents if folder has been unlinked in hierarchical bucket. This is not getting used in // non-hierarchical bucket. unlinked bool + + metadataCacheTtlSecs int64 } var _ DirInode = &dirInode{} @@ -306,6 +309,7 @@ func NewDirInode( isUnsupportedPathSupportEnabled: cfg.EnableUnsupportedPathSupport, isEnableTypeCacheDeprecation: cfg.EnableTypeCacheDeprecation, unlinked: false, + metadataCacheTtlSecs: cfg.MetadataCache.TtlSecs, } // readObjectsUnlocked is used by the prefetcher so the background worker performs GCS I/O without the lock, // acquiring d.mu only to update the cache. @@ -338,19 +342,19 @@ func (d *dirInode) checkInvariants() { } func (d *dirInode) lookUpChildFile(ctx context.Context, name string) (*Core, error) { - return findExplicitInode(ctx, d.Bucket(), NewFileName(d.Name(), name)) + return findExplicitInode(ctx, d.Bucket(), NewFileName(d.Name(), name), false) } func (d *dirInode) lookUpChildDir(ctx context.Context, name string) (*Core, error) { childName := NewDirName(d.Name(), name) if d.isBucketHierarchical() { - return findExplicitFolder(ctx, d.Bucket(), childName) + return findExplicitFolder(ctx, d.Bucket(), childName, false) } if d.implicitDirs { - return findDirInode(ctx, d.Bucket(), childName) + return findDirInode(ctx, d.Bucket(), childName, d.isEnableTypeCacheDeprecation) } - return findExplicitInode(ctx, d.Bucket(), childName) + return findExplicitInode(ctx, d.Bucket(), childName, false) } // Look up the file for a (file, dir) pair with conflicting names, overriding @@ -384,10 +388,11 @@ func (d *dirInode) lookUpConflicting(ctx context.Context, name string) (*Core, e // findExplicitInode finds the file or dir inode core backed by an explicit // object in GCS with the given name. Return nil if such object does not exist. -func findExplicitInode(ctx context.Context, bucket *gcsx.SyncerBucket, name Name) (*Core, error) { +func findExplicitInode(ctx context.Context, bucket *gcsx.SyncerBucket, name Name, fetchOnlyFromCache bool) (*Core, error) { // Call the bucket. req := &gcs.StatObjectRequest{ - Name: name.GcsObjectName(), + Name: name.GcsObjectName(), + FetchOnlyFromCache: fetchOnlyFromCache, } m, _, err := bucket.StatObject(ctx, req) @@ -403,6 +408,13 @@ func findExplicitInode(ctx context.Context, bucket *gcsx.SyncerBucket, name Name return nil, fmt.Errorf("StatObject: %w", err) } + // Treat this as an implicit directory. Since implicit objects lack metadata + // (only the name is preserved), we set MinObject to nil. This ensures the + // inode is correctly assigned the ImplicitDir Core Type. + if fetchOnlyFromCache && m.Generation == 0 { + m = nil + } + return &Core{ Bucket: bucket, FullName: name, @@ -410,10 +422,11 @@ func findExplicitInode(ctx context.Context, bucket *gcsx.SyncerBucket, name Name }, nil } -func findExplicitFolder(ctx context.Context, bucket *gcsx.SyncerBucket, name Name) (*Core, error) { +func findExplicitFolder(ctx context.Context, bucket *gcsx.SyncerBucket, name Name, fetchOnlyFromCache bool) (*Core, error) { // Call the bucket. req := &gcs.GetFolderRequest{ - Name: name.GcsObjectName(), + Name: name.GcsObjectName(), + FetchOnlyFromCache: fetchOnlyFromCache, } folder, err := bucket.GetFolder(ctx, req) @@ -437,14 +450,15 @@ func findExplicitFolder(ctx context.Context, bucket *gcsx.SyncerBucket, name Nam // findDirInode finds the dir inode core where the directory is either explicit // or implicit. Returns nil if no such directory exists. -func findDirInode(ctx context.Context, bucket *gcsx.SyncerBucket, name Name) (*Core, error) { +func findDirInode(ctx context.Context, bucket *gcsx.SyncerBucket, name Name, isTypeCacheDeprecated bool) (*Core, error) { if !name.IsDir() { return nil, fmt.Errorf("%q is not directory", name) } req := &gcs.ListObjectsRequest{ - Prefix: name.GcsObjectName(), - MaxResults: 1, + Prefix: name.GcsObjectName(), + MaxResults: 1, + IsTypeCacheDeprecated: isTypeCacheDeprecated, } listing, err := bucket.ListObjects(ctx, req) if err != nil { @@ -583,40 +597,93 @@ func (d *dirInode) LookUpChild(ctx context.Context, name string) (*Core, error) return d.lookUpConflicting(ctx, name) } + cachedType := metadata.UnknownType + + // 1. Optimization: If Type Cache is deprecated, attempt a lookup via the Stat Cache first. + // We skip this if the metadata cache TTL is 0, as the cache layer is inactive. + if d.IsTypeCacheDeprecated() && d.metadataCacheTtlSecs != 0 { + var cacheMissErr *caching.CacheMissError + + // Try File + fileResult, err := findExplicitInode(ctx, d.Bucket(), NewFileName(d.Name(), name), true) + if err != nil && !errors.As(err, &cacheMissErr) { + return nil, err + } + + // Try Directory + var dirResult *Core + if d.Bucket().BucketType().Hierarchical { + dirResult, err = findExplicitFolder(ctx, d.Bucket(), NewDirName(d.Name(), name), true) + } else { + dirResult, err = findExplicitInode(ctx, d.Bucket(), NewDirName(d.Name(), name), true) + } + if err != nil && !errors.As(err, &cacheMissErr) { + return nil, err + } + + if dirResult != nil { + return dirResult, nil + } else if fileResult != nil { + return fileResult, nil + } + } + + // 2. Legacy: If Type Cache is NOT deprecated, fetch the type hint. + if !d.IsTypeCacheDeprecated() { + cachedType = d.cache.Get(d.cacheClock.Now(), name) + } + + // 3. Main Lookup Logic (Unified) + result, err := d.fetchCoreEntity(ctx, name, cachedType) + if err != nil { + return nil, err + } + + // 4. Legacy: Update Type Cache if needed. + if !d.IsTypeCacheDeprecated() { + if result != nil { + d.cache.Insert(d.cacheClock.Now(), name, result.Type()) + } else if d.enableNonexistentTypeCache && cachedType == metadata.UnknownType { + d.cache.Insert(d.cacheClock.Now(), name, metadata.NonexistentType) + } + } + + return result, nil +} + +// fetchCoreEntity contains all the existing logic for looking up children +// without worrying about the isTypeCacheDeprecated flag. +func (d *dirInode) fetchCoreEntity(ctx context.Context, name string, cachedType metadata.Type) (*Core, error) { group, ctx := errgroup.WithContext(ctx) var fileResult *Core var dirResult *Core + var err error + lookUpFile := func() (err error) { - fileResult, err = findExplicitInode(ctx, d.Bucket(), NewFileName(d.Name(), name)) + fileResult, err = findExplicitInode(ctx, d.Bucket(), NewFileName(d.Name(), name), false) return } lookUpExplicitDir := func() (err error) { - dirResult, err = findExplicitInode(ctx, d.Bucket(), NewDirName(d.Name(), name)) + dirResult, err = findExplicitInode(ctx, d.Bucket(), NewDirName(d.Name(), name), false) return } lookUpImplicitOrExplicitDir := func() (err error) { - dirResult, err = findDirInode(ctx, d.Bucket(), NewDirName(d.Name(), name)) + dirResult, err = findDirInode(ctx, d.Bucket(), NewDirName(d.Name(), name), d.isEnableTypeCacheDeprecation) return } lookUpHNSDir := func() (err error) { - dirResult, err = findExplicitFolder(ctx, d.Bucket(), NewDirName(d.Name(), name)) + dirResult, err = findExplicitFolder(ctx, d.Bucket(), NewDirName(d.Name(), name), false) return } - var cachedType metadata.Type - if d.IsTypeCacheDeprecated() { - // TODO: Add deprecation logic. - cachedType = metadata.UnknownType - } else { - cachedType = d.cache.Get(d.cacheClock.Now(), name) - } + switch cachedType { case metadata.ImplicitDirType: - dirResult = &Core{ + return &Core{ Bucket: d.Bucket(), FullName: NewDirName(d.Name(), name), MinObject: nil, - } + }, nil case metadata.ExplicitDirType: if d.isBucketHierarchical() { group.Go(lookUpHNSDir) @@ -625,6 +692,7 @@ func (d *dirInode) LookUpChild(ctx context.Context, name string) (*Core, error) } case metadata.RegularFileType, metadata.SymlinkType: group.Go(lookUpFile) + case metadata.NonexistentType: return nil, nil case metadata.UnknownType: @@ -642,29 +710,16 @@ func (d *dirInode) LookUpChild(ctx context.Context, name string) (*Core, error) group.Go(lookUpExplicitDir) } } - } - if err := group.Wait(); err != nil { + if err = group.Wait(); err != nil { return nil, err } - var result *Core if dirResult != nil { - result = dirResult - } else if fileResult != nil { - result = fileResult + return dirResult, nil } - - if !d.IsTypeCacheDeprecated() { - if result != nil { - d.cache.Insert(d.cacheClock.Now(), name, result.Type()) - } else if d.enableNonexistentTypeCache && cachedType == metadata.UnknownType { - d.cache.Insert(d.cacheClock.Now(), name, metadata.NonexistentType) - } - } - - return result, nil + return fileResult, nil } func (d *dirInode) IsUnlinked() bool { @@ -681,10 +736,11 @@ func (d *dirInode) ReadDescendants(ctx context.Context, limit int) (map[Name]*Co descendants := make(map[Name]*Core) for { listing, err := d.bucket.ListObjects(ctx, &gcs.ListObjectsRequest{ - Delimiter: "", // recursively - Prefix: d.Name().GcsObjectName(), - ContinuationToken: tok, - MaxResults: limit + 1, // to exclude itself + Delimiter: "", // recursively + Prefix: d.Name().GcsObjectName(), + ContinuationToken: tok, + MaxResults: limit + 1, // to exclude itself + IsTypeCacheDeprecated: d.isEnableTypeCacheDeprecation, }) if err != nil { return nil, fmt.Errorf("list objects: %w", err) @@ -736,6 +792,7 @@ func (d *dirInode) listObjectsAndBuildCores(ctx context.Context, tok string, max ProjectionVal: gcs.NoAcl, IncludeFoldersAsPrefixes: d.includeFoldersAsPrefixes, StartOffset: listStartOffset, + IsTypeCacheDeprecated: d.isEnableTypeCacheDeprecation, } listing, err := d.bucket.ListObjects(ctx, req) @@ -1209,6 +1266,7 @@ func (d *dirInode) deletePrefixRecursively(ctx context.Context, prefix string) e Delimiter: "/", // Use Delimiter to separate nested folders (CollapsedRuns) ContinuationToken: tok, IncludeFoldersAsPrefixes: d.includeFoldersAsPrefixes, + IsTypeCacheDeprecated: d.isEnableTypeCacheDeprecation, }) if err != nil { return fmt.Errorf("listing objects under prefix %q: %w", prefix, err) diff --git a/internal/fs/inode/dir_test.go b/internal/fs/inode/dir_test.go index 8227ddb796..93300c7a10 100644 --- a/internal/fs/inode/dir_test.go +++ b/internal/fs/inode/dir_test.go @@ -27,11 +27,14 @@ import ( "github.com/googlecloudplatform/gcsfuse/v3/cfg" "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" "github.com/googlecloudplatform/gcsfuse/v3/internal/contentcache" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching" "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake" "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" + storagemock "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/mock" "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil" "github.com/googlecloudplatform/gcsfuse/v3/internal/util" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "golang.org/x/net/context" @@ -151,8 +154,11 @@ func (t *DirTest) createDirInode(dirInodeName string) DirInode { func (t *DirTest) createDirInodeWithTypeCacheDeprecationFlag(dirInodeName string, isTypeCacheDeprecated bool) DirInode { config := &cfg.Config{ - List: cfg.ListConfig{EnableEmptyManagedFolders: false}, - MetadataCache: cfg.MetadataCacheConfig{TypeCacheMaxSizeMb: 4}, + List: cfg.ListConfig{EnableEmptyManagedFolders: false}, + MetadataCache: cfg.MetadataCacheConfig{ + TypeCacheMaxSizeMb: 4, + TtlSecs: 60, + }, EnableHns: false, EnableUnsupportedPathSupport: true, EnableTypeCacheDeprecation: isTypeCacheDeprecated, @@ -2089,6 +2095,61 @@ func (t *DirTest) Test_readObjectsUnlocked_Empty() { assert.Equal(t.T(), "", newTok) } +func (t *DirTest) TestLookUpChild_TypeCacheDeprecated_File() { + t.in.Unlock() + t.in = t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + t.in.Lock() + const name = "file" + objName := path.Join(dirInodeName, name) + _, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("content")) + require.NoError(t.T(), err) + + entry, err := t.in.LookUpChild(t.ctx, name) + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + assert.Equal(t.T(), metadata.RegularFileType, entry.Type()) +} + +func (t *DirTest) TestLookUpChild_TypeCacheDeprecated_ExplicitDir() { + t.in.Unlock() + t.in = t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + t.in.Lock() + const name = "dir" + objName := path.Join(dirInodeName, name) + "/" + _, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("")) + require.NoError(t.T(), err) + + entry, err := t.in.LookUpChild(t.ctx, name) + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + assert.Equal(t.T(), metadata.ExplicitDirType, entry.Type()) +} + +func (t *DirTest) TestLookUpChild_TypeCacheDeprecated_ImplicitDir() { + t.in.Unlock() + t.in = t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + // Enable implicit dirs + t.in.(*dirInode).implicitDirs = true + t.in.Lock() + + const name = "implicit_dir" + // Create object that implies directory + objName := path.Join(dirInodeName, name, "file") + _, err := storageutil.CreateObject(t.ctx, t.bucket, objName, []byte("content")) + require.NoError(t.T(), err) + + entry, err := t.in.LookUpChild(t.ctx, name) + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), path.Join(dirInodeName, name)+"/", entry.FullName.GcsObjectName()) + assert.Equal(t.T(), metadata.ImplicitDirType, entry.Type()) +} + func (t *DirTest) Test_IsTypeCacheDeprecated_false() { dInode := t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, false) @@ -2100,3 +2161,84 @@ func (t *DirTest) Test_IsTypeCacheDeprecated_true() { assert.True(t.T(), dInode.IsTypeCacheDeprecated()) } + +func (t *DirTest) TestLookUpChild_TypeCacheDeprecated_CacheMiss() { + mockBucket := new(storagemock.TestifyMockBucket) + mockBucket.On("BucketType").Return(gcs.BucketType{}) + syncerBucket := gcsx.NewSyncerBucket(1, ChunkTransferTimeoutSecs, ".gcsfuse_tmp/", mockBucket) + oldBucket := t.bucket + t.bucket = syncerBucket + defer func() { t.bucket = oldBucket }() + in := t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + const name = "file" + objName := path.Join(dirInodeName, name) + dirObjName := objName + "/" + cacheMissErr := &caching.CacheMissError{} + // Expect cache lookup for file -> CacheMiss + mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == objName && req.FetchOnlyFromCache == true + })).Return(nil, nil, cacheMissErr).Once() + // Expect cache lookup for dir -> CacheMiss + mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == dirObjName && req.FetchOnlyFromCache == true + })).Return(nil, nil, cacheMissErr).Once() + // Expect actual lookup for file -> Success + minObject := &gcs.MinObject{ + Name: objName, + Generation: 1, + MetaGeneration: 1, + Size: 100, + } + mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == objName && req.FetchOnlyFromCache == false + })).Return(minObject, &gcs.ExtendedObjectAttributes{}, nil).Once() + // Expect actual lookup for dir -> NotFound + mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == dirObjName && req.FetchOnlyFromCache == false + })).Return(nil, nil, &gcs.NotFoundError{}).Once() + + in.Lock() + entry, err := in.LookUpChild(t.ctx, name) + in.Unlock() + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + mockBucket.AssertExpectations(t.T()) +} + +func (t *DirTest) TestLookUpChild_TypeCacheDeprecated_CacheHit() { + mockBucket := new(storagemock.TestifyMockBucket) + mockBucket.On("BucketType").Return(gcs.BucketType{}) + syncerBucket := gcsx.NewSyncerBucket(1, ChunkTransferTimeoutSecs, ".gcsfuse_tmp/", mockBucket) + oldBucket := t.bucket + t.bucket = syncerBucket + defer func() { t.bucket = oldBucket }() + in := t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + const name = "file" + objName := path.Join(dirInodeName, name) + dirObjName := objName + "/" + // Expect cache lookup for file -> Success + minObject := &gcs.MinObject{ + Name: objName, + Generation: 1, + MetaGeneration: 1, + Size: 100, + } + mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == objName && req.FetchOnlyFromCache == true + })).Return(minObject, &gcs.ExtendedObjectAttributes{}, nil).Once() + // Expect cache lookup for dir -> NotFound (nil, nil) + mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == dirObjName && req.FetchOnlyFromCache == true + })).Return(nil, nil, &gcs.NotFoundError{}).Once() + + in.Lock() + entry, err := in.LookUpChild(t.ctx, name) + in.Unlock() + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + mockBucket.AssertExpectations(t.T()) +} diff --git a/internal/fs/inode/hns_dir_test.go b/internal/fs/inode/hns_dir_test.go index a1bb0845e7..623ba769bd 100644 --- a/internal/fs/inode/hns_dir_test.go +++ b/internal/fs/inode/hns_dir_test.go @@ -24,6 +24,7 @@ import ( "github.com/googlecloudplatform/gcsfuse/v3/cfg" "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata" + "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching" "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs" storagemock "github.com/googlecloudplatform/gcsfuse/v3/internal/storage/mock" "github.com/jacobsa/fuse/fuseops" @@ -31,6 +32,7 @@ import ( "github.com/jacobsa/timeutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "golang.org/x/sync/semaphore" @@ -132,12 +134,19 @@ func (t *hnsDirTest) resetDirInodeWithTypeCacheConfigs(implicitDirs, enableNonex } func (t *hnsDirTest) createDirInode(dirInodeName string) DirInode { + return t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, isTypeCacheDeprecationEnabled) +} + +func (t *hnsDirTest) createDirInodeWithTypeCacheDeprecationFlag(dirInodeName string, isTypeCacheDeprecated bool) DirInode { config := &cfg.Config{ - List: cfg.ListConfig{EnableEmptyManagedFolders: false}, - MetadataCache: cfg.MetadataCacheConfig{TypeCacheMaxSizeMb: 4}, - EnableHns: false, + List: cfg.ListConfig{EnableEmptyManagedFolders: false}, + MetadataCache: cfg.MetadataCacheConfig{ + TypeCacheMaxSizeMb: 4, + TtlSecs: 60, + }, + EnableHns: true, EnableUnsupportedPathSupport: true, - EnableTypeCacheDeprecation: isTypeCacheDeprecationEnabled, + EnableTypeCacheDeprecation: isTypeCacheDeprecated, } return NewDirInode( @@ -176,7 +185,7 @@ func (t *HNSDirTest) TestShouldFindExplicitHNSFolder() { t.mockBucket.On("GetFolder", mock.Anything, mock.Anything).Return(folder, nil) // Look up with the name. - result, err := findExplicitFolder(t.ctx, &t.bucket, NewDirName(t.in.Name(), name)) + result, err := findExplicitFolder(t.ctx, &t.bucket, NewDirName(t.in.Name(), name), false) t.mockBucket.AssertExpectations(t.T()) assert.Nil(t.T(), err) @@ -190,7 +199,7 @@ func (t *HNSDirTest) TestShouldReturnNilWhenGCSFolderNotFoundForInHNS() { t.mockBucket.On("GetFolder", mock.Anything, mock.Anything).Return(nil, notFoundErr) // Look up with the name. - result, err := findExplicitFolder(t.ctx, &t.bucket, NewDirName(t.in.Name(), "not-present")) + result, err := findExplicitFolder(t.ctx, &t.bucket, NewDirName(t.in.Name(), "not-present"), false) t.mockBucket.AssertExpectations(t.T()) assert.Nil(t.T(), err) @@ -629,6 +638,7 @@ func (t *HNSDirTest) TestDeleteObjects() { Delimiter: "/", ContinuationToken: "", IncludeFoldersAsPrefixes: true, + IsTypeCacheDeprecated: t.in.IsTypeCacheDeprecated(), } listResp := &gcs.Listing{ MinObjects: []*gcs.MinObject{ @@ -645,6 +655,7 @@ func (t *HNSDirTest) TestDeleteObjects() { Delimiter: "/", ContinuationToken: "", IncludeFoldersAsPrefixes: true, + IsTypeCacheDeprecated: t.in.IsTypeCacheDeprecated(), } listRespSubdir := &gcs.Listing{} t.mockBucket.On("ListObjects", mock.Anything, listReqSubdir).Return(listRespSubdir, nil) @@ -690,6 +701,7 @@ func (t *HNSDirTest) TestReadEntriesInHierarchicalBucket() { IncludeTrailingDelimiter: false, MaxResults: 5000, ProjectionVal: gcs.NoAcl, + IsTypeCacheDeprecated: t.in.IsTypeCacheDeprecated(), } t.mockBucket.On("ListObjects", t.ctx, &listObjectReq).Return(&listing, nil) @@ -797,3 +809,119 @@ func (t *NonHNSDirTest) TestDeleteChildDir_TypeCacheDeprecated() { }) } } + +func (t *HNSDirTest) TestLookUpChild_TypeCacheDeprecated_File() { + t.in.Unlock() + t.in = t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + t.in.Lock() + const name = "file" + objName := path.Join(dirInodeName, name) + minObject := &gcs.MinObject{ + Name: objName, + MetaGeneration: int64(1), + Generation: int64(2), + } + attrs := &gcs.ExtendedObjectAttributes{} + // Mock StatObject for file lookup + t.mockBucket.On("StatObject", mock.Anything, mock.Anything).Return(minObject, attrs, nil) + // Mock GetFolder for dir lookup (should return not found or nil) + notFoundErr := &gcs.NotFoundError{Err: errors.New("not found")} + t.mockBucket.On("GetFolder", mock.Anything, mock.Anything).Return(nil, notFoundErr) + + entry, err := t.in.LookUpChild(t.ctx, name) + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + assert.Equal(t.T(), metadata.RegularFileType, entry.Type()) +} + +func (t *HNSDirTest) TestLookUpChild_TypeCacheDeprecated_Folder() { + t.in.Unlock() + t.in = t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + t.in.Lock() + const name = "folder" + folderName := path.Join(dirInodeName, name) + "/" + folder := &gcs.Folder{Name: folderName} + // Mock GetFolder for dir lookup + t.mockBucket.On("GetFolder", mock.Anything, mock.Anything).Return(folder, nil) + // Mock StatObject for file lookup (should return not found) + notFoundErr := &gcs.NotFoundError{Err: errors.New("not found")} + t.mockBucket.On("StatObject", mock.Anything, mock.Anything).Return(nil, nil, notFoundErr) + + entry, err := t.in.LookUpChild(t.ctx, name) + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), folderName, entry.FullName.GcsObjectName()) + assert.Equal(t.T(), metadata.ExplicitDirType, entry.Type()) +} + +func (t *HNSDirTest) TestLookUpChild_TypeCacheDeprecated_CacheMiss() { + in := t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + const name = "file" + objName := path.Join(dirInodeName, name) + dirObjName := objName + "/" + cacheMissErr := &caching.CacheMissError{} + // Expect cache lookup for file -> CacheMiss + t.mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == objName && req.FetchOnlyFromCache == true + })).Return(nil, nil, cacheMissErr).Once() + // Expect cache lookup for dir -> CacheMiss + t.mockBucket.On("GetFolder", mock.Anything, mock.MatchedBy(func(req *gcs.GetFolderRequest) bool { + return req.Name == dirObjName && req.FetchOnlyFromCache == true + })).Return(nil, cacheMissErr).Once() + // Expect actual lookup for file -> Success + minObject := &gcs.MinObject{ + Name: objName, + Generation: 1, + MetaGeneration: 1, + Size: 100, + } + t.mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == objName && req.FetchOnlyFromCache == false + })).Return(minObject, &gcs.ExtendedObjectAttributes{}, nil).Once() + // Expect actual lookup for dir -> NotFound + t.mockBucket.On("GetFolder", mock.Anything, mock.MatchedBy(func(req *gcs.GetFolderRequest) bool { + return req.Name == dirObjName && req.FetchOnlyFromCache == false + })).Return(nil, &gcs.NotFoundError{}).Once() + + in.Lock() + entry, err := in.LookUpChild(t.ctx, name) + in.Unlock() + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + t.mockBucket.AssertExpectations(t.T()) +} + +func (t *HNSDirTest) TestLookUpChild_TypeCacheDeprecated_CacheHit() { + in := t.createDirInodeWithTypeCacheDeprecationFlag(dirInodeName, true) + const name = "file" + objName := path.Join(dirInodeName, name) + dirObjName := objName + "/" + // Expect cache lookup for file -> Success + minObject := &gcs.MinObject{ + Name: objName, + Generation: 1, + MetaGeneration: 1, + Size: 100, + } + t.mockBucket.On("StatObject", mock.Anything, mock.MatchedBy(func(req *gcs.StatObjectRequest) bool { + return req.Name == objName && req.FetchOnlyFromCache == true + })).Return(minObject, &gcs.ExtendedObjectAttributes{}, nil).Once() + // Expect cache lookup for dir -> NotFound (nil, nil) + t.mockBucket.On("GetFolder", mock.Anything, mock.MatchedBy(func(req *gcs.GetFolderRequest) bool { + return req.Name == dirObjName && req.FetchOnlyFromCache == true + })).Return(nil, &gcs.NotFoundError{}).Once() + + in.Lock() + entry, err := in.LookUpChild(t.ctx, name) + in.Unlock() + + require.NoError(t.T(), err) + require.NotNil(t.T(), entry) + assert.Equal(t.T(), objName, entry.FullName.GcsObjectName()) + t.mockBucket.AssertExpectations(t.T()) +} diff --git a/internal/gcsx/bucket_manager.go b/internal/gcsx/bucket_manager.go index 8ee4d227af..e42c47114c 100644 --- a/internal/gcsx/bucket_manager.go +++ b/internal/gcsx/bucket_manager.go @@ -78,6 +78,8 @@ type BucketConfig struct { // any data read from GCS. // All the metadata operations like object listing and stats are real. DummyIOCfg cfg.DummyIoConfig + + IsTypeCacheDeprecated bool } // BucketManager manages the lifecycle of buckets. @@ -263,7 +265,7 @@ func (bm *bucketManager) SetUpBucket( if !bm.config.DisableListAccessCheck { // Check whether this bucket works, giving the user a warning early if there // is some problem. - _, err = b.ListObjects(ctx, &gcs.ListObjectsRequest{MaxResults: 1, IncludeFoldersAsPrefixes: true, Delimiter: "/"}) + _, err = b.ListObjects(ctx, &gcs.ListObjectsRequest{MaxResults: 1, IncludeFoldersAsPrefixes: true, Delimiter: "/", IsTypeCacheDeprecated: bm.config.IsTypeCacheDeprecated}) if err != nil { return } diff --git a/internal/storage/storageutil/list_prefix.go b/internal/storage/storageutil/list_prefix.go index d68292e105..b0be44fa48 100644 --- a/internal/storage/storageutil/list_prefix.go +++ b/internal/storage/storageutil/list_prefix.go @@ -21,6 +21,8 @@ import ( "golang.org/x/net/context" ) +const IsTypeCacheDeprecated = false + // List objects in the supplied bucket whose name starts with the given prefix. // Write them into the supplied channel in an undefined order. func ListPrefix( @@ -29,7 +31,8 @@ func ListPrefix( prefix string, minObjects chan<- *gcs.MinObject) (err error) { req := &gcs.ListObjectsRequest{ - Prefix: prefix, + Prefix: prefix, + IsTypeCacheDeprecated: IsTypeCacheDeprecated, } // List until we run out.