Skip to content

Recreate referenced files that were manually deleted on the data plane during a config apply #1084

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dhurley merged 1 commit into from
May 22, 2025
Merged

Recreate referenced files that were manually deleted on the data plane during a config apply #1084

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 7 additions & 23 deletions internal/file/file_manager_service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ func (fms *FileManagerService) UpdateOverview(
delta := files.ConvertToMapOfFiles(response.GetOverview().GetFiles())

if len(delta) != 0 {
return fms.updateFiles(ctx, delta, instanceID, iteration)
return fms.updateFiles(ctx, delta, request.GetOverview().GetFiles(), instanceID, iteration)
}

return err
Expand All @@ -202,6 +202,7 @@ func (fms *FileManagerService) setupIdentifiers(ctx context.Context, iteration i
func (fms *FileManagerService) updateFiles(
ctx context.Context,
delta map[string]*mpi.File,
fileOverview []*mpi.File,
instanceID string,
iteration int,
) error {
Expand All @@ -217,7 +218,7 @@ func (fms *FileManagerService) updateFiles(
iteration++
slog.Debug("Updating file overview", "attempt_number", iteration)

return fms.UpdateOverview(ctx, instanceID, diffFiles, iteration)
return fms.UpdateOverview(ctx, instanceID, fileOverview, iteration)
}

func (fms *FileManagerService) UpdateFile(
Expand Down Expand Up @@ -483,7 +484,7 @@ func (fms *FileManagerService) DetermineFileActions(
fileDiff := make(map[string]*model.FileCache) // Files that have changed, key is file name
fileContents := make(map[string][]byte) // contents of the file, key is file name

manifestFiles, filesMap, manifestFileErr := fms.manifestFile()
_, filesMap, manifestFileErr := fms.manifestFile()

if manifestFileErr != nil {
if errors.Is(manifestFileErr, os.ErrNotExist) {
Expand Down Expand Up @@ -524,14 +525,15 @@ func (fms *FileManagerService) DetermineFileActions(
}
// if file doesn't exist in the current files, file has been added
// set file action
if !ok {
if _, statErr := os.Stat(modifiedFile.File.GetFileMeta().GetName()); errors.Is(statErr, os.ErrNotExist) {
slog.Info("File is not present on disk", "file", modifiedFile.File.GetFileMeta().GetName())
modifiedFile.Action = model.Add
fileDiff[modifiedFile.File.GetFileMeta().GetName()] = modifiedFile

continue
// if file currently exists and file hash is different, file has been updated
// copy contents, set file action
} else if modifiedFile.File.GetFileMeta().GetHash() != currentFile.GetFileMeta().GetHash() {
} else if ok && modifiedFile.File.GetFileMeta().GetHash() != currentFile.GetFileMeta().GetHash() {
fileContent, readErr := os.ReadFile(fileName)
if readErr != nil {
return nil, nil, fmt.Errorf("error reading file %s, error: %w", fileName, readErr)
Expand All @@ -540,24 +542,6 @@ func (fms *FileManagerService) DetermineFileActions(
fileContents[fileName] = fileContent
fileDiff[modifiedFile.File.GetFileMeta().GetName()] = modifiedFile
}

// If the file is unreferenced we check if the file has been updated since the last time
// Also check if the unreferenced file still exists
if manifestFiles[modifiedFile.File.GetFileMeta().GetName()] != nil &&
!manifestFiles[modifiedFile.File.GetFileMeta().GetName()].ManifestFileMeta.Referenced {
if fileStats, err := os.Stat(modifiedFile.File.GetFileMeta().GetName()); errors.Is(err, os.ErrNotExist) {
modifiedFile.Action = model.Add
fileDiff[modifiedFile.File.GetFileMeta().GetName()] = modifiedFile
} else if timestamppb.New(fileStats.ModTime()) != modifiedFile.File.GetFileMeta().GetModifiedTime() {
fileContent, readErr := os.ReadFile(fileName)
if readErr != nil {
return nil, nil, fmt.Errorf("error reading file %s, error: %w", fileName, readErr)
}
modifiedFile.Action = model.Update
fileContents[fileName] = fileContent
fileDiff[modifiedFile.File.GetFileMeta().GetName()] = modifiedFile
}
}
}

return fileDiff, fileContents, nil
Expand Down
20 changes: 14 additions & 6 deletions internal/file/file_manager_service_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -490,14 +490,21 @@ func TestFileManagerService_DetermineFileActions(t *testing.T) {
updateErr := os.WriteFile(updateTestFile.Name(), updatedFileContent, 0o600)
require.NoError(t, updateErr)

addTestFileName := tempDir + "/nginx_add.conf"
addTestFileName := tempDir + "nginx_add.conf"

unmanagedFile := helpers.CreateFileWithErrorCheck(t, tempDir, "nginx_unmanaged.conf")
defer helpers.RemoveFileWithErrorCheck(t, unmanagedFile.Name())
unmanagedFileContent := []byte("test unmanaged file")
unmanagedErr := os.WriteFile(unmanagedFile.Name(), unmanagedFileContent, 0o600)
require.NoError(t, unmanagedErr)

addTestFile := helpers.CreateFileWithErrorCheck(t, tempDir, "nginx_add.conf")
defer helpers.RemoveFileWithErrorCheck(t, addTestFile.Name())
t.Logf("Adding file: %s", addTestFile.Name())
addFileContent := []byte("test add file")
addErr := os.WriteFile(addTestFile.Name(), addFileContent, 0o600)
require.NoError(t, addErr)

tests := []struct {
expectedError error
modifiedFiles map[string]*model.FileCache
Expand Down Expand Up @@ -572,9 +579,9 @@ func TestFileManagerService_DetermineFileActions(t *testing.T) {
{
name: "Test 2: Files same as on disk",
modifiedFiles: map[string]*model.FileCache{
addTestFileName: {
addTestFile.Name(): {
File: &mpi.File{
FileMeta: protos.FileMeta(addTestFileName, files.GenerateHash(fileContent)),
FileMeta: protos.FileMeta(addTestFile.Name(), files.GenerateHash(fileContent)),
},
},
updateTestFile.Name(): {
Expand All @@ -595,8 +602,8 @@ func TestFileManagerService_DetermineFileActions(t *testing.T) {
updateTestFile.Name(): {
FileMeta: protos.FileMeta(updateTestFile.Name(), files.GenerateHash(fileContent)),
},
addTestFileName: {
FileMeta: protos.FileMeta(addTestFileName, files.GenerateHash(fileContent)),
addTestFile.Name(): {
FileMeta: protos.FileMeta(addTestFile.Name(), files.GenerateHash(fileContent)),
},
},
expectedCache: make(map[string]*model.FileCache),
Expand All @@ -612,10 +619,11 @@ func TestFileManagerService_DetermineFileActions(t *testing.T) {
defer manifestFile.Close()
manifestDirPath = tempDir
manifestFilePath = manifestFile.Name()
t.Logf("path: %s", manifestFilePath)

fakeFileServiceClient := &v1fakes.FakeFileServiceClient{}
fileManagerService := NewFileManagerService(fakeFileServiceClient, types.AgentConfig())
require.NoError(tt, err)

diff, contents, fileActionErr := fileManagerService.DetermineFileActions(test.currentFiles,
test.modifiedFiles)
require.NoError(tt, fileActionErr)
Expand Down
8 changes: 3 additions & 5 deletions internal/watcher/watcher_plugin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,11 +198,9 @@ func (w *Watcher) handleConfigApplySuccess(ctx context.Context, msg *bus.Message

// If the config apply had no changes to any files, it is results in a ConfigApplySuccessfulTopic with an empty
// configContext being sent, there is no need to reparse the config as no change has occurred.
if successMessage.ConfigContext.InstanceID == "" {
slog.DebugContext(ctx, "NginxConfigContext is empty, no need to reparse config")
return
if successMessage.ConfigContext.InstanceID != "" {
w.instanceWatcherService.HandleNginxConfigContextUpdate(ctx, instanceID, successMessage.ConfigContext)
}
w.instanceWatcherService.HandleNginxConfigContextUpdate(ctx, instanceID, successMessage.ConfigContext)

w.watcherMutex.Lock()
w.instancesWithConfigApplyInProgress = slices.DeleteFunc(
Expand All @@ -213,8 +211,8 @@ func (w *Watcher) handleConfigApplySuccess(ctx context.Context, msg *bus.Message
)

w.fileWatcherService.SetEnabled(true)
w.watcherMutex.Unlock()
w.instanceWatcherService.SetEnabled(true)
w.watcherMutex.Unlock()
}

func (w *Watcher) handleHealthRequest(ctx context.Context) {
Expand Down