diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 4c51eb006..47a0ef0eb 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -97,6 +97,7 @@ func DefaultAllowedDirectories() []string { func DefaultExcludedFiles() []string { return []string{ "^.*(\\.log|.swx|~|.swp)$", + "/var/lib/nginx-agent/manifest.json", } } diff --git a/internal/file/file_manager_service.go b/internal/file/file_manager_service.go index 4cdf39b06..b8f8090a6 100644 --- a/internal/file/file_manager_service.go +++ b/internal/file/file_manager_service.go @@ -37,11 +37,17 @@ import ( //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6@v6.8.1 -generate //counterfeiter:generate . fileManagerServiceInterface -const maxAttempts = 5 +const ( + maxAttempts = 5 + dirPerm = 0o755 + filePerm = 0o600 +) type ( fileOperator interface { Write(ctx context.Context, fileContent []byte, file *mpi.FileMeta) error + ManifestFile(currentFiles map[string]*mpi.File) (map[string]*mpi.File, error) + UpdateManifestFile(currentFiles map[string]*mpi.File) (err error) } fileManagerServiceInterface interface { @@ -52,8 +58,9 @@ type ( UpdateFile(ctx context.Context, instanceID string, fileToUpdate *mpi.File) error ClearCache() UpdateCurrentFilesOnDisk(updateFiles map[string]*mpi.File) - DetermineFileActions(currentFiles, modifiedFiles map[string]*mpi.File) (map[string]*mpi.File, - map[string][]byte, error) + UpdateManifestFile(fileToUpdate map[string]*mpi.File) error + DetermineFileActions(currentFiles map[string]*mpi.File, modifiedFiles map[string]*model.FileCache) ( + map[string]*model.FileCache, map[string][]byte, error) IsConnected() bool SetIsConnected(isConnected bool) } @@ -65,7 +72,7 @@ type FileManagerService struct { isConnected *atomic.Bool fileOperator fileOperator // map of files and the actions performed on them during config apply - fileActions map[string]*mpi.File // key is file path + fileActions map[string]*model.FileCache // key is file path // map of the contents of files which have been updated or deleted during config apply, used during rollback rollbackFileContents map[string][]byte // key is file path // map of the files currently on disk, used to determine the file action during config apply @@ -81,7 +88,7 @@ func NewFileManagerService(fileServiceClient mpi.FileServiceClient, agentConfig fileServiceClient: fileServiceClient, agentConfig: agentConfig, fileOperator: NewFileOperator(), - fileActions: make(map[string]*mpi.File), + fileActions: make(map[string]*model.FileCache), rollbackFileContents: make(map[string][]byte), currentFilesOnDisk: make(map[string]*mpi.File), isConnected: isConnected, @@ -281,8 +288,7 @@ func (fms *FileManagerService) SetIsConnected(isConnected bool) { fms.isConnected.Store(isConnected) } -func (fms *FileManagerService) ConfigApply(ctx context.Context, - configApplyRequest *mpi.ConfigApplyRequest, +func (fms *FileManagerService) ConfigApply(ctx context.Context, configApplyRequest *mpi.ConfigApplyRequest, ) (status model.WriteStatus, err error) { fileOverview := configApplyRequest.GetOverview() @@ -296,7 +302,7 @@ func (fms *FileManagerService) ConfigApply(ctx context.Context, } diffFiles, fileContent, compareErr := fms.DetermineFileActions(fms.currentFilesOnDisk, - files.ConvertToMapOfFiles(fileOverview.GetFiles())) + ConvertToMapOfFileCache(fileOverview.GetFiles())) if compareErr != nil { return model.Error, compareErr @@ -313,72 +319,101 @@ func (fms *FileManagerService) ConfigApply(ctx context.Context, if fileErr != nil { return model.RollbackRequired, fileErr } - + fileOverviewFiles := files.ConvertToMapOfFiles(fileOverview.GetFiles()) // Update map of current files on disk - fms.UpdateCurrentFilesOnDisk(files.ConvertToMapOfFiles(fileOverview.GetFiles())) + fms.UpdateCurrentFilesOnDisk(fileOverviewFiles) + manifestFileErr := fms.fileOperator.UpdateManifestFile(fileOverviewFiles) + if manifestFileErr != nil { + return model.RollbackRequired, manifestFileErr + } return model.OK, nil } +func ConvertToMapOfFileCache(convertFiles []*mpi.File) map[string]*model.FileCache { + filesMap := make(map[string]*model.FileCache) + for _, convertFile := range convertFiles { + filesMap[convertFile.GetFileMeta().GetName()] = &model.FileCache{ + File: convertFile, + } + } + + return filesMap +} + func (fms *FileManagerService) ClearCache() { clear(fms.rollbackFileContents) clear(fms.fileActions) } +func (fms *FileManagerService) UpdateManifestFile(fileToUpdate map[string]*mpi.File) error { + return fms.fileOperator.UpdateManifestFile(fileToUpdate) +} + +// nolint:revive,cyclop func (fms *FileManagerService) Rollback(ctx context.Context, instanceID string) error { - slog.InfoContext(ctx, "Rolling back config for instance", "instanceid", instanceID) + slog.InfoContext(ctx, "Rolling back config for instance", "instance_id", instanceID) + areFilesUpdated := false fms.filesMutex.Lock() defer fms.filesMutex.Unlock() - for _, file := range fms.fileActions { - switch file.GetAction() { - case mpi.File_FILE_ACTION_ADD: - if err := os.Remove(file.GetFileMeta().GetName()); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("error deleting file: %s error: %w", file.GetFileMeta().GetName(), err) + for _, fileAction := range fms.fileActions { + switch fileAction.Action { + case model.Add: + if err := os.Remove(fileAction.File.GetFileMeta().GetName()); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error deleting file: %s error: %w", fileAction.File.GetFileMeta().GetName(), err) } // currentFilesOnDisk needs to be updated after rollback action is performed - delete(fms.currentFilesOnDisk, file.GetFileMeta().GetName()) + delete(fms.currentFilesOnDisk, fileAction.File.GetFileMeta().GetName()) + areFilesUpdated = true continue - case mpi.File_FILE_ACTION_DELETE, mpi.File_FILE_ACTION_UPDATE: - content := fms.rollbackFileContents[file.GetFileMeta().GetName()] - - err := fms.fileOperator.Write(ctx, content, file.GetFileMeta()) + case model.Delete, model.Update: + content := fms.rollbackFileContents[fileAction.File.GetFileMeta().GetName()] + err := fms.fileOperator.Write(ctx, content, fileAction.File.GetFileMeta()) if err != nil { return err } // currentFilesOnDisk needs to be updated after rollback action is performed - file.GetFileMeta().Hash = files.GenerateHash(content) - fms.currentFilesOnDisk[file.GetFileMeta().GetName()] = file - case mpi.File_FILE_ACTION_UNSPECIFIED, mpi.File_FILE_ACTION_UNCHANGED: + fileAction.File.GetFileMeta().Hash = files.GenerateHash(content) + fms.currentFilesOnDisk[fileAction.File.GetFileMeta().GetName()] = fileAction.File + areFilesUpdated = true + case model.Unchanged: fallthrough default: slog.DebugContext(ctx, "File Action not implemented") } } + if areFilesUpdated { + manifestFileErr := fms.fileOperator.UpdateManifestFile(fms.currentFilesOnDisk) + if manifestFileErr != nil { + return manifestFileErr + } + } + return nil } func (fms *FileManagerService) executeFileActions(ctx context.Context) error { - for _, file := range fms.fileActions { - switch file.GetAction() { - case mpi.File_FILE_ACTION_DELETE: - if err := os.Remove(file.GetFileMeta().GetName()); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("error deleting file: %s error: %w", file.GetFileMeta().GetName(), err) + for _, fileAction := range fms.fileActions { + switch fileAction.Action { + case model.Delete: + if err := os.Remove(fileAction.File.GetFileMeta().GetName()); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error deleting file: %s error: %w", fileAction.File.GetFileMeta().GetName(), err) } continue - case mpi.File_FILE_ACTION_ADD, mpi.File_FILE_ACTION_UPDATE: - updateErr := fms.fileUpdate(ctx, file) + case model.Add, model.Update: + updateErr := fms.fileUpdate(ctx, fileAction.File) if updateErr != nil { return updateErr } - case mpi.File_FILE_ACTION_UNSPECIFIED, mpi.File_FILE_ACTION_UNCHANGED: + case model.Unchanged: fallthrough default: - slog.DebugContext(ctx, "File Action not implemented", "action", file.GetAction()) + slog.DebugContext(ctx, "File Action not implemented", "action", fileAction.File.GetAction()) } } @@ -426,7 +461,7 @@ func (fms *FileManagerService) validateFileHash(filePath string) error { } fileHash := files.GenerateHash(content) - if fileHash != fms.fileActions[filePath].GetFileMeta().GetHash() { + if fileHash != fms.fileActions[filePath].File.GetFileMeta().GetHash() { return fmt.Errorf("error writing file, file hash does not match for file %s", filePath) } @@ -446,63 +481,65 @@ func (fms *FileManagerService) checkAllowedDirectory(checkFiles []*mpi.File) err // DetermineFileActions compares two sets of files to determine the file action for each file. Returns a map of files // that have changed and a map of the contents for each updated and deleted file. Key to both maps is file path -// nolint: revive -func (fms *FileManagerService) DetermineFileActions(currentFiles, modifiedFiles map[string]*mpi.File) ( - map[string]*mpi.File, map[string][]byte, error, +// nolint: revive,cyclop +func (fms *FileManagerService) DetermineFileActions(currentFiles map[string]*mpi.File, + modifiedFiles map[string]*model.FileCache) (map[string]*model.FileCache, map[string][]byte, error, ) { fms.filesMutex.Lock() defer fms.filesMutex.Unlock() - // Go doesn't allow address of numeric constant - addAction := mpi.File_FILE_ACTION_ADD - updateAction := mpi.File_FILE_ACTION_UPDATE - deleteAction := mpi.File_FILE_ACTION_DELETE - unchangedAction := mpi.File_FILE_ACTION_UNCHANGED - fileDiff := make(map[string]*mpi.File) // Files that have changed, key is file name - fileContents := make(map[string][]byte) // contents of the file, key is file name + fileDiff := make(map[string]*model.FileCache) // Files that have changed, key is file name + fileContents := make(map[string][]byte) // contents of the file, key is file name + + manifestFiles, manifestFileErr := fms.fileOperator.ManifestFile(currentFiles) - // if file is in currentFiles but not in modified files, file has been deleted + if manifestFileErr != nil && manifestFiles == nil { + return nil, nil, manifestFileErr + } + // if file is in manifestFiles but not in modified files, file has been deleted // copy contents, set file action - for _, currentFile := range currentFiles { - fileName := currentFile.GetFileMeta().GetName() - _, ok := modifiedFiles[fileName] + for fileName, currentFile := range manifestFiles { + _, exists := modifiedFiles[fileName] - if !ok { + if !exists { + // Read file contents before marking it deleted fileContent, readErr := os.ReadFile(fileName) if readErr != nil { return nil, nil, fmt.Errorf("error reading file %s, error: %w", fileName, readErr) } fileContents[fileName] = fileContent - currentFile.Action = &deleteAction - fileDiff[currentFile.GetFileMeta().GetName()] = currentFile + fileDiff[fileName] = &model.FileCache{ + File: currentFile, + Action: model.Delete, + } } } for _, file := range modifiedFiles { - fileName := file.GetFileMeta().GetName() - currentFile, ok := currentFiles[file.GetFileMeta().GetName()] + fileName := file.File.GetFileMeta().GetName() + currentFile, ok := manifestFiles[file.File.GetFileMeta().GetName()] // default to unchanged action - file.Action = &unchangedAction + file.Action = model.Unchanged // if file is unmanaged, action is set to unchanged so file is skipped when performing actions - if file.GetUnmanaged() { + if file.File.GetUnmanaged() { continue } // if file doesn't exist in the current files, file has been added // set file action if !ok { - file.Action = &addAction - fileDiff[file.GetFileMeta().GetName()] = file + file.Action = model.Add + fileDiff[file.File.GetFileMeta().GetName()] = file // if file currently exists and file hash is different, file has been updated // copy contents, set file action - } else if file.GetFileMeta().GetHash() != currentFile.GetFileMeta().GetHash() { + } else if file.File.GetFileMeta().GetHash() != currentFile.GetFileMeta().GetHash() { fileContent, readErr := os.ReadFile(fileName) if readErr != nil { return nil, nil, fmt.Errorf("error reading file %s, error: %w", fileName, readErr) } - file.Action = &updateAction + file.Action = model.Update fileContents[fileName] = fileContent - fileDiff[file.GetFileMeta().GetName()] = file + fileDiff[file.File.GetFileMeta().GetName()] = file } } @@ -517,7 +554,7 @@ func (fms *FileManagerService) UpdateCurrentFilesOnDisk(currentFiles map[string] clear(fms.currentFilesOnDisk) - for _, file := range currentFiles { - fms.currentFilesOnDisk[file.GetFileMeta().GetName()] = file + for _, currentFile := range currentFiles { + fms.currentFilesOnDisk[currentFile.GetFileMeta().GetName()] = currentFile } } diff --git a/internal/file/file_manager_service_test.go b/internal/file/file_manager_service_test.go index c0da7ad99..84fbb027a 100644 --- a/internal/file/file_manager_service_test.go +++ b/internal/file/file_manager_service_test.go @@ -153,6 +153,10 @@ func TestFileManagerService_ConfigApply_Add(t *testing.T) { overview := protos.FileOverview(filePath, fileHash) + manifestDirPath = tempDir + manifestFilePath = manifestDirPath + "/manifest.json" + helpers.CreateFileWithErrorCheck(t, manifestDirPath, "manifest.json") + fakeFileServiceClient := &v1fakes.FakeFileServiceClient{} fakeFileServiceClient.GetOverviewReturns(&mpi.GetOverviewResponse{ Overview: overview, @@ -173,7 +177,7 @@ func TestFileManagerService_ConfigApply_Add(t *testing.T) { data, readErr := os.ReadFile(filePath) require.NoError(t, readErr) assert.Equal(t, fileContent, data) - assert.Equal(t, fileManagerService.fileActions[filePath], overview.GetFiles()[0]) + assert.Equal(t, fileManagerService.fileActions[filePath].File, overview.GetFiles()[0]) } func TestFileManagerService_ConfigApply_Update(t *testing.T) { @@ -201,6 +205,10 @@ func TestFileManagerService_ConfigApply_Update(t *testing.T) { }, } + manifestDirPath = tempDir + manifestFilePath = manifestDirPath + "/manifest.json" + helpers.CreateFileWithErrorCheck(t, manifestDirPath, "manifest.json") + overview := protos.FileOverview(tempFile.Name(), fileHash) fakeFileServiceClient := &v1fakes.FakeFileServiceClient{} @@ -226,7 +234,7 @@ func TestFileManagerService_ConfigApply_Update(t *testing.T) { require.NoError(t, readErr) assert.Equal(t, fileContent, data) assert.Equal(t, fileManagerService.rollbackFileContents[tempFile.Name()], previousFileContent) - assert.Equal(t, fileManagerService.fileActions[tempFile.Name()], overview.GetFiles()[0]) + assert.Equal(t, fileManagerService.fileActions[tempFile.Name()].File, overview.GetFiles()[0]) } func TestFileManagerService_ConfigApply_Delete(t *testing.T) { @@ -253,6 +261,10 @@ func TestFileManagerService_ConfigApply_Delete(t *testing.T) { }, } + manifestDirPath = tempDir + manifestFilePath = manifestDirPath + "/manifest.json" + helpers.CreateFileWithErrorCheck(t, manifestDirPath, "manifest.json") + fakeFileServiceClient := &v1fakes.FakeFileServiceClient{} agentConfig := types.AgentConfig() agentConfig.AllowedDirectories = []string{tempDir} @@ -274,7 +286,7 @@ func TestFileManagerService_ConfigApply_Delete(t *testing.T) { require.NoError(t, err) assert.NoFileExists(t, tempFile.Name()) assert.Equal(t, fileManagerService.rollbackFileContents[tempFile.Name()], fileContent) - assert.Equal(t, fileManagerService.fileActions[tempFile.Name()], filesOnDisk[tempFile.Name()]) + assert.Equal(t, fileManagerService.fileActions[tempFile.Name()].File, filesOnDisk[tempFile.Name()]) assert.Equal(t, model.OK, writeStatus) } @@ -291,7 +303,6 @@ func TestFileManagerService_checkAllowedDirectory(t *testing.T) { Permissions: "", Size: 0, }, - Action: nil, }, } @@ -304,7 +315,6 @@ func TestFileManagerService_checkAllowedDirectory(t *testing.T) { Permissions: "", Size: 0, }, - Action: nil, }, } @@ -318,15 +328,16 @@ func TestFileManagerService_ClearCache(t *testing.T) { fakeFileServiceClient := &v1fakes.FakeFileServiceClient{} fileManagerService := NewFileManagerService(fakeFileServiceClient, types.AgentConfig()) - filesCache := map[string]*mpi.File{ + filesCache := map[string]*model.FileCache{ "file/path/test.conf": { - FileMeta: &mpi.FileMeta{ + File: &mpi.File{FileMeta: &mpi.FileMeta{ Name: "file/path/test.conf", Hash: "", ModifiedTime: nil, Permissions: "", Size: 0, - }, + }}, + Action: model.Update, }, } @@ -349,11 +360,6 @@ func TestFileManagerService_Rollback(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - addAction := mpi.File_FILE_ACTION_ADD - deleteAction := mpi.File_FILE_ACTION_DELETE - updateAction := mpi.File_FILE_ACTION_UPDATE - unspecifiedAction := mpi.File_FILE_ACTION_UNSPECIFIED - deleteFilePath := filepath.Join(tempDir, "nginx_delete.conf") newFileContent := []byte("location /test {\n return 200 \"This config needs to be rolled back\\n\";\n}") @@ -368,46 +374,58 @@ func TestFileManagerService_Rollback(t *testing.T) { _, writeErr = updateFile.Write(newFileContent) require.NoError(t, writeErr) - filesCache := map[string]*mpi.File{ + manifestDirPath = tempDir + manifestFilePath = manifestDirPath + "/manifest.json" + helpers.CreateFileWithErrorCheck(t, manifestDirPath, "manifest.json") + + filesCache := map[string]*model.FileCache{ addFile.Name(): { - FileMeta: &mpi.FileMeta{ - Name: addFile.Name(), - Hash: fileHash, - ModifiedTime: timestamppb.Now(), - Permissions: "0777", - Size: 0, + File: &mpi.File{ + FileMeta: &mpi.FileMeta{ + Name: addFile.Name(), + Hash: fileHash, + ModifiedTime: timestamppb.Now(), + Permissions: "0777", + Size: 0, + }, }, - Action: &addAction, + Action: model.Add, }, updateFile.Name(): { - FileMeta: &mpi.FileMeta{ - Name: updateFile.Name(), - Hash: fileHash, - ModifiedTime: timestamppb.Now(), - Permissions: "0777", - Size: 0, + File: &mpi.File{ + FileMeta: &mpi.FileMeta{ + Name: updateFile.Name(), + Hash: fileHash, + ModifiedTime: timestamppb.Now(), + Permissions: "0777", + Size: 0, + }, }, - Action: &updateAction, + Action: model.Update, }, deleteFilePath: { - FileMeta: &mpi.FileMeta{ - Name: deleteFilePath, - Hash: "", - ModifiedTime: timestamppb.Now(), - Permissions: "0777", - Size: 0, + File: &mpi.File{ + FileMeta: &mpi.FileMeta{ + Name: deleteFilePath, + Hash: "", + ModifiedTime: timestamppb.Now(), + Permissions: "0777", + Size: 0, + }, }, - Action: &deleteAction, + Action: model.Delete, }, "unspecified/file/test.conf": { - FileMeta: &mpi.FileMeta{ - Name: "unspecified/file/test.conf", - Hash: "", - ModifiedTime: timestamppb.Now(), - Permissions: "0777", - Size: 0, + File: &mpi.File{ + FileMeta: &mpi.FileMeta{ + Name: "unspecified/file/test.conf", + Hash: "", + ModifiedTime: timestamppb.Now(), + Permissions: "0777", + Size: 0, + }, }, - Action: &unspecifiedAction, + Action: model.Unchanged, }, } @@ -440,11 +458,6 @@ func TestFileManagerService_Rollback(t *testing.T) { } func TestFileManagerService_DetermineFileActions(t *testing.T) { - // Go doesn't allow address of numeric constant - addAction := mpi.File_FILE_ACTION_ADD - updateAction := mpi.File_FILE_ACTION_UPDATE - deleteAction := mpi.File_FILE_ACTION_DELETE - tempDir := os.TempDir() deleteTestFile := helpers.CreateFileWithErrorCheck(t, tempDir, "nginx_delete.conf") @@ -468,26 +481,33 @@ func TestFileManagerService_DetermineFileActions(t *testing.T) { unmanagedErr := os.WriteFile(unmanagedFile.Name(), unmanagedFileContent, 0o600) require.NoError(t, unmanagedErr) + manifestDirPath = tempDir + manifestFilePath = manifestDirPath + "/manifest.json" + helpers.CreateFileWithErrorCheck(t, manifestDirPath, "manifest.json") + tests := []struct { expectedError error - modifiedFiles map[string]*mpi.File + modifiedFiles map[string]*model.FileCache currentFiles map[string]*mpi.File - expectedCache map[string]*mpi.File + expectedCache map[string]*model.FileCache expectedContent map[string][]byte name string }{ { name: "Test 1: Add, Update & Delete Files", - modifiedFiles: map[string]*mpi.File{ + modifiedFiles: map[string]*model.FileCache{ addTestFileName: { - FileMeta: protos.FileMeta(addTestFileName, files.GenerateHash(fileContent)), + File: &mpi.File{FileMeta: protos.FileMeta(addTestFileName, files.GenerateHash(fileContent))}, }, updateTestFile.Name(): { - FileMeta: protos.FileMeta(updateTestFile.Name(), files.GenerateHash(updatedFileContent)), + File: &mpi.File{FileMeta: protos.FileMeta(updateTestFile.Name(), + files.GenerateHash(updatedFileContent))}, }, unmanagedFile.Name(): { - FileMeta: protos.FileMeta(unmanagedFile.Name(), files.GenerateHash(unmanagedFileContent)), - Unmanaged: true, + File: &mpi.File{ + FileMeta: protos.FileMeta(unmanagedFile.Name(), files.GenerateHash(unmanagedFileContent)), + Unmanaged: true, + }, }, }, currentFiles: map[string]*mpi.File{ @@ -502,18 +522,21 @@ func TestFileManagerService_DetermineFileActions(t *testing.T) { Unmanaged: true, }, }, - expectedCache: map[string]*mpi.File{ + expectedCache: map[string]*model.FileCache{ deleteTestFile.Name(): { - FileMeta: protos.FileMeta(deleteTestFile.Name(), files.GenerateHash(fileContent)), - Action: &deleteAction, + File: &mpi.File{FileMeta: protos.FileMeta(deleteTestFile.Name(), + files.GenerateHash(fileContent))}, + Action: model.Delete, }, updateTestFile.Name(): { - FileMeta: protos.FileMeta(updateTestFile.Name(), files.GenerateHash(updatedFileContent)), - Action: &updateAction, + File: &mpi.File{FileMeta: protos.FileMeta(updateTestFile.Name(), + files.GenerateHash(updatedFileContent))}, + Action: model.Update, }, addTestFileName: { - FileMeta: protos.FileMeta(addTestFileName, files.GenerateHash(fileContent)), - Action: &addAction, + File: &mpi.File{FileMeta: protos.FileMeta(addTestFileName, + files.GenerateHash(fileContent))}, + Action: model.Add, }, }, expectedContent: map[string][]byte{ @@ -524,15 +547,15 @@ func TestFileManagerService_DetermineFileActions(t *testing.T) { }, { name: "Test 2: Files same as on disk", - modifiedFiles: map[string]*mpi.File{ + modifiedFiles: map[string]*model.FileCache{ addTestFileName: { - FileMeta: protos.FileMeta(addTestFileName, files.GenerateHash(fileContent)), + File: &mpi.File{FileMeta: protos.FileMeta(addTestFileName, files.GenerateHash(fileContent))}, }, updateTestFile.Name(): { - FileMeta: protos.FileMeta(updateTestFile.Name(), files.GenerateHash(fileContent)), + File: &mpi.File{FileMeta: protos.FileMeta(updateTestFile.Name(), files.GenerateHash(fileContent))}, }, deleteTestFile.Name(): { - FileMeta: protos.FileMeta(deleteTestFile.Name(), files.GenerateHash(fileContent)), + File: &mpi.File{FileMeta: protos.FileMeta(deleteTestFile.Name(), files.GenerateHash(fileContent))}, }, }, currentFiles: map[string]*mpi.File{ @@ -546,7 +569,7 @@ func TestFileManagerService_DetermineFileActions(t *testing.T) { FileMeta: protos.FileMeta(addTestFileName, files.GenerateHash(fileContent)), }, }, - expectedCache: make(map[string]*mpi.File), + expectedCache: make(map[string]*model.FileCache), expectedContent: make(map[string][]byte), expectedError: nil, }, @@ -556,9 +579,10 @@ func TestFileManagerService_DetermineFileActions(t *testing.T) { t.Run(test.name, func(tt *testing.T) { fakeFileServiceClient := &v1fakes.FakeFileServiceClient{} fileManagerService := NewFileManagerService(fakeFileServiceClient, types.AgentConfig()) + err = fileManagerService.fileOperator.UpdateManifestFile(test.currentFiles) + require.NoError(tt, err) diff, contents, fileActionErr := fileManagerService.DetermineFileActions(test.currentFiles, test.modifiedFiles) - require.NoError(tt, fileActionErr) assert.Equal(tt, test.expectedContent, contents) assert.Equal(tt, test.expectedCache, diff) @@ -570,11 +594,6 @@ func TestFileManagerService_fileActions(t *testing.T) { ctx := context.Background() tempDir := t.TempDir() - addAction := mpi.File_FILE_ACTION_ADD - deleteAction := mpi.File_FILE_ACTION_DELETE - updateAction := mpi.File_FILE_ACTION_UPDATE - unspecifiedAction := mpi.File_FILE_ACTION_UNSPECIFIED - addFilePath := filepath.Join(tempDir, "nginx_add.conf") unspecifiedFilePath := "unspecified/file/test.conf" @@ -590,46 +609,50 @@ func TestFileManagerService_fileActions(t *testing.T) { _, writeErr = updateFile.Write(oldFileContent) require.NoError(t, writeErr) - filesCache := map[string]*mpi.File{ + filesCache := map[string]*model.FileCache{ addFilePath: { - FileMeta: &mpi.FileMeta{ - Name: addFilePath, - Hash: fileHash, - ModifiedTime: timestamppb.Now(), - Permissions: "0777", - Size: 0, + File: &mpi.File{ + FileMeta: &mpi.FileMeta{ + Name: addFilePath, + Hash: fileHash, + ModifiedTime: timestamppb.Now(), + Permissions: "0777", + Size: 0, + }, }, - Action: &addAction, + Action: model.Add, }, updateFile.Name(): { - FileMeta: &mpi.FileMeta{ + File: &mpi.File{FileMeta: &mpi.FileMeta{ Name: updateFile.Name(), Hash: fileHash, ModifiedTime: timestamppb.Now(), Permissions: "0777", Size: 0, - }, - Action: &updateAction, + }}, + Action: model.Update, }, deleteFile.Name(): { - FileMeta: &mpi.FileMeta{ - Name: deleteFile.Name(), - Hash: "", - ModifiedTime: timestamppb.Now(), - Permissions: "0777", - Size: 0, + File: &mpi.File{ + FileMeta: &mpi.FileMeta{ + Name: deleteFile.Name(), + Hash: "", + ModifiedTime: timestamppb.Now(), + Permissions: "0777", + Size: 0, + }, }, - Action: &deleteAction, + Action: model.Delete, }, unspecifiedFilePath: { - FileMeta: &mpi.FileMeta{ + File: &mpi.File{FileMeta: &mpi.FileMeta{ Name: unspecifiedFilePath, Hash: "", ModifiedTime: timestamppb.Now(), Permissions: "0777", Size: 0, - }, - Action: &unspecifiedAction, + }}, + Action: model.Unchanged, }, } diff --git a/internal/file/file_operator.go b/internal/file/file_operator.go index 06c174b90..c63ed6cd1 100644 --- a/internal/file/file_operator.go +++ b/internal/file/file_operator.go @@ -7,6 +7,7 @@ package file import ( "context" + "encoding/json" "fmt" "log/slog" "os" @@ -21,6 +22,11 @@ type FileOperator struct{} var _ fileOperator = (*FileOperator)(nil) +var ( + manifestDirPath = "/var/lib/nginx-agent" + manifestFilePath = manifestDirPath + "/manifest.json" +) + // FileOperator only purpose is to write files, func NewFileOperator() *FileOperator { @@ -45,3 +51,75 @@ func (fo *FileOperator) Write(ctx context.Context, fileContent []byte, file *mpi return nil } + +// nolint: musttag +func (fo *FileOperator) UpdateManifestFile(currentFiles map[string]*mpi.File) (err error) { + slog.Info("Updating NGINX config manifest file", "current_files", currentFiles) + + manifestJSON, err := json.MarshalIndent(currentFiles, "", " ") + if err != nil { + slog.Error("Unable to marshal manifest file json ", "err", err) + return err + } + + // 0755 allows read/execute for all, write for owner + if err = os.MkdirAll(manifestDirPath, dirPerm); err != nil { + slog.Error("Unable to create directory", "err", err) + return err + } + + // 0600 ensures only root can read/write + newFile, err := os.OpenFile(manifestFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, filePerm) + if err != nil { + slog.Error("Failed to read manifest file", "error", err) + return err + } + defer newFile.Close() + + _, err = newFile.Write(manifestJSON) + if err != nil { + slog.Error("Failed to write manifest file: %v\n", "error", err) + return err + } + + return nil +} + +// nolint: musttag +func (fo *FileOperator) ManifestFile(currentFiles map[string]*mpi.File) (map[string]*mpi.File, error) { + if _, err := os.Stat(manifestFilePath); err != nil { + return currentFiles, err // Return current files if manifest directory still doesn't exist + } + + file, err := os.ReadFile(manifestFilePath) + if err != nil { + slog.Error("Failed to read manifest file", "error", err) + return nil, err + } + + var manifestFiles map[string]*mpi.File + + err = json.Unmarshal(file, &manifestFiles) + if err != nil { + slog.Error("Failed to parse manifest file", "error", err) + return nil, err + } + + fileMap := fo.convertToFileMap(manifestFiles) + + return fileMap, nil +} + +func (fo *FileOperator) convertToFileMap(manifestFiles map[string]*mpi.File) map[string]*mpi.File { + currentFileMap := make(map[string]*mpi.File) + for name, manifestFile := range manifestFiles { + currentFile := fo.convertToFile(manifestFile) + currentFileMap[name] = currentFile + } + + return currentFileMap +} + +func (fo *FileOperator) convertToFile(manifestFile *mpi.File) *mpi.File { + return manifestFile +} diff --git a/internal/file/file_plugin.go b/internal/file/file_plugin.go index 6aa47a41a..992d038d3 100644 --- a/internal/file/file_plugin.go +++ b/internal/file/file_plugin.go @@ -281,6 +281,11 @@ func (fp *FilePlugin) handleNginxConfigUpdate(ctx context.Context, msg *bus.Mess fp.fileManagerService.UpdateCurrentFilesOnDisk(files.ConvertToMapOfFiles(nginxConfigContext.Files)) + manifestErr := fp.fileManagerService.UpdateManifestFile(files.ConvertToMapOfFiles(nginxConfigContext.Files)) + if manifestErr != nil { + slog.ErrorContext(ctx, "Unable to update manifest file", "manifest", manifestErr.Error()) + } + err := fp.fileManagerService.UpdateOverview(ctx, nginxConfigContext.InstanceID, nginxConfigContext.Files, 0) if err != nil { slog.ErrorContext( diff --git a/internal/file/filefakes/fake_file_manager_service_interface.go b/internal/file/filefakes/fake_file_manager_service_interface.go index 588318877..f835a2a9b 100644 --- a/internal/file/filefakes/fake_file_manager_service_interface.go +++ b/internal/file/filefakes/fake_file_manager_service_interface.go @@ -28,19 +28,19 @@ type FakeFileManagerServiceInterface struct { result1 model.WriteStatus result2 error } - DetermineFileActionsStub func(map[string]*v1.File, map[string]*v1.File) (map[string]*v1.File, map[string][]byte, error) + DetermineFileActionsStub func(map[string]*v1.File, map[string]*model.FileCache) (map[string]*model.FileCache, map[string][]byte, error) determineFileActionsMutex sync.RWMutex determineFileActionsArgsForCall []struct { arg1 map[string]*v1.File - arg2 map[string]*v1.File + arg2 map[string]*model.FileCache } determineFileActionsReturns struct { - result1 map[string]*v1.File + result1 map[string]*model.FileCache result2 map[string][]byte result3 error } determineFileActionsReturnsOnCall map[int]struct { - result1 map[string]*v1.File + result1 map[string]*model.FileCache result2 map[string][]byte result3 error } @@ -89,6 +89,17 @@ type FakeFileManagerServiceInterface struct { updateFileReturnsOnCall map[int]struct { result1 error } + UpdateManifestFileStub func(map[string]*v1.File) error + updateManifestFileMutex sync.RWMutex + updateManifestFileArgsForCall []struct { + arg1 map[string]*v1.File + } + updateManifestFileReturns struct { + result1 error + } + updateManifestFileReturnsOnCall map[int]struct { + result1 error + } UpdateOverviewStub func(context.Context, string, []*v1.File, int) error updateOverviewMutex sync.RWMutex updateOverviewArgsForCall []struct { @@ -196,12 +207,12 @@ func (fake *FakeFileManagerServiceInterface) ConfigApplyReturnsOnCall(i int, res }{result1, result2} } -func (fake *FakeFileManagerServiceInterface) DetermineFileActions(arg1 map[string]*v1.File, arg2 map[string]*v1.File) (map[string]*v1.File, map[string][]byte, error) { +func (fake *FakeFileManagerServiceInterface) DetermineFileActions(arg1 map[string]*v1.File, arg2 map[string]*model.FileCache) (map[string]*model.FileCache, map[string][]byte, error) { fake.determineFileActionsMutex.Lock() ret, specificReturn := fake.determineFileActionsReturnsOnCall[len(fake.determineFileActionsArgsForCall)] fake.determineFileActionsArgsForCall = append(fake.determineFileActionsArgsForCall, struct { arg1 map[string]*v1.File - arg2 map[string]*v1.File + arg2 map[string]*model.FileCache }{arg1, arg2}) stub := fake.DetermineFileActionsStub fakeReturns := fake.determineFileActionsReturns @@ -222,43 +233,43 @@ func (fake *FakeFileManagerServiceInterface) DetermineFileActionsCallCount() int return len(fake.determineFileActionsArgsForCall) } -func (fake *FakeFileManagerServiceInterface) DetermineFileActionsCalls(stub func(map[string]*v1.File, map[string]*v1.File) (map[string]*v1.File, map[string][]byte, error)) { +func (fake *FakeFileManagerServiceInterface) DetermineFileActionsCalls(stub func(map[string]*v1.File, map[string]*model.FileCache) (map[string]*model.FileCache, map[string][]byte, error)) { fake.determineFileActionsMutex.Lock() defer fake.determineFileActionsMutex.Unlock() fake.DetermineFileActionsStub = stub } -func (fake *FakeFileManagerServiceInterface) DetermineFileActionsArgsForCall(i int) (map[string]*v1.File, map[string]*v1.File) { +func (fake *FakeFileManagerServiceInterface) DetermineFileActionsArgsForCall(i int) (map[string]*v1.File, map[string]*model.FileCache) { fake.determineFileActionsMutex.RLock() defer fake.determineFileActionsMutex.RUnlock() argsForCall := fake.determineFileActionsArgsForCall[i] return argsForCall.arg1, argsForCall.arg2 } -func (fake *FakeFileManagerServiceInterface) DetermineFileActionsReturns(result1 map[string]*v1.File, result2 map[string][]byte, result3 error) { +func (fake *FakeFileManagerServiceInterface) DetermineFileActionsReturns(result1 map[string]*model.FileCache, result2 map[string][]byte, result3 error) { fake.determineFileActionsMutex.Lock() defer fake.determineFileActionsMutex.Unlock() fake.DetermineFileActionsStub = nil fake.determineFileActionsReturns = struct { - result1 map[string]*v1.File + result1 map[string]*model.FileCache result2 map[string][]byte result3 error }{result1, result2, result3} } -func (fake *FakeFileManagerServiceInterface) DetermineFileActionsReturnsOnCall(i int, result1 map[string]*v1.File, result2 map[string][]byte, result3 error) { +func (fake *FakeFileManagerServiceInterface) DetermineFileActionsReturnsOnCall(i int, result1 map[string]*model.FileCache, result2 map[string][]byte, result3 error) { fake.determineFileActionsMutex.Lock() defer fake.determineFileActionsMutex.Unlock() fake.DetermineFileActionsStub = nil if fake.determineFileActionsReturnsOnCall == nil { fake.determineFileActionsReturnsOnCall = make(map[int]struct { - result1 map[string]*v1.File + result1 map[string]*model.FileCache result2 map[string][]byte result3 error }) } fake.determineFileActionsReturnsOnCall[i] = struct { - result1 map[string]*v1.File + result1 map[string]*model.FileCache result2 map[string][]byte result3 error }{result1, result2, result3} @@ -506,6 +517,67 @@ func (fake *FakeFileManagerServiceInterface) UpdateFileReturnsOnCall(i int, resu }{result1} } +func (fake *FakeFileManagerServiceInterface) UpdateManifestFile(arg1 map[string]*v1.File) error { + fake.updateManifestFileMutex.Lock() + ret, specificReturn := fake.updateManifestFileReturnsOnCall[len(fake.updateManifestFileArgsForCall)] + fake.updateManifestFileArgsForCall = append(fake.updateManifestFileArgsForCall, struct { + arg1 map[string]*v1.File + }{arg1}) + stub := fake.UpdateManifestFileStub + fakeReturns := fake.updateManifestFileReturns + fake.recordInvocation("UpdateManifestFile", []interface{}{arg1}) + fake.updateManifestFileMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeFileManagerServiceInterface) UpdateManifestFileCallCount() int { + fake.updateManifestFileMutex.RLock() + defer fake.updateManifestFileMutex.RUnlock() + return len(fake.updateManifestFileArgsForCall) +} + +func (fake *FakeFileManagerServiceInterface) UpdateManifestFileCalls(stub func(map[string]*v1.File) error) { + fake.updateManifestFileMutex.Lock() + defer fake.updateManifestFileMutex.Unlock() + fake.UpdateManifestFileStub = stub +} + +func (fake *FakeFileManagerServiceInterface) UpdateManifestFileArgsForCall(i int) map[string]*v1.File { + fake.updateManifestFileMutex.RLock() + defer fake.updateManifestFileMutex.RUnlock() + argsForCall := fake.updateManifestFileArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeFileManagerServiceInterface) UpdateManifestFileReturns(result1 error) { + fake.updateManifestFileMutex.Lock() + defer fake.updateManifestFileMutex.Unlock() + fake.UpdateManifestFileStub = nil + fake.updateManifestFileReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeFileManagerServiceInterface) UpdateManifestFileReturnsOnCall(i int, result1 error) { + fake.updateManifestFileMutex.Lock() + defer fake.updateManifestFileMutex.Unlock() + fake.UpdateManifestFileStub = nil + if fake.updateManifestFileReturnsOnCall == nil { + fake.updateManifestFileReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.updateManifestFileReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeFileManagerServiceInterface) UpdateOverview(arg1 context.Context, arg2 string, arg3 []*v1.File, arg4 int) error { var arg3Copy []*v1.File if arg3 != nil { @@ -594,6 +666,8 @@ func (fake *FakeFileManagerServiceInterface) Invocations() map[string][][]interf defer fake.updateCurrentFilesOnDiskMutex.RUnlock() fake.updateFileMutex.RLock() defer fake.updateFileMutex.RUnlock() + fake.updateManifestFileMutex.RLock() + defer fake.updateManifestFileMutex.RUnlock() fake.updateOverviewMutex.RLock() defer fake.updateOverviewMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/internal/file/filefakes/fake_file_operator.go b/internal/file/filefakes/fake_file_operator.go index 6373caf06..c949ceefd 100644 --- a/internal/file/filefakes/fake_file_operator.go +++ b/internal/file/filefakes/fake_file_operator.go @@ -9,6 +9,30 @@ import ( ) type FakeFileOperator struct { + ManifestFileStub func(map[string]*v1.File) (map[string]*v1.File, error) + manifestFileMutex sync.RWMutex + manifestFileArgsForCall []struct { + arg1 map[string]*v1.File + } + manifestFileReturns struct { + result1 map[string]*v1.File + result2 error + } + manifestFileReturnsOnCall map[int]struct { + result1 map[string]*v1.File + result2 error + } + UpdateManifestFileStub func(map[string]*v1.File) error + updateManifestFileMutex sync.RWMutex + updateManifestFileArgsForCall []struct { + arg1 map[string]*v1.File + } + updateManifestFileReturns struct { + result1 error + } + updateManifestFileReturnsOnCall map[int]struct { + result1 error + } WriteStub func(context.Context, []byte, *v1.FileMeta) error writeMutex sync.RWMutex writeArgsForCall []struct { @@ -26,6 +50,131 @@ type FakeFileOperator struct { invocationsMutex sync.RWMutex } +func (fake *FakeFileOperator) ManifestFile(arg1 map[string]*v1.File) (map[string]*v1.File, error) { + fake.manifestFileMutex.Lock() + ret, specificReturn := fake.manifestFileReturnsOnCall[len(fake.manifestFileArgsForCall)] + fake.manifestFileArgsForCall = append(fake.manifestFileArgsForCall, struct { + arg1 map[string]*v1.File + }{arg1}) + stub := fake.ManifestFileStub + fakeReturns := fake.manifestFileReturns + fake.recordInvocation("ManifestFile", []interface{}{arg1}) + fake.manifestFileMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeFileOperator) ManifestFileCallCount() int { + fake.manifestFileMutex.RLock() + defer fake.manifestFileMutex.RUnlock() + return len(fake.manifestFileArgsForCall) +} + +func (fake *FakeFileOperator) ManifestFileCalls(stub func(map[string]*v1.File) (map[string]*v1.File, error)) { + fake.manifestFileMutex.Lock() + defer fake.manifestFileMutex.Unlock() + fake.ManifestFileStub = stub +} + +func (fake *FakeFileOperator) ManifestFileArgsForCall(i int) map[string]*v1.File { + fake.manifestFileMutex.RLock() + defer fake.manifestFileMutex.RUnlock() + argsForCall := fake.manifestFileArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeFileOperator) ManifestFileReturns(result1 map[string]*v1.File, result2 error) { + fake.manifestFileMutex.Lock() + defer fake.manifestFileMutex.Unlock() + fake.ManifestFileStub = nil + fake.manifestFileReturns = struct { + result1 map[string]*v1.File + result2 error + }{result1, result2} +} + +func (fake *FakeFileOperator) ManifestFileReturnsOnCall(i int, result1 map[string]*v1.File, result2 error) { + fake.manifestFileMutex.Lock() + defer fake.manifestFileMutex.Unlock() + fake.ManifestFileStub = nil + if fake.manifestFileReturnsOnCall == nil { + fake.manifestFileReturnsOnCall = make(map[int]struct { + result1 map[string]*v1.File + result2 error + }) + } + fake.manifestFileReturnsOnCall[i] = struct { + result1 map[string]*v1.File + result2 error + }{result1, result2} +} + +func (fake *FakeFileOperator) UpdateManifestFile(arg1 map[string]*v1.File) error { + fake.updateManifestFileMutex.Lock() + ret, specificReturn := fake.updateManifestFileReturnsOnCall[len(fake.updateManifestFileArgsForCall)] + fake.updateManifestFileArgsForCall = append(fake.updateManifestFileArgsForCall, struct { + arg1 map[string]*v1.File + }{arg1}) + stub := fake.UpdateManifestFileStub + fakeReturns := fake.updateManifestFileReturns + fake.recordInvocation("UpdateManifestFile", []interface{}{arg1}) + fake.updateManifestFileMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeFileOperator) UpdateManifestFileCallCount() int { + fake.updateManifestFileMutex.RLock() + defer fake.updateManifestFileMutex.RUnlock() + return len(fake.updateManifestFileArgsForCall) +} + +func (fake *FakeFileOperator) UpdateManifestFileCalls(stub func(map[string]*v1.File) error) { + fake.updateManifestFileMutex.Lock() + defer fake.updateManifestFileMutex.Unlock() + fake.UpdateManifestFileStub = stub +} + +func (fake *FakeFileOperator) UpdateManifestFileArgsForCall(i int) map[string]*v1.File { + fake.updateManifestFileMutex.RLock() + defer fake.updateManifestFileMutex.RUnlock() + argsForCall := fake.updateManifestFileArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeFileOperator) UpdateManifestFileReturns(result1 error) { + fake.updateManifestFileMutex.Lock() + defer fake.updateManifestFileMutex.Unlock() + fake.UpdateManifestFileStub = nil + fake.updateManifestFileReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeFileOperator) UpdateManifestFileReturnsOnCall(i int, result1 error) { + fake.updateManifestFileMutex.Lock() + defer fake.updateManifestFileMutex.Unlock() + fake.UpdateManifestFileStub = nil + if fake.updateManifestFileReturnsOnCall == nil { + fake.updateManifestFileReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.updateManifestFileReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeFileOperator) Write(arg1 context.Context, arg2 []byte, arg3 *v1.FileMeta) error { var arg2Copy []byte if arg2 != nil { @@ -97,6 +246,10 @@ func (fake *FakeFileOperator) WriteReturnsOnCall(i int, result1 error) { func (fake *FakeFileOperator) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() + fake.manifestFileMutex.RLock() + defer fake.manifestFileMutex.RUnlock() + fake.updateManifestFileMutex.RLock() + defer fake.updateManifestFileMutex.RUnlock() fake.writeMutex.RLock() defer fake.writeMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/internal/model/config.go b/internal/model/config.go index 830794028..543d090c8 100644 --- a/internal/model/config.go +++ b/internal/model/config.go @@ -27,6 +27,19 @@ type APIDetails struct { Location string } +type ManifestFile struct { + ManifestFileMeta *ManifestFileMeta `json:"manifest_file_meta"` +} + +type ManifestFileMeta struct { + // The full path of the file + Name string `json:"name"` + // The hash of the file contents sha256, hex encoded + Hash string `json:"hash"` + // The size of the file in bytes + Size int64 `json:"size"` +} + // Complexity is 11, allowed is 10 // nolint: revive, cyclop func (ncc *NginxConfigContext) Equal(otherNginxConfigContext *NginxConfigContext) bool { diff --git a/internal/model/file.go b/internal/model/file.go new file mode 100644 index 000000000..fc282c2b4 --- /dev/null +++ b/internal/model/file.go @@ -0,0 +1,34 @@ +// Copyright (c) F5, Inc. +// +// This source code is licensed under the Apache License, Version 2.0 license found in the +// LICENSE file in the root directory of this source tree. + +package model + +import mpi "github.com/nginx/agent/v3/api/grpc/mpi/v1" + +type FileCache struct { + File *mpi.File `json:"file"` + Action FileAction `json:"action"` +} + +type FileAction int + +const ( + Add FileAction = iota + 1 + Update + Delete + Unchanged +) + +// ConvertToMapOfFiles converts a list of files to a map of file caches (file and action) with the file name as the key +func ConvertToMapOfFileCache(convertFiles []*mpi.File) map[string]*FileCache { + filesMap := make(map[string]*FileCache) + for _, convertFile := range convertFiles { + filesMap[convertFile.GetFileMeta().GetName()] = &FileCache{ + File: convertFile, + } + } + + return filesMap +} diff --git a/internal/watcher/instance/nginx_config_parser.go b/internal/watcher/instance/nginx_config_parser.go index 7f732cbb7..b6b9a279e 100644 --- a/internal/watcher/instance/nginx_config_parser.go +++ b/internal/watcher/instance/nginx_config_parser.go @@ -20,6 +20,8 @@ import ( "strconv" "strings" + "github.com/nginx/agent/v3/internal/file" + mpi "github.com/nginx/agent/v3/api/grpc/mpi/v1" "github.com/nginx/agent/v3/internal/config" "github.com/nginx/agent/v3/internal/model" @@ -40,9 +42,15 @@ const ( locationDirective = "location" ) +var ( + manifestDirPath = "/var/lib/nginx-agent" + manifestFilePath = manifestDirPath + "/manifest.json" +) + type ( NginxConfigParser struct { - agentConfig *config.Config + agentConfig *config.Config + fileOperator *file.FileOperator } ) @@ -56,7 +64,8 @@ type ( func NewNginxConfigParser(agentConfig *config.Config) *NginxConfigParser { return &NginxConfigParser{ - agentConfig: agentConfig, + agentConfig: agentConfig, + fileOperator: file.NewFileOperator(), } } @@ -88,7 +97,59 @@ func (ncp *NginxConfigParser) Parse(ctx context.Context, instance *mpi.Instance) return nil, err } - return ncp.createNginxConfigContext(ctx, instance, payload) + configContext, configErr := ncp.createNginxConfigContext(ctx, instance, payload) + + if configContext == nil || configErr != nil { + slog.ErrorContext(ctx, fmt.Sprintf("Failed to parse NGINX config context: %s", err)) + return nil, err + } + + configContext = ncp.addAuxFiles(ctx, configContext) + + return configContext, err +} + +func (ncp *NginxConfigParser) addAuxFiles(ctx context.Context, + configContext *model.NginxConfigContext, +) *model.NginxConfigContext { + configContextFilesMap := files.ConvertToMapOfFiles(configContext.Files) + + auxFiles, auxErr := ncp.auxFiles() + if auxErr != nil { + slog.ErrorContext(ctx, "Failed to get aux files from manifest file", "error", auxErr) + return configContext + } + if len(auxFiles) > 0 { + for fileName, auxFile := range auxFiles { + if _, ok := configContextFilesMap[fileName]; !ok { + slog.DebugContext(ctx, "Aux file in manifest, adding to list of files", "file", fileName) + configContext.Files = append(configContext.Files, auxFile) + } + } + } + + updateErr := ncp.fileOperator.UpdateManifestFile(files.ConvertToMapOfFiles(configContext.Files)) + if updateErr != nil { + slog.ErrorContext(ctx, "Error updating manifest file", "error", updateErr) + } + + return configContext +} + +func (ncp *NginxConfigParser) auxFiles() (map[string]*mpi.File, error) { + _, err := os.Stat(manifestFilePath) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + + err = ncp.fileOperator.UpdateManifestFile(make(map[string]*mpi.File)) + if err != nil { + return nil, err + } + } + + return ncp.fileOperator.ManifestFile(make(map[string]*mpi.File)) } // nolint: cyclop,revive,gocognit @@ -253,13 +314,13 @@ func (ncp *NginxConfigParser) formatMap(directive *crossplane.Directive) map[str return formatMap } -func (ncp *NginxConfigParser) accessLog(file, format string, formatMap map[string]string) *model.AccessLog { +func (ncp *NginxConfigParser) accessLog(accessLogFile, format string, formatMap map[string]string) *model.AccessLog { accessLog := &model.AccessLog{ - Name: file, + Name: accessLogFile, Readable: false, } - info, err := os.Stat(file) + info, err := os.Stat(accessLogFile) if err == nil { accessLog.Readable = true accessLog.Permissions = files.Permissions(info.Mode()) @@ -288,13 +349,13 @@ func (ncp *NginxConfigParser) updateLogFormat( return accessLog } -func (ncp *NginxConfigParser) errorLog(file, level string) *model.ErrorLog { +func (ncp *NginxConfigParser) errorLog(errorLogFile, level string) *model.ErrorLog { errorLog := &model.ErrorLog{ - Name: file, + Name: errorLogFile, LogLevel: level, Readable: false, } - info, err := os.Stat(file) + info, err := os.Stat(errorLogFile) if err == nil { errorLog.Permissions = files.Permissions(info.Mode()) errorLog.Readable = true @@ -319,22 +380,22 @@ func (ncp *NginxConfigParser) errorLogDirectiveLevel(directive *crossplane.Direc return "" } -func (ncp *NginxConfigParser) sslCert(ctx context.Context, file, rootDir string) (sslCertFile *mpi.File) { - if strings.Contains(file, "$") { +func (ncp *NginxConfigParser) sslCert(ctx context.Context, certFile, rootDir string) (sslCertFile *mpi.File) { + if strings.Contains(certFile, "$") { // cannot process any filepath with variables return nil } - if !filepath.IsAbs(file) { - file = filepath.Join(rootDir, file) + if !filepath.IsAbs(certFile) { + certFile = filepath.Join(rootDir, certFile) } - if !ncp.agentConfig.IsDirectoryAllowed(file) { - slog.DebugContext(ctx, "File not in allowed directories", "file", file) + if !ncp.agentConfig.IsDirectoryAllowed(certFile) { + slog.DebugContext(ctx, "File not in allowed directories", "file", certFile) } else { - sslCertFileMeta, fileMetaErr := files.FileMetaWithCertificate(file) + sslCertFileMeta, fileMetaErr := files.FileMetaWithCertificate(certFile) if fileMetaErr != nil { - slog.ErrorContext(ctx, "Unable to get file metadata", "file", file, "error", fileMetaErr) + slog.ErrorContext(ctx, "Unable to get file metadata", "file", certFile, "error", fileMetaErr) } else { sslCertFile = &mpi.File{FileMeta: sslCertFileMeta} } diff --git a/internal/watcher/instance/nginx_config_parser_test.go b/internal/watcher/instance/nginx_config_parser_test.go index 0444b8749..68b86e0d0 100644 --- a/internal/watcher/instance/nginx_config_parser_test.go +++ b/internal/watcher/instance/nginx_config_parser_test.go @@ -383,6 +383,8 @@ func TestNginxConfigParser_Parse(t *testing.T) { }) test.instance.InstanceRuntime.ConfigPath = file.Name() + t.Log("------- File Name", file.Name()) + t.Log("------- Content Expected", test.content) agentConfig := types.AgentConfig() agentConfig.AllowedDirectories = test.allowedDirectories diff --git a/test/config/nginx/nginx-not-allowed-dir.conf b/test/config/nginx/nginx-not-allowed-dir.conf index b3132c339..e3d999874 100644 --- a/test/config/nginx/nginx-not-allowed-dir.conf +++ b/test/config/nginx/nginx-not-allowed-dir.conf @@ -29,15 +29,6 @@ http { index index.html index.htm; } - ## - # Enable Metrics - ## - location /api { - stub_status; - allow 127.0.0.1; - deny all; - } - # redirect server error pages to the static page /50x.html # error_page 500 502 503 504 /50x.html; diff --git a/test/protos/files.go b/test/protos/files.go index db34f617f..b59972084 100644 --- a/test/protos/files.go +++ b/test/protos/files.go @@ -73,9 +73,3 @@ func FileOverview(filePath, fileHash string) *mpi.FileOverview { ConfigVersion: CreateConfigVersion(), } } - -func FileContents(content []byte) *mpi.FileContents { - return &mpi.FileContents{ - Contents: content, - } -} diff --git a/test/protos/instances.go b/test/protos/instances.go index 3bf88555f..c88c40f51 100644 --- a/test/protos/instances.go +++ b/test/protos/instances.go @@ -7,7 +7,6 @@ package protos import ( "fmt" - "os" mpi "github.com/nginx/agent/v3/api/grpc/mpi/v1" "github.com/nginx/agent/v3/internal/config" @@ -138,13 +137,6 @@ func GetMultipleInstances(expectedModules []string) []*mpi.Instance { return []*mpi.Instance{process1, process2} } -func GetMultipleInstancesWithUnsupportedInstance() []*mpi.Instance { - process1 := GetNginxOssInstance([]string{}) - process2 := GetUnsupportedInstance() - - return []*mpi.Instance{process1, process2} -} - func GetInstancesNoParentProcess(expectedModules []string) []*mpi.Instance { process1 := GetNginxOssInstance(expectedModules) process1.GetInstanceRuntime().ProcessId = 0 @@ -155,24 +147,6 @@ func GetInstancesNoParentProcess(expectedModules []string) []*mpi.Instance { return []*mpi.Instance{process1, process2} } -func GetFileCache(files ...*os.File) (map[string]*mpi.FileMeta, error) { - cache := make(map[string]*mpi.FileMeta) - for _, file := range files { - lastModified, err := CreateProtoTime("2024-01-09T13:22:21Z") - if err != nil { - return nil, err - } - - cache[file.Name()] = &mpi.FileMeta{ - ModifiedTime: lastModified, - Name: file.Name(), - Hash: "BDEIFo9anKNvAwWm9O2LpfvNiNiGMx.c", - } - } - - return cache, nil -} - func getSecondNginxOssInstance(expectedModules []string) *mpi.Instance { process2 := GetNginxOssInstance(expectedModules) process2.GetInstanceRuntime().ProcessId = processID2