Add file output to foreach command (#141)

* Add file output to foreach command

* Reorder

* Simplify code

Author: rnorth

cmd/foreach/foreach.go

@@ -17,6 +17,7 @@ package foreach
 
 import (
 	"errors"
+	"fmt"
 	"os"
 	"path"
 	"strings"
@@ -34,7 +35,15 @@ import (
 var exec executor.Executor = executor.NewRealExecutor()
 
 var (
-	repoFile string = "repos.txt"
+	repoFile = "repos.txt"
+
+	overallResultsDirectory string
+
+	successfulResultsDirectory string
+	successfulReposFileName    string
+
+	failedResultsDirectory string
+	failedReposFileName    string
 )
 
 func formatArguments(arguments []string) string {
@@ -49,8 +58,7 @@ func NewForeachCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "foreach [flags] -- COMMAND [ARGUMENT...]",
 		Short: "Run COMMAND against each working copy",
-		Long:
-`Run COMMAND against each working copy. Make sure to include a
+		Long: `Run COMMAND against each working copy. Make sure to include a
 double hyphen -- with space on both sides before COMMAND, as this
 marks that no further options should be interpreted by turbolift.`,
 		RunE: runE,
@@ -83,6 +91,10 @@ func runE(c *cobra.Command, args []string) error {
 	// the user something they could copy and paste.
 	prettyArgs := formatArguments(args)
 
+	setupOutputFiles(dir.Name, prettyArgs)
+
+	logger.Printf("Logs for all executions will be stored under %s", overallResultsDirectory)
+
 	var doneCount, skippedCount, errorCount int
 	for _, repo := range dir.Repos {
 		repoDirPath := path.Join("work", repo.OrgName, repo.RepoName) // i.e. work/org/repo
@@ -99,9 +111,11 @@ func runE(c *cobra.Command, args []string) error {
 		err := exec.Execute(execActivity.Writer(), repoDirPath, args[0], args[1:]...)
 
 		if err != nil {
+			emitOutcomeToFiles(repo, failedReposFileName, failedResultsDirectory, execActivity.Logs(), logger)
 			execActivity.EndWithFailure(err)
 			errorCount++
 		} else {
+			emitOutcomeToFiles(repo, successfulReposFileName, successfulResultsDirectory, execActivity.Logs(), logger)
 			execActivity.EndWithSuccessAndEmitLogs()
 			doneCount++
 		}
@@ -113,5 +127,55 @@ func runE(c *cobra.Command, args []string) error {
 		logger.Warnf("turbolift foreach completed with %s %s(%s, %s, %s)\n", colors.Red("errors"), colors.Normal(), colors.Green(doneCount, " OK"), colors.Yellow(skippedCount, " skipped"), colors.Red(errorCount, " errored"))
 	}
 
+	logger.Printf("Logs for all executions have been stored under %s", overallResultsDirectory)
+	logger.Printf("Names of successful repos have been written to %s", successfulReposFileName)
+	logger.Printf("Names of failed repos have been written to %s", failedReposFileName)
+
 	return nil
 }
+
+// sets up a temporary directory to store success/failure logs etc
+func setupOutputFiles(campaignName string, command string) {
+	overallResultsDirectory, _ = os.MkdirTemp("", fmt.Sprintf("turbolift-foreach-%s-", campaignName))
+	successfulResultsDirectory = path.Join(overallResultsDirectory, "successful")
+	failedResultsDirectory = path.Join(overallResultsDirectory, "failed")
+	_ = os.MkdirAll(successfulResultsDirectory, 0755)
+	_ = os.MkdirAll(failedResultsDirectory, 0755)
+
+	successfulReposFileName = path.Join(successfulResultsDirectory, "repos.txt")
+	failedReposFileName = path.Join(failedResultsDirectory, "repos.txt")
+
+	// create the files
+	successfulReposFile, _ := os.Create(successfulReposFileName)
+	failedReposFile, _ := os.Create(failedReposFileName)
+	defer successfulReposFile.Close()
+	defer failedReposFile.Close()
+
+	_, _ = successfulReposFile.WriteString(fmt.Sprintf("# This file contains the list of repositories that were successfully processed by turbolift foreach\n# for the command: %s\n", command))
+	_, _ = failedReposFile.WriteString(fmt.Sprintf("# This file contains the list of repositories that failed to be processed by turbolift foreach\n# for the command: %s\n", command))
+}
+
+func emitOutcomeToFiles(repo campaign.Repo, reposFileName string, logsDirectoryParent string, executionLogs string, logger *logging.Logger) {
+	// write the repo name to the repos file
+	reposFile, _ := os.OpenFile(reposFileName, os.O_RDWR|os.O_APPEND, 0644)
+	defer reposFile.Close()
+	_, err := reposFile.WriteString(repo.FullRepoName + "\n")
+	if err != nil {
+		logger.Errorf("Failed to write repo name to %s: %s", reposFile.Name(), err)
+	}
+
+	// write logs to a file under the logsParent directory, in a directory structure that mirrors that of the work directory
+	logsDir := path.Join(logsDirectoryParent, repo.FullRepoName)
+	logsFile := path.Join(logsDir, "logs.txt")
+	err = os.MkdirAll(logsDir, 0755)
+	if err != nil {
+		logger.Errorf("Failed to create directory %s: %s", logsDir, err)
+	}
+
+	logs, _ := os.Create(logsFile)
+	defer logs.Close()
+	_, err = logs.WriteString(executionLogs)
+	if err != nil {
+		logger.Errorf("Failed to write logs to %s: %s", logsFile, err)
+	}
+}

cmd/foreach/foreach_test.go

@@ -18,6 +18,7 @@ package foreach
 import (
 	"bytes"
 	"os"
+	"regexp"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -162,6 +163,47 @@ func TestFormatArguments(t *testing.T) {
 	}
 }
 
+func TestItCreatesLogFiles(t *testing.T) {
+	fakeExecutor := executor.NewAlternatingSuccessFakeExecutor()
+	exec = fakeExecutor
+
+	testsupport.PrepareTempCampaign(true, "org/repo1", "org/repo2")
+
+	out, err := runCommand("--", "some", "command")
+	assert.NoError(t, err)
+	assert.Contains(t, out, "turbolift foreach completed")
+	assert.Contains(t, out, "1 OK, 0 skipped, 1 errored")
+
+	// Logs should describe where output was written
+	r := regexp.MustCompile(`Logs for all executions have been stored under (.+)`)
+	matches := r.FindStringSubmatch(out)
+	assert.Len(t, matches, 2, "Expected to find the log directory path")
+	path := matches[1]
+
+	// check that expected static directories and files exist
+	_, err = os.Stat(path)
+	assert.NoError(t, err, "Expected the log directory to exist")
+
+	_, err = os.Stat(path + "/successful")
+	assert.NoError(t, err, "Expected the successful log directory to exist")
+
+	_, err = os.Stat(path + "/failed")
+	assert.NoError(t, err, "Expected the failure log directory to exist")
+
+	_, err = os.Stat(path + "/successful/repos.txt")
+	assert.NoError(t, err, "Expected the successful repos.txt file to exist")
+
+	_, err = os.Stat(path + "/failed/repos.txt")
+	assert.NoError(t, err, "Expected the failure repos.txt file to exist")
+
+	// check that the expected logs files exist
+	_, err = os.Stat(path + "/successful/org/repo1/logs.txt")
+	assert.NoError(t, err, "Expected the successful log file for org/repo1 to exist")
+
+	_, err = os.Stat(path + "/failed/org/repo2/logs.txt")
+	assert.NoError(t, err, "Expected the failure log file for org/repo2 to exist")
+}
+
 func runCommand(args ...string) (string, error) {
 	cmd := NewForeachCmd()
 	outBuffer := bytes.NewBufferString("")

internal/executor/fake_executor.go

@@ -70,3 +70,25 @@ func NewAlwaysFailsFakeExecutor() *FakeExecutor {
 		return "", errors.New("synthetic error")
 	})
 }
+
+func NewAlternatingSuccessFakeExecutor() *FakeExecutor {
+	i := 0
+	return NewFakeExecutor(
+		func(s string, s2 string, s3 ...string) error {
+			i++
+			if i%2 == 1 {
+				return nil
+			} else {
+				return errors.New("synthetic error")
+			}
+		},
+		func(s string, s2 string, s3 ...string) (string, error) {
+			i++
+			if i%2 == 1 {
+				return "", nil
+			} else {
+				return "", errors.New("synthetic error")
+			}
+		},
+	)
+}

internal/logging/activity.go

@@ -110,3 +110,7 @@ func (a *Activity) Writer() io.Writer {
 		activity: a,
 	}
 }
+
+func (a *Activity) Logs() string {
+	return strings.Join(a.logs, "\n")
+}