Adding function to execute Buildozer commands on a single in-memory file

A buildozer client are looking to execute the commands of buildozer on the content of a build file, in a service environment (without direct access to the file storage). The new function enables this by running the commands without accessing the file system.

Author: Tim Malmström

buildozer/README.md

@@ -350,6 +350,25 @@ Buildozer commands can be made executable by means of a shebang line, too:
 add deps //base //strings|-:foo|-:bar
 ```
 
+## Using Buildozer in-memory
+
+Some clients of Buildozer have the need to execute buildozer actions in memory
+(due to a service environment which does not have access to their file system).
+This can be done using, `edit.ExecuteCommandsOnInlineFile`, which accepts
+commands and BUILD file content as bytes, applies the changes and returns the
+raw file content.
+For more details and implementation, see [`/edit/buildozer.go`](../edit/buildozer.go)
+
+Some caveats of running Buildozer in-memory:
+
+* The function assumes (and validates to some extent) that all commands apply to
+  the same file and will return errors if there are commands affecting different
+  paths.
+* When referencing targets, the function will not reliably determine if targets
+  are local or remote. Hence redundant path references may be included in
+  output. (e.g. `add dep //package/path:bar|//package/path:foo` would add the dep
+  `//package/path:bar` instead of just `:bar`).
+
 ## Error code
 
 The return code is:

edit/buildozer.go

@@ -26,6 +26,7 @@ import (
 	"io"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -1183,11 +1184,9 @@ func rewrite(opts *Options, commandsForFile commandsForFile) *rewriteResult {
 	}
 	var errs []error
 	changed := false
-	for _, commands := range commandsForFile.commands {
-		target := commands.target
-		commands := commands.commands
-		_, _, absPkg, rule := InterpretLabelForWorkspaceLocation(opts.RootDir, target)
-		if label := labels.Parse(target); label.Package == stdinPackageName {
+	for _, cft := range commandsForFile.commands {
+		_, _, absPkg, rule := InterpretLabelForWorkspaceLocation(opts.RootDir, cft.target)
+		if label := labels.Parse(cft.target); label.Package == stdinPackageName {
 			// Special-case: This is already absolute
 			absPkg = stdinPackageName
 		}
@@ -1198,48 +1197,27 @@ func rewrite(opts *Options, commandsForFile commandsForFile) *rewriteResult {
 
 		targets, err := expandTargets(f, rule)
 		if err != nil {
-			cerr := commandError(commands, target, err)
+			cerr := commandError(cft.commands, cft.target, err)
 			errs = append(errs, cerr)
 			if !opts.KeepGoing {
 				return &rewriteResult{file: name, errs: errs, records: records}
 			}
 		}
 		targets = filterRules(opts, targets)
-		for _, cmd := range commands {
-			cmdInfo := AllCommands[cmd.tokens[0]]
-			// Depending on whether a transformation is rule-specific or not, it should be applied to
-			// every rule that satisfies the filter or just once to the file.
-			cmdTargets := targets
-			if !cmdInfo.PerRule {
-				cmdTargets = []*build.Rule{nil}
-			}
-			for _, r := range cmdTargets {
-				record := &apipb.Output_Record{}
-				newf, err := cmdInfo.Fn(opts, CmdEnvironment{f, r, vars, absPkg, cmd.tokens[1:], record})
-				if len(record.Fields) != 0 {
-					records = append(records, record)
-				}
-				if err != nil {
-					cerr := commandError([]command{cmd}, target, err)
-					if opts.KeepGoing {
-						errs = append(errs, cerr)
-					} else {
-						return &rewriteResult{file: name, errs: []error{cerr}, records: records}
-					}
-				}
-				if newf != nil {
-					changed = true
-					f = newf
-				}
-			}
+
+		newf, err := executeCommandsInFile(opts, f, cft, targets, &records, vars, absPkg, &errs)
+		if err != nil {
+			return &rewriteResult{file: name, errs: []error{err}, records: records}
+		}
+		if newf != nil {
+			changed = true
+			f = newf
 		}
 	}
 	if !changed {
 		return &rewriteResult{file: name, errs: errs, records: records}
 	}
-	f = RemoveEmptyPackage(f)
-	f = RemoveEmptyUseRepoCalls(f)
-	ndata, err := buildifier.Buildify(opts, f)
+	ndata, err := cleanAndBuildify(opts, f)
 	if err != nil {
 		return &rewriteResult{file: name, errs: []error{fmt.Errorf("running buildifier: %v", err)}, records: records}
 	}
@@ -1264,6 +1242,58 @@ func rewrite(opts *Options, commandsForFile commandsForFile) *rewriteResult {
 	return &rewriteResult{file: name, errs: errs, modified: true, records: records}
 }
 
+// executeCommandsInFile executes the provided commandsForTarget in the provided build.File.
+func executeCommandsInFile(
+	opts *Options,
+	f *build.File,
+	cft commandsForTarget,
+	rules []*build.Rule,
+	records *[]*apipb.Output_Record,
+	vars map[string]*build.AssignExpr,
+	absPkg string,
+	errs *[]error,
+) (*build.File, error) {
+	changed := false
+	for _, cmd := range cft.commands {
+		cmdInfo := AllCommands[cmd.tokens[0]]
+		// Depending on whether a transformation is rule-specific or not, it should be applied to
+		// every rule that satisfies the filter or just once to the file.
+		cmdTargets := rules
+		if !cmdInfo.PerRule {
+			cmdTargets = []*build.Rule{nil}
+		}
+		for _, r := range cmdTargets {
+			record := &apipb.Output_Record{}
+			newf, err := cmdInfo.Fn(opts, CmdEnvironment{f, r, vars, absPkg, cmd.tokens[1:], record})
+			if len(record.Fields) != 0 {
+				*records = append(*records, record)
+			}
+			if err != nil {
+				cerr := commandError([]command{cmd}, cft.target, err)
+				if opts.KeepGoing {
+					*errs = append(*errs, cerr)
+				} else {
+					return nil, cerr
+				}
+			}
+			if newf != nil {
+				f = newf
+				changed = true
+			}
+		}
+	}
+	if changed {
+		return f, nil
+	}
+	return nil, nil
+}
+
+func cleanAndBuildify(opts *Options, f *build.File) ([]byte, error) {
+	f = RemoveEmptyPackage(f)
+	f = RemoveEmptyUseRepoCalls(f)
+	return buildifier.Buildify(opts, f)
+}
+
 // EditFile is a function that does any prework needed before editing a file.
 // e.g. "checking out for write" from a locking source control repo.
 var EditFile = func(fi os.FileInfo, name string) error {
@@ -1547,3 +1577,82 @@ func Buildozer(opts *Options, args []string) int {
 	}
 	return 0
 }
+
+// ExecuteCommandsOnInlineFile executes the given commands on the given file content.
+// Returns the new file content after applying the commands.
+func ExecuteCommandsOnInlineFile(fileContent []byte, commands []string) ([]byte, error) {
+	opts := Options{}
+	commandsByTargetName, filename, err := groupCommandsForInlineFile(commands, opts)
+	if err != nil {
+		return nil, err
+	}
+	f, err := build.Parse(*filename, fileContent)
+	if err != nil {
+		return nil, err
+	}
+	if f.Type == build.TypeDefault {
+		// Buildozer is unable to infer the file type, fall back to BUILD by default.
+		f.Type = build.TypeBuild
+	}
+	for _, cft := range commandsByTargetName {
+		rules, err := expandTargets(f, cft.target)
+		if err != nil {
+			return nil, err
+		}
+		newf, err := executeCommandsInFile(
+			&opts,
+			f,
+			cft,
+			rules,
+			// Output records are ignored in inline file execution.
+			nil,
+			// Global variables not supported in inline file execution.
+			nil,
+			f.Pkg,
+			// Errors-list is ignored since opts.keepGoing is always false.
+			nil,
+		)
+		if err != nil {
+			return nil, err
+		}
+		if newf != nil {
+			f = newf
+		}
+	}
+	outputFileContent, err := cleanAndBuildify(&opts, f)
+	if err != nil {
+		return nil, err
+	}
+	return outputFileContent, nil
+}
+
+// groupCommandsForInlineFile groups the given commands by file and returns the commands for a
+// single file. Returns an error if the commands modify multiple files or if commands are invalid.
+func groupCommandsForInlineFile(commands []string, opts Options) ([]commandsForTarget, *string, error) {
+	commandsByFile := make(map[string][]commandsForTarget)
+	commandReader := strings.NewReader(strings.Join(commands, "\n"))
+	err := appendCommandsFromReader(&opts, commandReader, commandsByFile, nil)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error parsing commands %s", err)
+	}
+	if len(commandsByFile) != 1 {
+		return nil, nil, fmt.Errorf("invalid input commands, expected all commands to reference a single file")
+	}
+	for filepath, commandsForTargets := range commandsByFile {
+		for i := range commandsForTargets {
+			cft := &commandsForTargets[i]
+			splitTarget := strings.Split(cft.target, ":")
+			switch len(splitTarget) {
+			case 1: // No-op
+			case 2:
+				// Only keeps target (what is after the ":" character) since path is redundant.
+				cft.target = splitTarget[1]
+			default:
+				return nil, nil, fmt.Errorf("invalid target name %q", cft.target)
+			}
+		}
+		_, filename := path.Split(filepath)
+		return commandsForTargets, &filename, nil
+	}
+	panic("unreachable")
+}