Get input output with symlinks

Add a function to return inputs and outputs. Note that symlinks point foo to bar will be represented as foo->bar.

Author: ywmei-brt1

go/pkg/fakes/server.go

@@ -28,7 +28,7 @@ import (
 	repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
 	bsgrpc "google.golang.org/genproto/googleapis/bytestream"
 	bspb "google.golang.org/genproto/googleapis/bytestream"
-	anypb "google.golang.org/protobuf/types/known/anypb"
+	"google.golang.org/protobuf/types/known/anypb"
 	dpb "google.golang.org/protobuf/types/known/durationpb"
 	tspb "google.golang.org/protobuf/types/known/timestamppb"
 )
@@ -304,6 +304,26 @@ func (f *InputFile) apply(ac *repb.ActionResult, s *Server, execRoot string) err
 	return nil
 }
 
+// InputSymlink to be made available to the fake action.
+type InputSymlink struct {
+	Path    string // newname
+	Content string
+	Target  string // oldname
+}
+
+// Apply puts the target file in the fake CAS and create a symlink in OS.
+func (ins *InputSymlink) apply(ac *repb.ActionResult, s *Server, execRoot string) error {
+	inf := InputFile{Path: ins.Target, Contents: ins.Content}
+	if err := inf.apply(ac, s, execRoot); err != nil {
+		return err
+	}
+	// create a symlink from old name (target) to new name (path)
+	if err := os.Symlink(filepath.Join(execRoot, ins.Target), filepath.Join(execRoot, ins.Path)); err != nil {
+		return fmt.Errorf("error creating symlink: %w", err)
+	}
+	return nil
+}
+
 // OutputFile is to be added as an output of the fake action.
 type OutputFile struct {
 	Path     string

go/pkg/tool/tool.go

@@ -770,7 +770,7 @@ func (c *Client) getOutputs(ctx context.Context, actionRes *repb.ActionResult) (
 	return res.String(), nil
 }
 
-func (c *Client) getInputTree(ctx context.Context, root *repb.Digest) (string, []string, error) {
+func (c *Client) getInputTree(ctx context.Context, root *repb.Digest) (string, map[string]string, error) {
 	var res bytes.Buffer
 
 	dg, err := digest.NewFromProto(root)
@@ -800,7 +800,7 @@ func (c *Client) getInputTree(ctx context.Context, root *repb.Digest) (string, [
 	return res.String(), paths, nil
 }
 
-func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, []string, error) {
+func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, map[string]string, error) {
 	var res bytes.Buffer
 	outputs, err := c.GrpcClient.FlattenTree(t, "")
 	if err != nil {
@@ -816,8 +816,10 @@ func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, []strin
 		paths = append(paths, path)
 	}
 	sort.Strings(paths)
+	pathDgs := make(map[string]string, len(paths))
 	for _, path := range paths {
 		output := outputs[path]
+		dg := output.Digest
 		var np string
 		if output.NodeProperties != nil {
 			np = fmt.Sprintf(" [Node properties: %v]", prototext.MarshalOptions{Multiline: false}.Format(output.NodeProperties))
@@ -826,11 +828,16 @@ func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, []strin
 			res.WriteString(fmt.Sprintf("%v: [Directory digest: %v]%s\n", path, output.Digest, np))
 		} else if output.SymlinkTarget != "" {
 			res.WriteString(fmt.Sprintf("%v: [Symlink digest: %v, Symlink Target: %v]%s\n", path, output.Digest, output.SymlinkTarget, np))
+			path = path + "->" + output.SymlinkTarget
+			if o, ok := outputs[output.SymlinkTarget]; ok {
+				dg = o.Digest
+			}
 		} else {
 			res.WriteString(fmt.Sprintf("%v: [File digest: %v]%s\n", path, output.Digest, np))
 		}
+		pathDgs[path] = fmt.Sprintf("%v", dg)
 	}
-	return res.String(), paths, nil
+	return res.String(), pathDgs, nil
 }
 
 func (c *Client) getActionResult(ctx context.Context, actionDigest string) (*repb.ActionResult, error) {
@@ -848,3 +855,91 @@ func (c *Client) getActionResult(ctx context.Context, actionDigest string) (*rep
 	}
 	return resPb, nil
 }
+
+// IO is a collection input root Digest, Inputs, and Outputs for an action.
+type IO struct {
+	RootDg  string
+	Inputs  *Inputs
+	Outputs *Outputs
+}
+
+// Inputs are input Paths (with digests) of an action. For a symlink, the key is
+// `<path>-><target>`, the value is the dg of the target file.
+type Inputs struct {
+	Paths map[string]string
+}
+
+// Outputs are output Files/Dirs (with digests) of an action. For a symlink, the
+// key is `<path>-><target>`, the value is the dg of the target file.
+type Outputs struct {
+	Files        map[string]string
+	Dirs         map[string]string
+	FileSymlinks map[string]string
+	DirSymlinks  map[string]string
+}
+
+// GetIO returns the Inputs and Outputs of an action Digest.
+func (c *Client) GetIO(ctx context.Context, actionDigest string) (*IO, error) {
+	acDg, err := digest.NewFromString(actionDigest)
+	if err != nil {
+		return nil, fmt.Errorf("error creating action digest: %w", err)
+	}
+	actionProto := &repb.Action{}
+	// Get all the Inputs.
+	if _, err := c.GrpcClient.ReadProto(ctx, acDg, actionProto); err != nil {
+		return nil, fmt.Errorf("error reading from proto: %w", err)
+	}
+	rootDg, err := digest.NewFromProto(actionProto.GetInputRootDigest())
+	if err != nil {
+		return nil, fmt.Errorf("error getting input root digest: %w", err)
+	}
+	_, inputs, err := c.getInputTree(ctx, actionProto.GetInputRootDigest())
+	if err != nil {
+		return nil, err
+	}
+	// Get all the Outputs.
+	resPb, err := c.getActionResult(ctx, actionDigest)
+	if err != nil {
+		return nil, err
+	}
+
+	files := map[string]string{}
+	dirs := map[string]string{}
+	fileSymlinks := map[string]string{}
+	dirSymlinks := map[string]string{}
+	for _, f := range resPb.GetOutputFiles() {
+		if f != nil {
+			dg, err := digest.NewFromProto(f.GetDigest())
+			if err != nil {
+				log.Errorf("error creating Digest from proto %v: %w", f.GetDigest(), err)
+			}
+			files[f.GetPath()] = fmt.Sprintf("%v", dg)
+		}
+	}
+	for _, d := range resPb.GetOutputDirectories() {
+		if d != nil {
+			dg, err := digest.NewFromProto(d.GetTreeDigest())
+			if err != nil {
+				log.Errorf("error creating Digest from proto %v: %w", d.GetTreeDigest(), err)
+			}
+			dirs[d.GetPath()] = fmt.Sprintf("%v", dg)
+		}
+	}
+	for _, fs := range resPb.GetOutputFileSymlinks() {
+		if fs != nil {
+			fileSymlinks[fs.GetPath()+"->"+fs.GetTarget()] = files[fs.GetTarget()]
+		}
+	}
+	for _, ds := range resPb.GetOutputDirectorySymlinks() {
+		if ds != nil {
+			dirSymlinks[ds.GetPath()+"->"+ds.GetTarget()] = dirs[ds.GetTarget()]
+		}
+	}
+
+	return &IO{
+		RootDg:  fmt.Sprintf("%v", rootDg),
+		Inputs:  &Inputs{inputs},
+		Outputs: &Outputs{Files: files, Dirs: dirs, FileSymlinks: fileSymlinks, DirSymlinks: dirSymlinks},
+	}, nil
+
+}

go/pkg/tool/tool_test.go

@@ -444,3 +444,58 @@ func TestTool_UploadBlob(t *testing.T) {
 		t.Fatalf("Expected 1 write for blob '%v', got %v", dg.String(), cas.BlobWrites(dg))
 	}
 }
+
+func TestTool_GetIO(t *testing.T) {
+	e, cleanup := fakes.NewTestEnv(t)
+	defer cleanup()
+	cmd := &command.Command{
+		Args:     []string{"tool"},
+		ExecRoot: e.ExecRoot,
+		InputSpec: &command.InputSpec{
+			Inputs:          []string{"foo.c", "bar.c"},
+			SymlinkBehavior: command.PreserveSymlink,
+		},
+	}
+	opt := command.DefaultExecutionOptions()
+	_, acDg, _, _ := e.Set(
+		cmd,
+		opt,
+		&command.Result{Status: command.CacheHitResultStatus},
+		&fakes.InputFile{Path: "foo.c", Contents: "foo"},
+		&fakes.InputSymlink{Path: "bar.c", Content: "bar", Target: "previous_dir/old_target_file"},
+		&fakes.OutputFile{Path: "a/b/out", Contents: "foo"},
+		&fakes.OutputSymlink{Path: "a/b/sl", Target: "a/b/out"},
+	)
+	toolClient := &Client{GrpcClient: e.Client.GrpcClient}
+	tmpDir := t.TempDir()
+	io, err := toolClient.GetIO(context.Background(), acDg.String())
+	if err != nil {
+		t.Fatalf("DownloadActionResult(%v,%v) failed: %v", acDg.String(), tmpDir, err)
+	}
+
+	getInputRootDg := io.RootDg
+	wantInputRootDg := "ea520b417ef2887249b4ea2d24ff64f5d7d6c419eb1a0ce2067c39ece5c37b56/206"
+	if diff := cmp.Diff(wantInputRootDg, getInputRootDg); diff != "" {
+		t.Errorf("GetIO returned diff in Inputs' list: (-want +got)\n%s", diff)
+	}
+	getInputs := io.Inputs.Paths
+	wantInputs := map[string]string{
+		"foo.c":                               "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/3",
+		"previous_dir/old_target_file":        "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9/3",
+		"bar.c->previous_dir/old_target_file": "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9/3",
+	}
+	if diff := cmp.Diff(wantInputs, getInputs); diff != "" {
+		t.Errorf("GetIO returned diff in Inputs' list: (-want +got)\n%s", diff)
+	}
+
+	getOutputFiles := io.Outputs.Files
+	wantOutputFiles := map[string]string{"a/b/out": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/3"}
+	if diff := cmp.Diff(wantOutputFiles, getOutputFiles); diff != "" {
+		t.Errorf("GetIO returned diff in Outputs' list: (-want +got)\n%s", diff)
+	}
+	getOutputFileSyms := io.Outputs.FileSymlinks
+	wantOutputFileSyms := map[string]string{"a/b/sl->a/b/out": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/3"}
+	if diff := cmp.Diff(wantOutputFileSyms, getOutputFileSyms); diff != "" {
+		t.Errorf("GetIO returned diff in Outputs' list: (-want +got)\n%s", diff)
+	}
+}