Skip to content

Commit 2e39e56

Browse files
ywmei-brt1ywmei-brt1
committed
Get input output with symlinks
Add a function to return inputs and outputs. Note that symlinks point foo to bar will be represented as foo->bar.
1 parent bcc51f0 commit 2e39e56

File tree

3 files changed

+191
-13
lines changed

3 files changed

+191
-13
lines changed

‎go/pkg/fakes/server.go‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import (
2828
repb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
2929
bsgrpc "google.golang.org/genproto/googleapis/bytestream"
3030
bspb "google.golang.org/genproto/googleapis/bytestream"
31-
anypb "google.golang.org/protobuf/types/known/anypb"
31+
"google.golang.org/protobuf/types/known/anypb"
3232
dpb "google.golang.org/protobuf/types/known/durationpb"
3333
tspb "google.golang.org/protobuf/types/known/timestamppb"
3434
)
@@ -304,6 +304,26 @@ func (f *InputFile) apply(ac *repb.ActionResult, s *Server, execRoot string) err
304304
return nil
305305
}
306306

307+
// InputSymlink to be made available to the fake action.
308+
type InputSymlink struct {
309+
Path string // newname
310+
Content string
311+
Target string // oldname
312+
}
313+
314+
// Apply puts the target file in the fake CAS and create a symlink in OS.
315+
func (ins *InputSymlink) apply(ac *repb.ActionResult, s *Server, execRoot string) error {
316+
inf := InputFile{Path: ins.Target, Contents: ins.Content}
317+
if err := inf.apply(ac, s, execRoot); err != nil {
318+
return err
319+
}
320+
// create a symlink from old name (target) to new name (path)
321+
if err := os.Symlink(filepath.Join(execRoot, ins.Target), filepath.Join(execRoot, ins.Path)); err != nil {
322+
return fmt.Errorf("error creating symlink: %w", err)
323+
}
324+
return nil
325+
}
326+
307327
// OutputFile is to be added as an output of the fake action.
308328
type OutputFile struct {
309329
Path string

‎go/pkg/tool/tool.go‎

Lines changed: 111 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -692,7 +692,7 @@ func (c *Client) formatAction(ctx context.Context, actionProto *repb.Action, res
692692
}
693693
showActionRes.WriteString("\nInputs\n======\n")
694694
log.Infof("Fetching input tree from input root digest..")
695-
inpTree, _, err := c.getInputTree(ctx, actionProto.GetInputRootDigest())
695+
inpTree, _, _, err := c.getInputTree(ctx, actionProto.GetInputRootDigest())
696696
if err != nil {
697697
showActionRes.WriteString("Failed to fetch input tree:\n")
698698
showActionRes.WriteString(err.Error())
@@ -759,7 +759,7 @@ func (c *Client) getOutputs(ctx context.Context, actionRes *repb.ActionResult) (
759759
return "", err
760760
}
761761

762-
outputs, _, err := c.flattenTree(ctx, outDirTree)
762+
outputs, _, _, err := c.flattenTree(ctx, outDirTree)
763763
if err != nil {
764764
return "", err
765765
}
@@ -770,41 +770,41 @@ func (c *Client) getOutputs(ctx context.Context, actionRes *repb.ActionResult) (
770770
return res.String(), nil
771771
}
772772

773-
func (c *Client) getInputTree(ctx context.Context, root *repb.Digest) (string, []string, error) {
773+
func (c *Client) getInputTree(ctx context.Context, root *repb.Digest) (string, map[string]string, map[string]string, error) {
774774
var res bytes.Buffer
775775

776776
dg, err := digest.NewFromProto(root)
777777
if err != nil {
778-
return "", nil, err
778+
return "", nil, nil, err
779779
}
780780
res.WriteString(fmt.Sprintf("[Root directory digest: %v]", dg))
781781

782782
dirs, err := c.GrpcClient.GetDirectoryTree(ctx, root)
783783
if err != nil {
784-
return "", nil, err
784+
return "", nil, nil, err
785785
}
786786
if len(dirs) == 0 {
787-
return "", nil, fmt.Errorf("Empty directories returned by GetTree for %v", dg)
787+
return "", nil, nil, fmt.Errorf("Empty directories returned by GetTree for %v", dg)
788788
}
789789
t := &repb.Tree{
790790
Root: dirs[0],
791791
Children: dirs,
792792
}
793-
inputs, paths, err := c.flattenTree(ctx, t)
793+
inputs, paths, symlinks, err := c.flattenTree(ctx, t)
794794
if err != nil {
795-
return "", nil, err
795+
return "", nil, nil, err
796796
}
797797
res.WriteString("\n")
798798
res.WriteString(inputs)
799799

800-
return res.String(), paths, nil
800+
return res.String(), paths, symlinks, nil
801801
}
802802

803-
func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, []string, error) {
803+
func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, map[string]string, map[string]string, error) {
804804
var res bytes.Buffer
805805
outputs, err := c.GrpcClient.FlattenTree(t, "")
806806
if err != nil {
807-
return "", nil, err
807+
return "", nil, nil, err
808808
}
809809
// Sort the values by path.
810810
paths := make([]string, 0, len(outputs))
@@ -816,8 +816,11 @@ func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, []strin
816816
paths = append(paths, path)
817817
}
818818
sort.Strings(paths)
819+
pathToDgs := make(map[string]string, len(paths))
820+
symToDgs := map[string]string{}
819821
for _, path := range paths {
820822
output := outputs[path]
823+
dg := output.Digest
821824
var np string
822825
if output.NodeProperties != nil {
823826
np = fmt.Sprintf(" [Node properties: %v]", prototext.MarshalOptions{Multiline: false}.Format(output.NodeProperties))
@@ -826,11 +829,18 @@ func (c *Client) flattenTree(ctx context.Context, t *repb.Tree) (string, []strin
826829
res.WriteString(fmt.Sprintf("%v: [Directory digest: %v]%s\n", path, output.Digest, np))
827830
} else if output.SymlinkTarget != "" {
828831
res.WriteString(fmt.Sprintf("%v: [Symlink digest: %v, Symlink Target: %v]%s\n", path, output.Digest, output.SymlinkTarget, np))
832+
path = path + "->" + output.SymlinkTarget
833+
if o, ok := outputs[output.SymlinkTarget]; ok {
834+
dg = o.Digest
835+
}
836+
symToDgs[path] = fmt.Sprintf("%v", dg)
837+
continue
829838
} else {
830839
res.WriteString(fmt.Sprintf("%v: [File digest: %v]%s\n", path, output.Digest, np))
831840
}
841+
pathToDgs[path] = fmt.Sprintf("%v", dg)
832842
}
833-
return res.String(), paths, nil
843+
return res.String(), pathToDgs, symToDgs, nil
834844
}
835845

836846
func (c *Client) getActionResult(ctx context.Context, actionDigest string) (*repb.ActionResult, error) {
@@ -848,3 +858,92 @@ func (c *Client) getActionResult(ctx context.Context, actionDigest string) (*rep
848858
}
849859
return resPb, nil
850860
}
861+
862+
// IO is a collection input root Digest, Inputs, and Outputs for an action.
863+
type IO struct {
864+
RootDg string
865+
Inputs *Inputs
866+
Outputs *Outputs
867+
}
868+
869+
// Inputs are input Paths (with digests) of an action. For a symlink, the key is
870+
// `<path>-><target>`, the value is the dg of the target file.
871+
type Inputs struct {
872+
Paths map[string]string
873+
PathSymlinks map[string]string
874+
}
875+
876+
// Outputs are output Files/Dirs (with digests) of an action. For a symlink, the
877+
// key is `<path>-><target>`, the value is the dg of the target file.
878+
type Outputs struct {
879+
Files map[string]string
880+
Dirs map[string]string
881+
FileSymlinks map[string]string
882+
DirSymlinks map[string]string
883+
}
884+
885+
// GetIO returns the Inputs and Outputs of an action Digest.
886+
func (c *Client) GetIO(ctx context.Context, actionDigest string) (*IO, error) {
887+
acDg, err := digest.NewFromString(actionDigest)
888+
if err != nil {
889+
return nil, fmt.Errorf("error creating action digest: %w", err)
890+
}
891+
actionProto := &repb.Action{}
892+
// Get all the Inputs.
893+
if _, err := c.GrpcClient.ReadProto(ctx, acDg, actionProto); err != nil {
894+
return nil, fmt.Errorf("error reading from proto: %w", err)
895+
}
896+
rootDg, err := digest.NewFromProto(actionProto.GetInputRootDigest())
897+
if err != nil {
898+
return nil, fmt.Errorf("error getting input root digest: %w", err)
899+
}
900+
_, inputPaths, inputSymlinks, err := c.getInputTree(ctx, actionProto.GetInputRootDigest())
901+
if err != nil {
902+
return nil, err
903+
}
904+
// Get all the Outputs.
905+
resPb, err := c.getActionResult(ctx, actionDigest)
906+
if err != nil {
907+
return nil, err
908+
}
909+
910+
files := map[string]string{}
911+
dirs := map[string]string{}
912+
fileSymlinks := map[string]string{}
913+
dirSymlinks := map[string]string{}
914+
for _, f := range resPb.GetOutputFiles() {
915+
if f != nil {
916+
dg, err := digest.NewFromProto(f.GetDigest())
917+
if err != nil {
918+
log.Errorf("error creating Digest from proto %v: %w", f.GetDigest(), err)
919+
}
920+
files[f.GetPath()] = fmt.Sprintf("%v", dg)
921+
}
922+
}
923+
for _, d := range resPb.GetOutputDirectories() {
924+
if d != nil {
925+
dg, err := digest.NewFromProto(d.GetTreeDigest())
926+
if err != nil {
927+
log.Errorf("error creating Digest from proto %v: %w", d.GetTreeDigest(), err)
928+
}
929+
dirs[d.GetPath()] = fmt.Sprintf("%v", dg)
930+
}
931+
}
932+
for _, fs := range resPb.GetOutputFileSymlinks() {
933+
if fs != nil {
934+
fileSymlinks[fs.GetPath()+"->"+fs.GetTarget()] = files[fs.GetTarget()]
935+
}
936+
}
937+
for _, ds := range resPb.GetOutputDirectorySymlinks() {
938+
if ds != nil {
939+
dirSymlinks[ds.GetPath()+"->"+ds.GetTarget()] = dirs[ds.GetTarget()]
940+
}
941+
}
942+
943+
return &IO{
944+
RootDg: fmt.Sprintf("%v", rootDg),
945+
Inputs: &Inputs{Paths: inputPaths, PathSymlinks: inputSymlinks},
946+
Outputs: &Outputs{Files: files, Dirs: dirs, FileSymlinks: fileSymlinks, DirSymlinks: dirSymlinks},
947+
}, nil
948+
949+
}

‎go/pkg/tool/tool_test.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -444,3 +444,62 @@ func TestTool_UploadBlob(t *testing.T) {
444444
t.Fatalf("Expected 1 write for blob '%v', got %v", dg.String(), cas.BlobWrites(dg))
445445
}
446446
}
447+
448+
func TestTool_GetIO(t *testing.T) {
449+
e, cleanup := fakes.NewTestEnv(t)
450+
defer cleanup()
451+
cmd := &command.Command{
452+
Args: []string{"tool"},
453+
ExecRoot: e.ExecRoot,
454+
InputSpec: &command.InputSpec{
455+
Inputs: []string{"foo.c", "bar.c"},
456+
SymlinkBehavior: command.PreserveSymlink,
457+
},
458+
}
459+
opt := command.DefaultExecutionOptions()
460+
_, acDg, _, _ := e.Set(
461+
cmd,
462+
opt,
463+
&command.Result{Status: command.CacheHitResultStatus},
464+
&fakes.InputFile{Path: "foo.c", Contents: "foo"},
465+
&fakes.InputSymlink{Path: "bar.c", Content: "bar", Target: "previous_dir/old_target_file"},
466+
&fakes.OutputFile{Path: "a/b/out", Contents: "foo"},
467+
&fakes.OutputSymlink{Path: "a/b/sl", Target: "a/b/out"},
468+
)
469+
toolClient := &Client{GrpcClient: e.Client.GrpcClient}
470+
tmpDir := t.TempDir()
471+
io, err := toolClient.GetIO(context.Background(), acDg.String())
472+
if err != nil {
473+
t.Fatalf("DownloadActionResult(%v,%v) failed: %v", acDg.String(), tmpDir, err)
474+
}
475+
476+
getInputRootDg := io.RootDg
477+
wantInputRootDg := "ea520b417ef2887249b4ea2d24ff64f5d7d6c419eb1a0ce2067c39ece5c37b56/206"
478+
if diff := cmp.Diff(wantInputRootDg, getInputRootDg); diff != "" {
479+
t.Errorf("GetIO returned diff in Inputs' list: (-want +got)\n%s", diff)
480+
}
481+
getInputs := io.Inputs
482+
wantInputs := Inputs{
483+
Paths: map[string]string{"foo.c": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/3",
484+
"previous_dir/old_target_file": "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9/3"},
485+
PathSymlinks: map[string]string{"bar.c->previous_dir/old_target_file": "fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9/3"},
486+
}
487+
if diff := cmp.Diff(wantInputs.Paths, getInputs.Paths); diff != "" {
488+
t.Errorf("GetIO returned diff in Input Paths: (-want +got)\n%s", diff)
489+
}
490+
if diff := cmp.Diff(wantInputs.PathSymlinks, getInputs.PathSymlinks); diff != "" {
491+
t.Errorf("GetIO returned diff in Inputs Symlinks: (-want +got)\n%s", diff)
492+
}
493+
494+
getOutput := io.Outputs
495+
wantOutpus := Outputs{
496+
Files: map[string]string{"a/b/out": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/3"},
497+
FileSymlinks: map[string]string{"a/b/sl->a/b/out": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae/3"},
498+
}
499+
if diff := cmp.Diff(wantOutpus.Files, getOutput.Files); diff != "" {
500+
t.Errorf("GetIO returned diff in Output Files: (-want +got)\n%s", diff)
501+
}
502+
if diff := cmp.Diff(wantOutpus.FileSymlinks, getOutput.FileSymlinks); diff != "" {
503+
t.Errorf("GetIO returned diff in Output File Symlinks: (-want +got)\n%s", diff)
504+
}
505+
}

0 commit comments

Comments
 (0)