Skip to content

Commit ba06d08

Browse files
jvolkmanjvolkman
authored
feat: add optional Platform to action cache Command and Actions. (#72)
1 parent 965663f commit ba06d08

6 files changed

Lines changed: 119 additions & 22 deletions

File tree

‎pkg/configuration/new_asset_store.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ func NewAssetStoreFromConfiguration(
5252
if err != nil {
5353
return nil, err
5454
}
55-
assetStore = storage.NewActionCacheAssetStore(actionCache.BlobAccess, contentAddressableStorage.BlobAccess, maximumMessageSizeBytes)
55+
assetStore = storage.NewActionCacheAssetStore(actionCache.BlobAccess, contentAddressableStorage.BlobAccess, maximumMessageSizeBytes, configuration.ActionCachePlatform)
5656
default:
5757
return nil, status.Errorf(codes.InvalidArgument, "Asset Cache configuration is invalid as no supported Asset Cache is defined.")
5858
}

‎pkg/proto/configuration/bb_remote_asset/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ proto_library(
99
visibility = ["//visibility:public"],
1010
deps = [
1111
"//pkg/proto/configuration/bb_remote_asset/fetch:fetch_proto",
12+
"@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_proto",
1213
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/auth:auth_proto",
1314
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore:blobstore_proto",
1415
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/global:global_proto",
@@ -24,6 +25,7 @@ go_proto_library(
2425
visibility = ["//visibility:public"],
2526
deps = [
2627
"//pkg/proto/configuration/bb_remote_asset/fetch",
28+
"@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto",
2729
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/auth",
2830
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore",
2931
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/global",

‎pkg/proto/configuration/bb_remote_asset/bb_remote_asset.pb.go‎

Lines changed: 23 additions & 11 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/proto/configuration/bb_remote_asset/bb_remote_asset.proto‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ syntax = "proto3";
22

33
package buildbarn.configuration.bb_remote_asset;
44

5+
import "build/bazel/remote/execution/v2/remote_execution.proto";
56
import "github.com/buildbarn/bb-remote-asset/pkg/proto/configuration/bb_remote_asset/fetch/fetcher.proto";
67
import "github.com/buildbarn/bb-storage/pkg/proto/configuration/auth/auth.proto";
78
import "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto";
@@ -66,4 +67,8 @@ message AssetCacheConfiguration {
6667
// Cache assets in an existing action cache
6768
buildbarn.configuration.blobstore.BlobAccessConfiguration action_cache = 2;
6869
}
70+
71+
// Optional platform properties to attach to actions created for asset
72+
// mappings.
73+
build.bazel.remote.execution.v2.Platform action_cache_platform = 3;
6974
}

‎pkg/storage/action_cache_asset_store.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,17 @@ type actionCacheAssetStore struct {
3636
actionCache blobstore.BlobAccess
3737
contentAddressableStorage blobstore.BlobAccess
3838
maximumMessageSizeBytes int
39+
platform *remoteexecution.Platform
3940
}
4041

4142
// NewActionCacheAssetStore creates a new AssetStore which stores it's
4243
// references as ActionResults in the Action Cache.
43-
func NewActionCacheAssetStore(actionCache, contentAddressableStorage blobstore.BlobAccess, maximumMessageSizeBytes int) AssetStore {
44+
func NewActionCacheAssetStore(actionCache, contentAddressableStorage blobstore.BlobAccess, maximumMessageSizeBytes int, platform *remoteexecution.Platform) AssetStore {
4445
return &actionCacheAssetStore{
4546
actionCache: actionCache,
4647
contentAddressableStorage: contentAddressableStorage,
4748
maximumMessageSizeBytes: maximumMessageSizeBytes,
49+
platform: platform,
4850
}
4951
}
5052

@@ -88,6 +90,7 @@ func (rs *actionCacheAssetStore) assetReferenceToAction(ref *asset.AssetReferenc
8890
Arguments: ref.Uris,
8991
OutputPaths: []string{"out"},
9092
OutputDirectoryFormat: remoteexecution.Command_TREE_AND_DIRECTORY,
93+
Platform: rs.platform,
9194
}
9295
_, commandDigest, err := ProtoSerialise(command, digestFunction)
9396
if err != nil {
@@ -97,10 +100,14 @@ func (rs *actionCacheAssetStore) assetReferenceToAction(ref *asset.AssetReferenc
97100
action = &remoteexecution.Action{
98101
CommandDigest: commandDigest.GetProto(),
99102
InputRootDigest: directoryDigest.GetProto(),
103+
Platform: rs.platform,
100104
}
101105
} else {
102106
// Generate a command based on the qualifiers
103107
command := commandGenerator(ref.Uris[0])
108+
if command.Platform == nil {
109+
command.Platform = rs.platform
110+
}
104111
_, commandDigest, err := ProtoSerialise(command, digestFunction)
105112
if err != nil {
106113
return nil, nil, err
@@ -109,6 +116,7 @@ func (rs *actionCacheAssetStore) assetReferenceToAction(ref *asset.AssetReferenc
109116
action = &remoteexecution.Action{
110117
CommandDigest: commandDigest.GetProto(),
111118
InputRootDigest: EmptyDigest(digestFunction).GetProto(),
119+
Platform: command.Platform,
112120
}
113121
}
114122

‎pkg/storage/action_cache_asset_store_test.go‎

Lines changed: 79 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,77 @@ func TestActionCacheAssetStorePutBlob(t *testing.T) {
9191
}
9292
return status.Error(codes.Internal, "Blob digest not found")
9393
})
94-
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
94+
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)
95+
96+
err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
97+
require.NoError(t, err)
98+
}
99+
100+
func TestActionCacheAssetStorePutBlobWithPlatform(t *testing.T) {
101+
ctrl, ctx := gomock.WithContext(context.Background(), t)
102+
103+
instanceName := util.Must(digest.NewInstanceName(""))
104+
digestFunction, err := instanceName.GetDigestFunction(remoteexecution.DigestFunction_SHA256, 0)
105+
require.NoError(t, err)
106+
107+
blobDigest := &remoteexecution.Digest{
108+
Hash: "58de0f27ce0f781e5c109f18b0ee6905bdf64f2b1009e225ac67a27f656a0643",
109+
SizeBytes: 111,
110+
}
111+
uri := "https://example.com/example.txt"
112+
assetRef := storage.NewAssetReference([]string{uri},
113+
[]*remoteasset.Qualifier{{Name: "test", Value: "test"}})
114+
assetData := storage.NewBlobAsset(blobDigest, timestamppb.Now())
115+
116+
platform := &remoteexecution.Platform{
117+
Properties: []*remoteexecution.Platform_Property{
118+
{Name: "OSFamily", Value: "Linux"},
119+
},
120+
}
121+
122+
// Compute expected digests
123+
qr := storage.NewAssetReference(nil, assetRef.Qualifiers)
124+
_, qrDigest, err := storage.ProtoSerialise(qr, digestFunction)
125+
require.NoError(t, err)
126+
127+
directory := &remoteexecution.Directory{
128+
Files: []*remoteexecution.FileNode{{
129+
Name: "AssetReference",
130+
Digest: qrDigest.GetProto(),
131+
}},
132+
}
133+
_, directoryDigest, err := storage.ProtoSerialise(directory, digestFunction)
134+
require.NoError(t, err)
135+
136+
command := &remoteexecution.Command{
137+
Arguments: assetRef.Uris,
138+
OutputPaths: []string{"out"},
139+
OutputDirectoryFormat: remoteexecution.Command_TREE_AND_DIRECTORY,
140+
Platform: platform,
141+
}
142+
_, commandDigest, err := storage.ProtoSerialise(command, digestFunction)
143+
require.NoError(t, err)
144+
145+
action := &remoteexecution.Action{
146+
CommandDigest: commandDigest.GetProto(),
147+
InputRootDigest: directoryDigest.GetProto(),
148+
Platform: platform,
149+
}
150+
_, actionDigest, err := storage.ProtoSerialise(action, digestFunction)
151+
require.NoError(t, err)
152+
153+
ac := mock.NewMockBlobAccess(ctrl)
154+
cas := mock.NewMockBlobAccess(ctrl)
155+
156+
// Expect uploads to CAS with specific digests!
157+
cas.EXPECT().Put(ctx, qrDigest, gomock.Any()).Return(nil)
158+
cas.EXPECT().Put(ctx, directoryDigest, gomock.Any()).Return(nil)
159+
cas.EXPECT().Put(ctx, commandDigest, gomock.Any()).Return(nil)
160+
cas.EXPECT().Put(ctx, actionDigest, gomock.Any()).Return(nil)
161+
162+
ac.EXPECT().Put(ctx, actionDigest, gomock.Any()).Return(nil)
163+
164+
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, platform)
95165

96166
err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
97167
require.NoError(t, err)
@@ -184,7 +254,7 @@ func TestActionCacheAssetStorePutDirectory(t *testing.T) {
184254
}
185255
return status.Error(codes.Internal, "Directory digest not found")
186256
})
187-
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
257+
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)
188258

189259
err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
190260
require.NoError(t, err)
@@ -230,7 +300,7 @@ func TestActionCacheAssetStorePutMalformedDirectory(t *testing.T) {
230300
},
231301
buffer.UserProvided))
232302

233-
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
303+
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)
234304

235305
err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
236306
require.NotNil(t, err)
@@ -337,7 +407,7 @@ func TestActionCacheAssetStorePutRecursiveDirectory(t *testing.T) {
337407
return status.Error(codes.Internal, "Directory digest not found")
338408
})
339409

340-
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
410+
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)
341411

342412
err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
343413
require.NoError(t, err)
@@ -386,7 +456,7 @@ func TestActionCacheAssetStorePutMalformedDirectoryAsBlob(t *testing.T) {
386456
}
387457
return status.Error(codes.Internal, "Directory digest not found")
388458
})
389-
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
459+
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)
390460

391461
err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
392462
require.NoError(t, err)
@@ -424,7 +494,7 @@ func roundTripTest(t *testing.T, assetRef *asset.AssetReference, assetData *asse
424494
return nil
425495
})
426496

427-
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
497+
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)
428498

429499
err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
430500
require.NoError(t, err)
@@ -443,7 +513,7 @@ func roundTripTest(t *testing.T, assetRef *asset.AssetReference, assetData *asse
443513
return buffer.NewBufferFromError(fmt.Errorf("not in AC"))
444514
})
445515

446-
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
516+
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)
447517

448518
asset, err := assetStore.Get(ctx, assetRef, digestFunction)
449519
require.NoError(t, err)
@@ -534,7 +604,7 @@ func TestActionCacheAssetStoreGetBlob(t *testing.T) {
534604
ac := mock.NewMockBlobAccess(ctrl)
535605
cas := mock.NewMockBlobAccess(ctrl)
536606
ac.EXPECT().Get(ctx, actionDigest).Return(buf)
537-
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
607+
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)
538608

539609
_, err = assetStore.Get(ctx, assetRef, digestFunction)
540610
require.NoError(t, err)
@@ -576,7 +646,7 @@ func TestActionCacheAssetStoreGetDirectory(t *testing.T) {
576646
ac := mock.NewMockBlobAccess(ctrl)
577647
cas := mock.NewMockBlobAccess(ctrl)
578648
ac.EXPECT().Get(ctx, actionDigest).Return(buf)
579-
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
649+
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)
580650

581651
_, err = assetStore.Get(ctx, assetRef, digestFunction)
582652
require.NoError(t, err)

0 commit comments

Comments
 (0)