Skip to content

feat: add optional Platform to action cache Command and Actions. #72

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mortenmj merged 1 commit into from
May 2, 2026
Merged

feat: add optional Platform to action cache Command and Actions. #72

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pkg/configuration/new_asset_store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func NewAssetStoreFromConfiguration(
if err != nil {
return nil, err
}
assetStore = storage.NewActionCacheAssetStore(actionCache.BlobAccess, contentAddressableStorage.BlobAccess, maximumMessageSizeBytes)
assetStore = storage.NewActionCacheAssetStore(actionCache.BlobAccess, contentAddressableStorage.BlobAccess, maximumMessageSizeBytes, configuration.ActionCachePlatform)
default:
return nil, status.Errorf(codes.InvalidArgument, "Asset Cache configuration is invalid as no supported Asset Cache is defined.")
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/proto/configuration/bb_remote_asset/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ proto_library(
visibility = ["//visibility:public"],
deps = [
"//pkg/proto/configuration/bb_remote_asset/fetch:fetch_proto",
"@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_proto",
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/auth:auth_proto",
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore:blobstore_proto",
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/global:global_proto",
Expand All @@ -24,6 +25,7 @@ go_proto_library(
visibility = ["//visibility:public"],
deps = [
"//pkg/proto/configuration/bb_remote_asset/fetch",
"@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto",
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/auth",
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/blobstore",
"@com_github_buildbarn_bb_storage//pkg/proto/configuration/global",
Expand Down
34 changes: 23 additions & 11 deletions pkg/proto/configuration/bb_remote_asset/bb_remote_asset.pb.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions pkg/proto/configuration/bb_remote_asset/bb_remote_asset.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ syntax = "proto3";

package buildbarn.configuration.bb_remote_asset;

import "build/bazel/remote/execution/v2/remote_execution.proto";
import "github.com/buildbarn/bb-remote-asset/pkg/proto/configuration/bb_remote_asset/fetch/fetcher.proto";
import "github.com/buildbarn/bb-storage/pkg/proto/configuration/auth/auth.proto";
import "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore/blobstore.proto";
Expand Down Expand Up @@ -66,4 +67,8 @@ message AssetCacheConfiguration {
// Cache assets in an existing action cache
buildbarn.configuration.blobstore.BlobAccessConfiguration action_cache = 2;
}

// Optional platform properties to attach to actions created for asset
// mappings.
build.bazel.remote.execution.v2.Platform action_cache_platform = 3;
}
10 changes: 9 additions & 1 deletion pkg/storage/action_cache_asset_store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,17 @@ type actionCacheAssetStore struct {
actionCache blobstore.BlobAccess
contentAddressableStorage blobstore.BlobAccess
maximumMessageSizeBytes int
platform *remoteexecution.Platform
}

// NewActionCacheAssetStore creates a new AssetStore which stores it's
// references as ActionResults in the Action Cache.
func NewActionCacheAssetStore(actionCache, contentAddressableStorage blobstore.BlobAccess, maximumMessageSizeBytes int) AssetStore {
func NewActionCacheAssetStore(actionCache, contentAddressableStorage blobstore.BlobAccess, maximumMessageSizeBytes int, platform *remoteexecution.Platform) AssetStore {
return &actionCacheAssetStore{
actionCache: actionCache,
contentAddressableStorage: contentAddressableStorage,
maximumMessageSizeBytes: maximumMessageSizeBytes,
platform: platform,
}
}

Expand Down Expand Up @@ -88,6 +90,7 @@ func (rs *actionCacheAssetStore) assetReferenceToAction(ref *asset.AssetReferenc
Arguments: ref.Uris,
OutputPaths: []string{"out"},
OutputDirectoryFormat: remoteexecution.Command_TREE_AND_DIRECTORY,
Platform: rs.platform,
}
_, commandDigest, err := ProtoSerialise(command, digestFunction)
if err != nil {
Expand All @@ -97,10 +100,14 @@ func (rs *actionCacheAssetStore) assetReferenceToAction(ref *asset.AssetReferenc
action = &remoteexecution.Action{
CommandDigest: commandDigest.GetProto(),
InputRootDigest: directoryDigest.GetProto(),
Platform: rs.platform,
}
} else {
// Generate a command based on the qualifiers
command := commandGenerator(ref.Uris[0])
if command.Platform == nil {
command.Platform = rs.platform
}
_, commandDigest, err := ProtoSerialise(command, digestFunction)
if err != nil {
return nil, nil, err
Expand All @@ -109,6 +116,7 @@ func (rs *actionCacheAssetStore) assetReferenceToAction(ref *asset.AssetReferenc
action = &remoteexecution.Action{
CommandDigest: commandDigest.GetProto(),
InputRootDigest: EmptyDigest(digestFunction).GetProto(),
Platform: command.Platform,
}
}

Expand Down
88 changes: 79 additions & 9 deletions pkg/storage/action_cache_asset_store_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,77 @@ func TestActionCacheAssetStorePutBlob(t *testing.T) {
}
return status.Error(codes.Internal, "Blob digest not found")
})
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)

err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
require.NoError(t, err)
}

func TestActionCacheAssetStorePutBlobWithPlatform(t *testing.T) {
ctrl, ctx := gomock.WithContext(context.Background(), t)

instanceName := util.Must(digest.NewInstanceName(""))
digestFunction, err := instanceName.GetDigestFunction(remoteexecution.DigestFunction_SHA256, 0)
require.NoError(t, err)

blobDigest := &remoteexecution.Digest{
Hash: "58de0f27ce0f781e5c109f18b0ee6905bdf64f2b1009e225ac67a27f656a0643",
SizeBytes: 111,
}
uri := "https://example.com/example.txt"
assetRef := storage.NewAssetReference([]string{uri},
[]*remoteasset.Qualifier{{Name: "test", Value: "test"}})
assetData := storage.NewBlobAsset(blobDigest, timestamppb.Now())

platform := &remoteexecution.Platform{
Properties: []*remoteexecution.Platform_Property{
{Name: "OSFamily", Value: "Linux"},
},
}

// Compute expected digests
qr := storage.NewAssetReference(nil, assetRef.Qualifiers)
_, qrDigest, err := storage.ProtoSerialise(qr, digestFunction)
require.NoError(t, err)

directory := &remoteexecution.Directory{
Files: []*remoteexecution.FileNode{{
Name: "AssetReference",
Digest: qrDigest.GetProto(),
}},
}
_, directoryDigest, err := storage.ProtoSerialise(directory, digestFunction)
require.NoError(t, err)

command := &remoteexecution.Command{
Arguments: assetRef.Uris,
OutputPaths: []string{"out"},
OutputDirectoryFormat: remoteexecution.Command_TREE_AND_DIRECTORY,
Platform: platform,
}
_, commandDigest, err := storage.ProtoSerialise(command, digestFunction)
require.NoError(t, err)

action := &remoteexecution.Action{
CommandDigest: commandDigest.GetProto(),
InputRootDigest: directoryDigest.GetProto(),
Platform: platform,
}
_, actionDigest, err := storage.ProtoSerialise(action, digestFunction)
require.NoError(t, err)

ac := mock.NewMockBlobAccess(ctrl)
cas := mock.NewMockBlobAccess(ctrl)

// Expect uploads to CAS with specific digests!
cas.EXPECT().Put(ctx, qrDigest, gomock.Any()).Return(nil)
cas.EXPECT().Put(ctx, directoryDigest, gomock.Any()).Return(nil)
cas.EXPECT().Put(ctx, commandDigest, gomock.Any()).Return(nil)
cas.EXPECT().Put(ctx, actionDigest, gomock.Any()).Return(nil)

ac.EXPECT().Put(ctx, actionDigest, gomock.Any()).Return(nil)

assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, platform)

err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
require.NoError(t, err)
Expand Down Expand Up @@ -184,7 +254,7 @@ func TestActionCacheAssetStorePutDirectory(t *testing.T) {
}
return status.Error(codes.Internal, "Directory digest not found")
})
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)

err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
require.NoError(t, err)
Expand Down Expand Up @@ -230,7 +300,7 @@ func TestActionCacheAssetStorePutMalformedDirectory(t *testing.T) {
},
buffer.UserProvided))

assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)

err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
require.NotNil(t, err)
Expand Down Expand Up @@ -337,7 +407,7 @@ func TestActionCacheAssetStorePutRecursiveDirectory(t *testing.T) {
return status.Error(codes.Internal, "Directory digest not found")
})

assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)

err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
require.NoError(t, err)
Expand Down Expand Up @@ -386,7 +456,7 @@ func TestActionCacheAssetStorePutMalformedDirectoryAsBlob(t *testing.T) {
}
return status.Error(codes.Internal, "Directory digest not found")
})
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)

err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
require.NoError(t, err)
Expand Down Expand Up @@ -424,7 +494,7 @@ func roundTripTest(t *testing.T, assetRef *asset.AssetReference, assetData *asse
return nil
})

assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)

err = assetStore.Put(ctx, assetRef, assetData, digestFunction)
require.NoError(t, err)
Expand All @@ -443,7 +513,7 @@ func roundTripTest(t *testing.T, assetRef *asset.AssetReference, assetData *asse
return buffer.NewBufferFromError(fmt.Errorf("not in AC"))
})

assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)

asset, err := assetStore.Get(ctx, assetRef, digestFunction)
require.NoError(t, err)
Expand Down Expand Up @@ -534,7 +604,7 @@ func TestActionCacheAssetStoreGetBlob(t *testing.T) {
ac := mock.NewMockBlobAccess(ctrl)
cas := mock.NewMockBlobAccess(ctrl)
ac.EXPECT().Get(ctx, actionDigest).Return(buf)
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)

_, err = assetStore.Get(ctx, assetRef, digestFunction)
require.NoError(t, err)
Expand Down Expand Up @@ -576,7 +646,7 @@ func TestActionCacheAssetStoreGetDirectory(t *testing.T) {
ac := mock.NewMockBlobAccess(ctrl)
cas := mock.NewMockBlobAccess(ctrl)
ac.EXPECT().Get(ctx, actionDigest).Return(buf)
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024)
assetStore := storage.NewActionCacheAssetStore(ac, cas, 16*1024*1024, nil)

_, err = assetStore.Get(ctx, assetRef, digestFunction)
require.NoError(t, err)
Expand Down