Skip to content

add blobaccess for Spanner/GCS #194

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
srago wants to merge 6 commits into
base: main
Choose a base branch
from
Open

add blobaccess for Spanner/GCS #194

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d825170
add blobaccess for Spanner/GCS
srago Mar 10, 2024
1806717
rename spanner_blob_access to spanner_gcs_blob_access
srago Mar 11, 2024
8d9e809
address code review comments from first PR attempt
srago Mar 18, 2024
f1baeef
update spannerGcs blob expiration time to use time.Duration
srago Mar 19, 2024
90d6a4e
bug fixes to get builds working again
srago Mar 19, 2024
34729d6
remove storage type from spannerGcsBlobAccess config
srago Mar 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions pkg/blobstore/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ go_library(
"read_buffer_factory.go",
"read_canarying_blob_access.go",
"reference_expanding_blob_access.go",
"spanner_blob_access.go",
"validation_caching_read_buffer_factory.go",
"visit_topologically_sorted_tree.go",
"zip_reading_blob_access.go",
Expand All @@ -38,6 +39,7 @@ go_library(
"//pkg/cloud/gcp",
"//pkg/digest",
"//pkg/eviction",
"//pkg/proto/configuration/blobstore",
"//pkg/proto/fsac",
"//pkg/proto/icas",
"//pkg/proto/iscc",
Expand All @@ -47,6 +49,11 @@ go_library(
"@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution",
"@com_github_klauspost_compress//zstd",
"@com_github_prometheus_client_golang//prometheus",
"@com_google_cloud_go_spanner//:spanner",
"@com_google_cloud_go_spanner//admin/database/apiv1",
"@com_google_cloud_go_storage//:storage",
"@org_golang_google_api//iterator",
"@org_golang_google_genproto//googleapis/spanner/admin/database/v1:database",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//status",
"@org_golang_google_protobuf//encoding/protowire",
Expand Down
22 changes: 22 additions & 0 deletions pkg/blobstore/configuration/new_blob_access.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,28 @@ func (nc *simpleNestedBlobAccessCreator) newNestedBlobAccessBare(configuration *
BlobAccess: localBlobAccess,
DigestKeyFormat: digestKeyFormat,
}, backendType, nil
case *pb.BlobAccessConfiguration_Spanner:
var digestKeyFormat digest.KeyFormat
if backend.Spanner.StorageType == pb.StorageType_ACTION_CACHE {
digestKeyFormat = digest.KeyWithInstance
} else if backend.Spanner.StorageType == pb.StorageType_CASTORE {
digestKeyFormat = digest.KeyWithoutInstance
} else {
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Unknown Spanner storage type")
}
blobAccess, err := blobstore.NewSpannerBlobAccess(
backend.Spanner.SpannerDbName,
backend.Spanner.GcsBucketName,
readBufferFactory,
backend.Spanner.StorageType,
backend.Spanner.ExpirationDays)
if err != nil {
return BlobAccessInfo{}, "", util.StatusWrap(err, "Failed to create Spanner blob access")
}
return BlobAccessInfo{
BlobAccess: blobAccess,
DigestKeyFormat: digestKeyFormat,
}, "spanner", nil
case *pb.BlobAccessConfiguration_ReadFallback:
primary, err := nc.NewNestedBlobAccess(backend.ReadFallback.Primary, creator)
if err != nil {
Expand Down
Loading