feat: shared bucket support (#92)

* refactor: make template wrapper generic and move into its own package

* feat: add shared bucket support

* fix: replace slashes with double hyphen in volume names

* refactor: create explicit shared bucket spec type

Author: nicolst

cmd/folder-precreator/go.mod

@@ -0,0 +1,58 @@
+module github.com/statisticsnorway/ssbucketeer-folder-precreator
+
+go 1.23.1
+
+require (
+	cloud.google.com/go/storage v1.48.0
+	golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e
+	google.golang.org/api v0.211.0
+)
+
+require (
+	cel.dev/expr v0.16.1 // indirect
+	cloud.google.com/go v0.116.0 // indirect
+	cloud.google.com/go/auth v0.12.1 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect
+	cloud.google.com/go/compute/metadata v0.5.2 // indirect
+	cloud.google.com/go/iam v1.2.2 // indirect
+	cloud.google.com/go/monitoring v1.21.2 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect
+	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect
+	github.com/envoyproxy/go-control-plane v0.13.0 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/s2a-go v0.1.8 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
+	github.com/googleapis/gax-go/v2 v2.14.0 // indirect
+	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.29.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
+	go.opentelemetry.io/otel v1.29.0 // indirect
+	go.opentelemetry.io/otel/metric v1.29.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.29.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.29.0 // indirect
+	go.opentelemetry.io/otel/trace v1.29.0 // indirect
+	golang.org/x/crypto v0.30.0 // indirect
+	golang.org/x/net v0.32.0 // indirect
+	golang.org/x/oauth2 v0.24.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/time v0.8.0 // indirect
+	google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583 // indirect
+	google.golang.org/grpc v1.67.2 // indirect
+	google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect
+	google.golang.org/protobuf v1.35.2 // indirect
+)

cmd/folder-precreator/go.sum

@@ -52,14 +52,15 @@ import (
 )
 
 type config struct {
-	DaplaGroupSaProjectId string                         `env:"DAPLA_GROUP_SA_PROJECT_ID,required,notEmpty"`
-	TeamsFolderNumber     string                         `env:"TEAMS_FOLDER_NUMBER,required,notEmpty"`
-	Stage                 string                         `env:"STAGE,required,notEmpty"`
-	IamProbeImage         string                         `env:"IAM_PROBE_IMAGE"`
-	PrecreatorImage       string                         `env:"PRECREATOR_IMAGE"`
-	ADCGroupEnvName       string                         `env:"ADC_GROUP_ENV_NAME"`
-	GroupConfigs          []controller.AccessGroupConfig `env:"GROUP_CONFIG,required,notEmpty"`
-	AuditSinks            []auditSink                    `env:"AUDIT_SINKS"`
+	DaplaGroupSaProjectId string                          `env:"DAPLA_GROUP_SA_PROJECT_ID,required,notEmpty"`
+	TeamsFolderNumber     string                          `env:"TEAMS_FOLDER_NUMBER,required,notEmpty"`
+	Stage                 string                          `env:"STAGE,required,notEmpty"`
+	IamProbeImage         string                          `env:"IAM_PROBE_IMAGE"`
+	PrecreatorImage       string                          `env:"PRECREATOR_IMAGE"`
+	ADCGroupEnvName       string                          `env:"ADC_GROUP_ENV_NAME"`
+	GroupConfigs          []controller.AccessGroupConfig  `env:"GROUP_CONFIG,required,notEmpty"`
+	AuditSinks            []auditSink                     `env:"AUDIT_SINKS"`
+	SharedBucketTemplate  controller.SharedBucketTemplate `env:"SHARED_BUCKET_TEMPLATE" envDefault:"ssb-{{.TeamName}}-data-delt-{{.BucketShortName}}-{{.Stage}}"`
 }
 
 type auditSink struct {
@@ -228,17 +229,18 @@ func main() {
 
 	if os.Getenv("ENABLE_WEBHOOKS") != "false" {
 		(&controller.StatefulsetMutator{
-			Client:            mgr.GetClient(),
-			Decoder:           admission.NewDecoder(mgr.GetScheme()),
-			Storage:           storageClient,
-			Projects:          projectsClient,
-			Folders:           foldersClient,
-			TeamsFolderNumber: cfg.TeamsFolderNumber,
-			Stage:             cfg.Stage,
-			IamProbeImage:     cfg.IamProbeImage,
-			PrecreatorImage:   cfg.PrecreatorImage,
-			ADCGroupEnvName:   cfg.ADCGroupEnvName,
-			GroupConfigs:      cfg.GroupConfigs,
+			Client:               mgr.GetClient(),
+			Decoder:              admission.NewDecoder(mgr.GetScheme()),
+			Storage:              storageClient,
+			Projects:             projectsClient,
+			Folders:              foldersClient,
+			TeamsFolderNumber:    cfg.TeamsFolderNumber,
+			Stage:                cfg.Stage,
+			IamProbeImage:        cfg.IamProbeImage,
+			PrecreatorImage:      cfg.PrecreatorImage,
+			ADCGroupEnvName:      cfg.ADCGroupEnvName,
+			GroupConfigs:         cfg.GroupConfigs,
+			SharedBucketTemplate: cfg.SharedBucketTemplate,
 		}).SetupWithManager(mgr)
 
 		if err = (&controller.ServiceAccountValidator{

cmd/main.go

@@ -3,14 +3,16 @@ package controller
 import (
 	"fmt"
 	"strings"
-	"text/template"
 	"time"
+
+	"github.com/statisticsnorway/ssbucketeer/internal/template"
 )
 
 const (
 	impersonateGroupAnnotation         = "dapla.ssb.no/impersonate-group"
 	mountBucketsAnnotation             = "dapla.ssb.no/mount-buckets"
 	mountStandardBucketsAnnotation     = "dapla.ssb.no/mount-standard-buckets"
+	mountSharedBucketsAnnotation       = "dapla.ssb.no/mount-shared-buckets"
 	serviceContainerAnnotation         = "dapla.ssb.no/service-container-name"
 	requestedServiceDurationAnnotation = "dapla.ssb.no/requested-service-duration"
 	accessReasonAnnotation             = "dapla.ssb.no/access-reason"
@@ -45,43 +47,11 @@ type Auther interface {
 	UserIsMemberOf(username, group string) (bool, error)
 }
 
-// projectTemplate wraps a *template.Template and implements yaml.Unmarshaler interface
-// so we can unmarshal a template string into a template.
-type projectTemplate struct {
-	template *template.Template
-}
-
-// Execute wraps template.Template.Execute to provide an easier-to-use interface
-// and only allow ProjectTemplateData to be passed as data.
-func (t projectTemplate) Execute(data ProjectTemplateData) (string, error) {
-	if t.template == nil {
-		return "", fmt.Errorf("template is nil, trying to execute for data %q", data)
-	}
-	sb := strings.Builder{}
-	if err := t.template.Execute(&sb, data); err != nil {
-		return "", err
-	}
-	return sb.String(), nil
-}
-
-// UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (t *projectTemplate) UnmarshalYAML(unmarshal func(any) error) error {
-	var templateString string
-	if err := unmarshal(&templateString); err != nil {
-		return fmt.Errorf("unmarshal template string: %w", err)
-	}
-	t.template = template.New(templateString)
-	if _, err := t.template.Parse(templateString); err != nil {
-		return fmt.Errorf("parse template string: %w", err)
-	}
-	return nil
-}
-
 type AccessGroupConfig struct {
-	Name            string          `yaml:"name"`
-	ProjectTemplate projectTemplate `yaml:"projectTemplate"`
-	MaxDuration     time.Duration   `yaml:"maxDuration"`
-	ReasonRequired  bool            `yaml:"reasonRequired"`
+	Name            string                                          `yaml:"name"`
+	ProjectTemplate template.AnonymousTemplate[ProjectTemplateData] `yaml:"projectTemplate"`
+	MaxDuration     time.Duration                                   `yaml:"maxDuration"`
+	ReasonRequired  bool                                            `yaml:"reasonRequired"`
 }
 
 func (c AccessGroupConfig) ToTeam(group string) string {
@@ -93,6 +63,14 @@ type ProjectTemplateData struct {
 	Stage    string
 }
 
+type SharedBucketTemplateData struct {
+	TeamName        string
+	BucketShortName string
+	Stage           string
+}
+
+type SharedBucketTemplate = template.AnonymousTemplate[SharedBucketTemplateData]
+
 type AccessGroupConfigs []AccessGroupConfig
 
 func (cs AccessGroupConfigs) GetConfig(group string) *AccessGroupConfig {

internal/controller/common.go

@@ -30,6 +30,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	klog "sigs.k8s.io/controller-runtime/pkg/log"
@@ -92,7 +93,7 @@ func (r *JobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R
 		replicas = 1
 	}
 
-	sfs.Spec.Replicas = ptr[int32](int32(replicas))
+	sfs.Spec.Replicas = ptr.To(int32(replicas))
 	sfs.Annotations[iamProbeStatus] = iamProbeDone
 
 	// Find the service container, so we can add volumeMounts
@@ -132,7 +133,7 @@ func (r *JobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R
 				LocalObjectReference: corev1.LocalObjectReference{
 					Name: refreshBucketsConfigMap.Name,
 				},
-				DefaultMode: ptr[int32](0o555), // Read, execute
+				DefaultMode: ptr.To[int32](0o555), // Read, execute
 			},
 		},
 	}

internal/controller/job_controller.go

@@ -12,12 +12,14 @@ import (
 	rm "cloud.google.com/go/resourcemanager/apiv3"
 	rmpb "cloud.google.com/go/resourcemanager/apiv3/resourcemanagerpb"
 	"cloud.google.com/go/storage"
+	"github.com/statisticsnorway/ssbucketeer/internal/template"
 	"google.golang.org/api/iterator"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	klog "sigs.k8s.io/controller-runtime/pkg/log"
@@ -50,6 +52,13 @@ type StatefulsetMutator struct {
 	ADCGroupEnvName   string
 
 	GroupConfigs AccessGroupConfigs
+
+	SharedBucketTemplate template.AnonymousTemplate[SharedBucketTemplateData]
+}
+
+type SharedBucketSpec struct {
+	Team      string `json:"team"`
+	ShortName string `json:"sharedBucket"`
 }
 
 func (m *StatefulsetMutator) SetupWithManager(mgr ctrl.Manager) {
@@ -120,6 +129,12 @@ func (m *StatefulsetMutator) Handle(ctx context.Context, req admission.Request)
 		}
 	}
 
+	if sharedBuckets, ok := sfs.Annotations[mountSharedBucketsAnnotation]; ok {
+		if err := m.addSharedBuckets(bucketMounts, sharedBuckets); err != nil {
+			log.Error(err, "failed to add shared buckets")
+		}
+	}
+
 	// TODO: Use Dapla Team API for this?
 	if sfs.Annotations[mountStandardBucketsAnnotation] == "true" {
 		if err := m.addStandardBuckets(ctx, team, *groupConfig, bucketMounts); err != nil {
@@ -167,7 +182,7 @@ func getServiceContainer(pod *corev1.PodTemplateSpec, name string) *corev1.Conta
 
 func (m *StatefulsetMutator) launchIamProbe(ctx context.Context, sfs *appsv1.StatefulSet) error {
 	sfs.Annotations[iamProbeStatus] = fmt.Sprintf("%s%d", iamProbeRunningPrefix, *sfs.Spec.Replicas)
-	sfs.Spec.Replicas = ptr[int32](0)
+	sfs.Spec.Replicas = ptr.To[int32](0)
 
 	probeJob := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
@@ -178,8 +193,8 @@ func (m *StatefulsetMutator) launchIamProbe(ctx context.Context, sfs *appsv1.Sta
 			},
 		},
 		Spec: batchv1.JobSpec{
-			ActiveDeadlineSeconds:   ptr[int64](300),
-			TTLSecondsAfterFinished: ptr[int32](0),
+			ActiveDeadlineSeconds:   ptr.To[int64](300),
+			TTLSecondsAfterFinished: ptr.To[int32](0),
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Annotations: map[string]string{
@@ -257,7 +272,7 @@ func (m *StatefulsetMutator) addStandardBuckets(ctx context.Context, team string
 
 	projectName, err := gc.ProjectTemplate.Execute(ProjectTemplateData{TeamName: team, Stage: m.Stage})
 	if err != nil {
-		return fmt.Errorf("execute template %s: %w", gc.ProjectTemplate.template.Name(), err)
+		return fmt.Errorf("execute template %s: %w", gc.ProjectTemplate.Name(), err)
 	}
 
 	projectIt := m.Projects.SearchProjects(ctx, &rmpb.SearchProjectsRequest{
@@ -291,6 +306,28 @@ func (m *StatefulsetMutator) addStandardBuckets(ctx context.Context, team string
 	return nil
 }
 
+func (m *StatefulsetMutator) addSharedBuckets(bucketMounts map[string]string, sharedBuckets string) error {
+	bucketSpecs := []SharedBucketSpec{}
+
+	if err := json.Unmarshal([]byte(sharedBuckets), &bucketSpecs); err != nil {
+		return err
+	}
+
+	for _, bucket := range bucketSpecs {
+		mountPoint := fmt.Sprintf("shared/%s/%s", bucket.Team, bucket.ShortName)
+		bucket, err := m.SharedBucketTemplate.Execute(SharedBucketTemplateData{
+			TeamName:        bucket.Team,
+			BucketShortName: bucket.ShortName,
+			Stage:           m.Stage,
+		})
+		if err != nil {
+			return err
+		}
+		bucketMounts[mountPoint] = bucket
+	}
+	return nil
+}
+
 func ensurePodAnnotations(podTemplate *corev1.PodTemplateSpec) {
 	if podTemplate.Annotations == nil {
 		podTemplate.Annotations = make(map[string]string, 2)

internal/controller/statefulset_webhook.go

@@ -3,15 +3,12 @@ package controller
 import (
 	"fmt"
 	"slices"
+	"strings"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/ptr"
 )
 
-// ptr is a convenience function for generating pointers of primitive types
-func ptr[T any](t T) *T {
-	return &t
-}
-
 func volumeNameIs(name string) func(v corev1.Volume) bool {
 	return func(v corev1.Volume) bool {
 		return v.Name == name
@@ -28,7 +25,7 @@ func addBucketsToPodSpec(podspec *corev1.PodSpec, container *corev1.Container, b
 	volumes := make([]corev1.Volume, 0, len(bucketMounts))
 	volumeMounts := make([]corev1.VolumeMount, 0, len(bucketMounts))
 	for mountPoint, bucket := range bucketMounts {
-		volumeName := fmt.Sprintf("gcsfuse-%s", mountPoint)
+		volumeName := fmt.Sprintf("gcsfuse-%s", strings.ReplaceAll(mountPoint, "/", "--"))
 
 		// TODO: Test whether the group has read/write access
 		if !slices.ContainsFunc(podspec.Volumes, volumeNameIs(volumeName)) {
@@ -37,7 +34,7 @@ func addBucketsToPodSpec(podspec *corev1.PodSpec, container *corev1.Container, b
 				VolumeSource: corev1.VolumeSource{
 					CSI: &corev1.CSIVolumeSource{
 						Driver:   "gcsfuse.csi.storage.gke.io",
-						ReadOnly: ptr(false),
+						ReadOnly: ptr.To(false),
 						VolumeAttributes: map[string]string{
 							"bucketName":             bucket,
 							"mountOptions":           "uid=1000,gid=100",
@@ -72,7 +69,7 @@ func addBucketsToPodSpec(podspec *corev1.PodSpec, container *corev1.Container, b
 			Name:  "bucket-folders-precreator",
 		}
 		for mountPoint, bucket := range bucketMounts {
-			volumeName := fmt.Sprintf("gcsfuse-%s", mountPoint)
+			volumeName := fmt.Sprintf("gcsfuse-%s", strings.ReplaceAll(mountPoint, "/", "--"))
 			precreator.VolumeMounts = append(precreator.VolumeMounts, corev1.VolumeMount{
 				Name:      volumeName,
 				MountPath: fmt.Sprintf("/buckets/%s", bucket),