diff --git a/fakestorage/server.go b/fakestorage/server.go index b4367cebc5..e94d32c14e 100644 --- a/fakestorage/server.go +++ b/fakestorage/server.go @@ -117,6 +117,11 @@ type Options struct { // of the Google cloud function such events should be published to. EventOptions EventManagerOptions + // EventConfigs specifies multiple per-bucket Pub/Sub notification + // configurations. Each entry defines a topic, a set of event types, and + // the bucket it applies to. When non-empty this replaces EventOptions. + EventConfigs []EventManagerOptions + // Location used for buckets in the server. BucketsLocation string @@ -158,10 +163,19 @@ func NewServerWithOptions(options Options) (*Server, error) { s.handler = requestCompressHandler(s.handler) s.transport = &muxTransport{handler: s.handler} - s.eventManager, err = notification.NewPubsubEventManager(options.EventOptions, options.Writer) - if err != nil { - return nil, err + configs := options.EventConfigs + if len(configs) == 0 { + configs = []EventManagerOptions{options.EventOptions} + } + var managers []notification.EventManager + for _, cfg := range configs { + mgr, err := notification.NewPubsubEventManager(cfg, options.Writer) + if err != nil { + return nil, err + } + managers = append(managers, mgr) } + s.eventManager = notification.NewMultiEventManager(managers) if options.NoListener { return s, nil @@ -225,7 +239,7 @@ func newServer(options Options) (*Server, error) { externalURL: options.ExternalURL, publicHost: publicHost, options: options, - eventManager: ¬ification.PubsubEventManager{}, + eventManager: notification.NewMultiEventManager(nil), } s.buildMuxer() _, err = s.seed() diff --git a/fakestorage/server_test.go b/fakestorage/server_test.go index 4f6765a408..ea87daa65e 100644 --- a/fakestorage/server_test.go +++ b/fakestorage/server_test.go @@ -1076,6 +1076,114 @@ func TestServerEventNotification(t *testing.T) { } } +func TestServerMultiEventNotification(t *testing.T) { + t.Parallel() + + objA := Object{ + ObjectAttrs: ObjectAttrs{BucketName: "bucket-a", Name: "file-a.txt"}, + Content: []byte("content-a"), + } + objB := Object{ + ObjectAttrs: ObjectAttrs{BucketName: "bucket-b", Name: "file-b.txt"}, + Content: []byte("content-b"), + } + + t.Run("fan-out to multiple managers", func(t *testing.T) { + t.Parallel() + server, err := NewServerWithOptions(Options{}) + if err != nil { + t.Fatal(err) + } + defer server.Stop() + + managerA := &fakeEventManager{} + managerB := &fakeEventManager{} + server.eventManager = notification.NewMultiEventManager([]notification.EventManager{managerA, managerB}) + + if err := server.backend.CreateBucket("bucket-a", backend.BucketAttrs{}); err != nil { + t.Fatal(err) + } + if err := server.backend.CreateBucket("bucket-b", backend.BucketAttrs{}); err != nil { + t.Fatal(err) + } + + if err := createObjectAction(objA)(server.Client()); err != nil { + t.Fatal(err) + } + if err := createObjectAction(objB)(server.Client()); err != nil { + t.Fatal(err) + } + + // Both managers should have seen both Finalize events (no bucket filter on fakeEventManager). + expectedEvents := []fakeEvent{ + {obj: fakeEventFieldsFromObject(objA), eventType: notification.EventFinalize}, + {obj: fakeEventFieldsFromObject(objB), eventType: notification.EventFinalize}, + } + assert.ElementsMatch(t, expectedEvents, managerA.events) + assert.ElementsMatch(t, expectedEvents, managerB.events) + }) + + t.Run("events carry correct bucket names", func(t *testing.T) { + t.Parallel() + server, err := NewServerWithOptions(Options{}) + if err != nil { + t.Fatal(err) + } + defer server.Stop() + + manager := &fakeEventManager{} + server.eventManager = notification.NewMultiEventManager([]notification.EventManager{manager}) + + if err := server.backend.CreateBucket("bucket-a", backend.BucketAttrs{}); err != nil { + t.Fatal(err) + } + if err := server.backend.CreateBucket("bucket-b", backend.BucketAttrs{}); err != nil { + t.Fatal(err) + } + + if err := createObjectAction(objA)(server.Client()); err != nil { + t.Fatal(err) + } + if err := createObjectAction(objB)(server.Client()); err != nil { + t.Fatal(err) + } + + if len(manager.events) != 2 { + t.Fatalf("expected 2 events, got %d", len(manager.events)) + } + + buckets := map[string]bool{} + for _, ev := range manager.events { + buckets[ev.obj.BucketName] = true + } + if !buckets["bucket-a"] { + t.Error("expected an event for bucket-a") + } + if !buckets["bucket-b"] { + t.Error("expected an event for bucket-b") + } + }) + + t.Run("empty managers list is no-op", func(t *testing.T) { + t.Parallel() + server, err := NewServerWithOptions(Options{}) + if err != nil { + t.Fatal(err) + } + defer server.Stop() + + server.eventManager = notification.NewMultiEventManager(nil) + + if err := server.backend.CreateBucket("bucket-a", backend.BucketAttrs{}); err != nil { + t.Fatal(err) + } + // Must not panic. + if err := createObjectAction(objA)(server.Client()); err != nil { + t.Fatal(err) + } + }) +} + func TestServerBatchRequest(t *testing.T) { objects := []Object{ { diff --git a/internal/config/config.go b/internal/config/config.go index 2634905dd3..5b4cff63a9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -48,6 +48,8 @@ type Config struct { backend string fsRoot string event EventConfig + events eventConfigFlag + parsedEvents []notification.EventManagerOptions bucketLocation string LogLevel slog.Level } @@ -60,6 +62,109 @@ type EventConfig struct { list []string } +// eventConfigFlag is a flag.Value that accumulates repeated -event.config +// values. Each invocation of Set appends the raw string; validation and +// conversion into EventManagerOptions happens in Config.validate. +type eventConfigFlag []string + +func (f *eventConfigFlag) String() string { + if f == nil { + return "" + } + return strings.Join(*f, ", ") +} + +func (f *eventConfigFlag) Set(value string) error { + *f = append(*f, value) + return nil +} + +// parseEventConfig parses a single -event.config value into an +// EventManagerOptions. The expected format is: +// +// bucket=;project=;topic=[;events=,][;prefix=] +// +// bucket, project, and topic are required. events defaults to finalize if not +// specified. prefix is optional. +func parseEventConfig(raw string) (notification.EventManagerOptions, error) { + var opts notification.EventManagerOptions + var hasBucket, hasProject, hasTopic, hasEvents bool + + for _, field := range strings.Split(raw, ";") { + parts := strings.SplitN(field, "=", 2) + if len(parts) != 2 { + return opts, fmt.Errorf("invalid event config field %q: expected key=value format", field) + } + key := strings.TrimSpace(parts[0]) + val := strings.TrimSpace(parts[1]) + + switch key { + case "bucket": + opts.Bucket = val + hasBucket = true + case "project": + opts.ProjectID = val + hasProject = true + case "topic": + opts.TopicName = val + hasTopic = true + case "events": + notifyOn, err := eventListToNotifyOn(strings.Split(val, ",")) + if err != nil { + return opts, err + } + opts.NotifyOn = notifyOn + hasEvents = true + case "prefix": + opts.ObjectPrefix = val + default: + return opts, fmt.Errorf("unknown event config key %q", key) + } + + if val == "" { + return opts, fmt.Errorf("event config key %q must not be empty", key) + } + } + + if !hasBucket { + return opts, fmt.Errorf("event config missing required field \"bucket\"") + } + if !hasProject { + return opts, fmt.Errorf("event config missing required field \"project\"") + } + if !hasTopic { + return opts, fmt.Errorf("event config missing required field \"topic\"") + } + if !hasEvents { + opts.NotifyOn = notification.EventNotificationOptions{Finalize: true} + } + return opts, nil +} + +// eventListToNotifyOn validates a list of event name strings and maps them to an +// EventNotificationOptions struct. +func eventListToNotifyOn(events []string) (notification.EventNotificationOptions, error) { + var notifyOn notification.EventNotificationOptions + if len(events) == 0 { + return notifyOn, fmt.Errorf("events list must not be empty") + } + for _, e := range events { + switch strings.TrimSpace(e) { + case eventFinalize: + notifyOn.Finalize = true + case eventDelete: + notifyOn.Delete = true + case eventMetadataUpdate: + notifyOn.MetadataUpdate = true + case eventArchive: + notifyOn.Archive = true + default: + return notifyOn, fmt.Errorf("%q is an invalid event", strings.TrimSpace(e)) + } + } + return notifyOn, nil +} + // Load parses the given arguments list and return a config object (and/or an // error in case of failures). func Load(args []string) (Config, error) { @@ -84,6 +189,7 @@ func Load(args []string) (Config, error) { fs.StringVar(&cfg.event.bucket, "event.bucket", "", "if not empty, only objects in this bucket will generate trigger events") fs.StringVar(&cfg.event.prefix, "event.object-prefix", "", "if not empty, only objects having this prefix will generate trigger events") fs.StringVar(&eventList, "event.list", eventFinalize, "comma separated list of events to publish on cloud function URl. Options are: finalize, delete, and metadataUpdate") + fs.Var(&cfg.events, "event.config", "notification configuration in the format: bucket=;project=;topic=[;events=,][;prefix=]. Can be specified multiple times for multiple configurations. events defaults to finalize. Supported events: finalize, delete, metadataUpdate, archive") fs.StringVar(&cfg.bucketLocation, "location", "US-CENTRAL1", "location for buckets") fs.StringVar(&cfg.CertificateLocation, "cert-location", "", "location for server certificate") fs.StringVar(&cfg.PrivateKeyLocation, "private-key-location", "", "location for private key") @@ -160,7 +266,19 @@ func (c *Config) validate() error { return fmt.Errorf("port-http %d is too high, maximum value is %d", c.PortHTTP, math.MaxUint16) } - return c.event.validate() + if err := c.event.validate(); err != nil { + return err + } + + for _, raw := range c.events { + opts, err := parseEventConfig(raw) + if err != nil { + return err + } + c.parsedEvents = append(c.parsedEvents, opts) + } + + return nil } func (c *EventConfig) validate() error { @@ -230,6 +348,7 @@ func (c *Config) ToFakeGcsOptions(logger *slog.Logger, scheme string) fakestorag AllowedCORSHeaders: c.allowedCORSHeaders, Writer: &slogWriter{logger: logger, level: slog.LevelInfo}, EventOptions: eventOptions, + EventConfigs: c.parsedEvents, BucketsLocation: c.bucketLocation, CertificateLocation: c.CertificateLocation, PrivateKeyLocation: c.PrivateKeyLocation, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 54b2d44b61..3b826bda5c 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -274,6 +274,74 @@ func TestLoadConfig(t *testing.T) { LogLevel: slog.LevelInfo, }, }, + { + name: "single event.config", + args: []string{ + "-event.config", "bucket=my-bucket;project=test-project;topic=gcs-events;events=finalize;prefix=uploads/", + }, + expectedConfig: Config{ + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:4443", + Host: "0.0.0.0", + Port: 4443, + Scheme: "https", + event: EventConfig{list: []string{"finalize"}}, + events: eventConfigFlag{"bucket=my-bucket;project=test-project;topic=gcs-events;events=finalize;prefix=uploads/"}, + bucketLocation: "US-CENTRAL1", + LogLevel: slog.LevelInfo, + parsedEvents: []notification.EventManagerOptions{ + { + ProjectID: "test-project", + TopicName: "gcs-events", + Bucket: "my-bucket", + ObjectPrefix: "uploads/", + NotifyOn: notification.EventNotificationOptions{Finalize: true}, + }, + }, + }, + }, + { + name: "multiple event.config", + args: []string{ + "-event.config", "bucket=bucket-a;project=proj1;topic=topic-a;events=finalize,delete", + "-event.config", "bucket=bucket-b;project=proj2;topic=topic-b;events=metadataUpdate", + }, + expectedConfig: Config{ + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:4443", + Host: "0.0.0.0", + Port: 4443, + Scheme: "https", + event: EventConfig{list: []string{"finalize"}}, + bucketLocation: "US-CENTRAL1", + LogLevel: slog.LevelInfo, + events: eventConfigFlag{ + "bucket=bucket-a;project=proj1;topic=topic-a;events=finalize,delete", + "bucket=bucket-b;project=proj2;topic=topic-b;events=metadataUpdate", + }, + parsedEvents: []notification.EventManagerOptions{ + { + ProjectID: "proj1", + TopicName: "topic-a", + Bucket: "bucket-a", + NotifyOn: notification.EventNotificationOptions{ + Finalize: true, + Delete: true, + }, + }, + { + ProjectID: "proj2", + TopicName: "topic-b", + Bucket: "bucket-b", + NotifyOn: notification.EventNotificationOptions{MetadataUpdate: true}, + }, + }, + }, + }, { name: "invalid port value type", args: []string{"-port", "not-a-number"}, @@ -324,6 +392,71 @@ func TestLoadConfig(t *testing.T) { args: []string{"-event.list", "invalid,stuff", "-event.pubsub-topic", "gcs-events", "-event.pubsub-project-id", "test-project"}, expectErr: true, }, + { + name: "event.config missing topic", + args: []string{"-event.config", "bucket=my-bucket;project=test-project"}, + expectErr: true, + }, + { + name: "event.config empty topic", + args: []string{"-event.config", "bucket=my-bucket;project=test-project;topic=;"}, + expectErr: true, + }, + { + name: "event.config missing project", + args: []string{"-event.config", "bucket=my-bucket;topic=gcs-events"}, + expectErr: true, + }, + { + name: "event.config empty project", + args: []string{"-event.config", "bucket=my-bucket;project=;topic=gcs-events"}, + expectErr: true, + }, + { + name: "event.config invalid event name", + args: []string{"-event.config", "bucket=my-bucket;project=test-project;topic=gcs-events;events=bogus"}, + expectErr: true, + }, + { + name: "event.config unknown key", + args: []string{"-event.config", "bucket=my-bucket;project=test-project;topic=gcs-events;unknown=x"}, + expectErr: true, + }, + { + name: "event.config missing bucket", + args: []string{"-event.config", "project=test-project;topic=gcs-events"}, + expectErr: true, + }, + { + name: "event.config empty bucket", + args: []string{"-event.config", "bucket=;project=test-project;topic=gcs-events"}, + expectErr: true, + }, + { + name: "event.config defaults events to finalize", + args: []string{"-event.config", "bucket=my-bucket;project=test-project;topic=gcs-events"}, + expectedConfig: Config{ + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:4443", + Host: "0.0.0.0", + Port: 4443, + Scheme: "https", + event: EventConfig{list: []string{"finalize"}}, + events: eventConfigFlag{"bucket=my-bucket;project=test-project;topic=gcs-events"}, + bucketLocation: "US-CENTRAL1", + LogLevel: slog.LevelInfo, + parsedEvents: []notification.EventManagerOptions{ + { + ProjectID: "test-project", + TopicName: "gcs-events", + Bucket: "my-bucket", + NotifyOn: notification.EventNotificationOptions{Finalize: true}, + }, + }, + }, + }, { name: "invalid log level", args: []string{"-log-level", "non-existent-level"}, @@ -437,6 +570,54 @@ func TestToFakeGcsOptions(t *testing.T) { NoListener: true, }, }, + { + "event configs via parsedEvents", + Config{ + backend: "memory", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:4443", + Host: "0.0.0.0", + Port: 4443, + Scheme: "https", + parsedEvents: []notification.EventManagerOptions{ + { + ProjectID: "proj-a", + TopicName: "topic-a", + Bucket: "bucket-a", + NotifyOn: notification.EventNotificationOptions{Finalize: true}, + }, + { + ProjectID: "proj-b", + TopicName: "topic-b", + Bucket: "bucket-b", + NotifyOn: notification.EventNotificationOptions{Delete: true}, + }, + }, + }, + fakestorage.Options{ + StorageRoot: "", + PublicHost: "storage.googleapis.com", + ExternalURL: "https://0.0.0.0:4443", + Host: "0.0.0.0", + Port: 4443, + Scheme: "https", + EventConfigs: []notification.EventManagerOptions{ + { + ProjectID: "proj-a", + TopicName: "topic-a", + Bucket: "bucket-a", + NotifyOn: notification.EventNotificationOptions{Finalize: true}, + }, + { + ProjectID: "proj-b", + TopicName: "topic-b", + Bucket: "bucket-b", + NotifyOn: notification.EventNotificationOptions{Delete: true}, + }, + }, + NoListener: true, + }, + }, } for _, test := range tests { diff --git a/internal/notification/event.go b/internal/notification/event.go index ac68856e56..6041b25b51 100644 --- a/internal/notification/event.go +++ b/internal/notification/event.go @@ -57,6 +57,24 @@ type EventManager interface { Trigger(o *backend.StreamingObject, eventType EventType, extraEventAttr map[string]string) } +// MultiEventManager fans out Trigger calls to multiple underlying EventManagers. +// A nil or empty managers slice is a no-op. +type MultiEventManager struct { + managers []EventManager +} + +// NewMultiEventManager creates a new MultiEventManager that dispatches events +// to all provided managers. +func NewMultiEventManager(managers []EventManager) *MultiEventManager { + return &MultiEventManager{managers: managers} +} + +func (m *MultiEventManager) Trigger(o *backend.StreamingObject, eventType EventType, extraEventAttr map[string]string) { + for _, mgr := range m.managers { + mgr.Trigger(o, eventType, extraEventAttr) + } +} + // PubsubEventManager checks if an event should be published. type PubsubEventManager struct { // publishSynchronously is a flag that if true, events will be published diff --git a/internal/notification/event_test.go b/internal/notification/event_test.go index 7fb22ca6a9..fac6099fcc 100644 --- a/internal/notification/event_test.go +++ b/internal/notification/event_test.go @@ -193,3 +193,126 @@ func TestPubsubEventManager_Trigger(t *testing.T) { }) } } + +func TestMultiEventManager_Trigger(t *testing.T) { + t.Parallel() + content := []byte("something") + newObject := func(bucket string) backend.StreamingObject { + obj := backend.Object{ + ObjectAttrs: backend.ObjectAttrs{ + BucketName: bucket, + Name: "files/obj.txt", + Size: int64(len(content)), + }, + Content: content, + } + return obj.StreamingObject() + } + + tests := []struct { + name string + managerA *PubsubEventManager + managerB *PubsubEventManager + triggerBucket string + expectAReceived bool + expectBReceived bool + }{ + { + name: "both managers no bucket filter", + managerA: &PubsubEventManager{ + notifyOn: EventNotificationOptions{Finalize: true}, + publishSynchronously: true, + }, + managerB: &PubsubEventManager{ + notifyOn: EventNotificationOptions{Finalize: true}, + publishSynchronously: true, + }, + triggerBucket: "bucket-x", + expectAReceived: true, + expectBReceived: true, + }, + { + name: "only manager A matches bucket", + managerA: &PubsubEventManager{ + notifyOn: EventNotificationOptions{Finalize: true}, + bucket: "bucket-x", + publishSynchronously: true, + }, + managerB: &PubsubEventManager{ + notifyOn: EventNotificationOptions{Finalize: true}, + bucket: "bucket-y", + publishSynchronously: true, + }, + triggerBucket: "bucket-x", + expectAReceived: true, + expectBReceived: false, + }, + { + name: "only manager B matches bucket", + managerA: &PubsubEventManager{ + notifyOn: EventNotificationOptions{Finalize: true}, + bucket: "bucket-x", + publishSynchronously: true, + }, + managerB: &PubsubEventManager{ + notifyOn: EventNotificationOptions{Finalize: true}, + bucket: "bucket-y", + publishSynchronously: true, + }, + triggerBucket: "bucket-y", + expectAReceived: false, + expectBReceived: true, + }, + { + name: "neither manager matches bucket", + managerA: &PubsubEventManager{ + notifyOn: EventNotificationOptions{Finalize: true}, + bucket: "bucket-x", + publishSynchronously: true, + }, + managerB: &PubsubEventManager{ + notifyOn: EventNotificationOptions{Finalize: true}, + bucket: "bucket-y", + publishSynchronously: true, + }, + triggerBucket: "bucket-z", + expectAReceived: false, + expectBReceived: false, + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + pubA := &mockPublisher{} + pubB := &mockPublisher{} + test.managerA.publisher = pubA + test.managerB.publisher = pubB + + multi := NewMultiEventManager([]EventManager{test.managerA, test.managerB}) + obj := newObject(test.triggerBucket) + multi.Trigger(&obj, EventFinalize, nil) + + if test.expectAReceived && pubA.lastMessage == nil { + t.Error("manager A: expected to receive event, got nil") + } + if !test.expectAReceived && pubA.lastMessage != nil { + t.Errorf("manager A: expected no event, got %v", pubA.lastMessage) + } + if test.expectBReceived && pubB.lastMessage == nil { + t.Error("manager B: expected to receive event, got nil") + } + if !test.expectBReceived && pubB.lastMessage != nil { + t.Errorf("manager B: expected no event, got %v", pubB.lastMessage) + } + }) + } + + t.Run("empty managers slice", func(t *testing.T) { + t.Parallel() + multi := NewMultiEventManager(nil) + obj := newObject("any-bucket") + multi.Trigger(&obj, EventFinalize, nil) // must not panic + }) +}