From f55316cb8f673d1dd407ad266c70e7028c2cbebf Mon Sep 17 00:00:00 2001 From: Catalina Radu Date: Fri, 6 Sep 2024 13:05:36 +0300 Subject: [PATCH 1/8] [wip] Add initial logic for the client --- cmd/sync/client/client.go | 96 ++++++++++++++++++++++++++ pkg/sqs/synconsumer.go | 141 ++++++++++++++++++++++++++++++++++++++ pkg/sync/client/k8s.go | 47 +++++++++++++ pkg/sync/client/utils.go | 107 +++++++++++++++++++++++++++++ 4 files changed, 391 insertions(+) create mode 100644 cmd/sync/client/client.go create mode 100644 pkg/sqs/synconsumer.go create mode 100644 pkg/sync/client/k8s.go create mode 100644 pkg/sync/client/utils.go diff --git a/cmd/sync/client/client.go b/cmd/sync/client/client.go new file mode 100644 index 00000000..4bd06a26 --- /dev/null +++ b/cmd/sync/client/client.go @@ -0,0 +1,96 @@ +package main + +import ( + "fmt" + "os" + + "github.com/adobe/cluster-registry/pkg/config" + "github.com/adobe/cluster-registry/pkg/sqs" + client "github.com/adobe/cluster-registry/pkg/sync/client" + awssqs "github.com/aws/aws-sdk-go/service/sqs" + "github.com/davecgh/go-spew/spew" + "github.com/sirupsen/logrus" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var ( + logLevel, logFormat string + appConfig *config.AppConfig + namespace string + //clusterName string + //cfgFile string +) + +func InitCLI() *cobra.Command { + + var rootCmd = &cobra.Command{ + Use: "cluster-registry-client-sync", + Short: "Cluster Registry Sync Client is a service that keep in sync the cluster CRD", + Long: "\nCluster Registry Sync Client is a service that creates or updates the cluster CRD based on the messages received from the Cluster Registry Sync manager", + PersistentPreRun: loadAppConfig, + Run: run, + } + + initFlags(rootCmd) + + return rootCmd +} + +func initFlags(rootCmd *cobra.Command) { + + rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", logrus.DebugLevel.String(), "The verbosity level of the logs, can be [panic|fatal|error|warn|info|debug|trace]") + rootCmd.PersistentFlags().StringVar(&logFormat, "log-format", "text", "The output format of the logs, can be [text|json]") + //rootCmd.PersistentFlags().StringVar(&cfgFile, "config-file", "", "The path to the configuration file") + rootCmd.PersistentFlags().StringVar(&namespace, "namespace", "cluster-registry", "The namespace where cluster-registry-sync-client will run.") +} + +func loadAppConfig(cmd *cobra.Command, args []string) { + + client.InitLogger(logLevel, logFormat) + + log.Info("Starting the Cluster Registry Sync Client") + + log.Info("Loading the configuration") + appConfig, err := config.LoadSyncClientConfig() + if err != nil { + log.Error("Cannot load the cluster-registry-sync-client configuration:", err.Error()) + os.Exit(1) + } + log.Info("Config loaded successfully") + log.Info("Cluster (custom resource) to be checked:", appConfig.ClusterName) +} + +func run(cmd *cobra.Command, args []string) { + + log.Info("Cluster Registry Sync Client is running") + + // Consume the messages from the queue using a sync consumer + sqsInstance := sqs.NewSQS(appConfig) + log.Info("Starting the SQS sync consumer") + + handler := func(m *awssqs.Message) error { + spew.Dump(m) + // TODO + return nil + } + syncConsumer := sqs.NewSyncConsumer(sqsInstance, appConfig, handler) + go syncConsumer.Consume() + + // Block the thread + select {} +} + +func main() { + + rootCmd := InitCLI() + + // Execute the CLI application + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } + + //TODO + +} diff --git a/pkg/sqs/synconsumer.go b/pkg/sqs/synconsumer.go new file mode 100644 index 00000000..9e955be3 --- /dev/null +++ b/pkg/sqs/synconsumer.go @@ -0,0 +1,141 @@ +/* +Copyright 2024 Adobe. All rights reserved. +This file is licensed to you under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. You may obtain a copy +of the License at http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under +the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS +OF ANY KIND, either express or implied. See the License for the specific language +governing permissions and limitations under the License. +*/ + +package sqs + +import ( + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/sqs" + "github.com/aws/aws-sdk-go/service/sqs/sqsiface" + "github.com/labstack/gommon/log" + + "github.com/adobe/cluster-registry/pkg/config" + monitoring "github.com/adobe/cluster-registry/pkg/monitoring/apiserver" +) + +// consumer struct +type synconsumer struct { + sqs sqsiface.SQSAPI + queueURL string + workerPool int + maxMessages int64 + pollWaitSeconds int64 + retrySeconds int + messageHandler func(*sqs.Message) error +} + +// NewSyncConsumer - creates a new SQS message queue consumer +// used by the sync consumer service +// TODO: add metrics later +func NewSyncConsumer(sqsSvc sqsiface.SQSAPI, appConfig *config.AppConfig, h func(*sqs.Message) error) Consumer { + + urlResult, err := sqsSvc.GetQueueUrl(&sqs.GetQueueUrlInput{ + QueueName: &appConfig.SqsQueueName, + }) + if err != nil { + log.Fatal(err.Error()) + } + + return &synconsumer{ + sqs: sqsSvc, + queueURL: *urlResult.QueueUrl, + workerPool: 10, + maxMessages: 1, + pollWaitSeconds: 1, + retrySeconds: 5, + messageHandler: h, + } +} + +// Status verifies the status/connectivity of the sqs service +func (c *synconsumer) Status(appConfig *config.AppConfig, m monitoring.MetricsI) error { + _, err := c.sqs.GetQueueUrl(&sqs.GetQueueUrlInput{ + QueueName: &appConfig.SqsQueueName, + }) + + if err != nil { + log.Error(err.Error()) + } + + return err +} + +// Consume - long pooling +func (c *synconsumer) Consume() { + var wg sync.WaitGroup + + for w := 1; w <= c.workerPool; w++ { + wg.Add(1) + go func(w int) { + defer wg.Done() + c.worker(w) + }(w) + } + wg.Wait() +} + +func (c *synconsumer) worker(id int) { + for { + output, err := c.sqs.ReceiveMessage((&sqs.ReceiveMessageInput{ + QueueUrl: &c.queueURL, + AttributeNames: aws.StringSlice([]string{ + "ClusterName", "SentTimestamp", + }), + MaxNumberOfMessages: aws.Int64(c.maxMessages), + WaitTimeSeconds: aws.Int64(c.pollWaitSeconds), + })) + + if err != nil { + log.Error(err.Error()) + log.Info("Retrying in", c.retrySeconds, " seconds") + time.Sleep(time.Duration(c.retrySeconds) * time.Second) + continue + } + + for _, m := range output.Messages { + log.Debug("Messsage ID: ", *m.MessageId) + log.Debug("Message Body: ", *m.Body) + + err := c.processMessage(m) + + if err != nil { + log.Error(err.Error()) + continue + } + err = c.delete(m) + if err != nil { + log.Error(err.Error()) + } + } + } +} + +// processMessage - process the recieved message +func (c *synconsumer) processMessage(m *sqs.Message) error { + + err := c.messageHandler(m) + return err +} + +func (c *synconsumer) delete(m *sqs.Message) error { + + _, err := c.sqs.DeleteMessage( + &sqs.DeleteMessageInput{QueueUrl: &c.queueURL, ReceiptHandle: m.ReceiptHandle}) + + if err != nil { + log.Error(err.Error()) + } + return err +} diff --git a/pkg/sync/client/k8s.go b/pkg/sync/client/k8s.go new file mode 100644 index 00000000..097ffce7 --- /dev/null +++ b/pkg/sync/client/k8s.go @@ -0,0 +1,47 @@ +/* +Copyright 2024 Adobe. All rights reserved. +This file is licensed to you under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. You may obtain a copy +of the License at http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under +the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS +OF ANY KIND, either express or implied. See the License for the specific language +governing permissions and limitations under the License. +*/ + +package client + +import ( + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +func getClientSet() (*kubernetes.Clientset, error) { + cfg, err := config.GetConfig() + if err != nil { + return nil, err + } + + clientSet, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, err + } + + return clientSet, nil +} + +func getDynamicClientSet() (*dynamic.DynamicClient, error) { + cfg, err := config.GetConfig() + if err != nil { + return nil, err + } + + dnc, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, err + } + + return dnc, nil +} diff --git a/pkg/sync/client/utils.go b/pkg/sync/client/utils.go new file mode 100644 index 00000000..d923b7e3 --- /dev/null +++ b/pkg/sync/client/utils.go @@ -0,0 +1,107 @@ +/* +Copyright 2024 Adobe. All rights reserved. +This file is licensed to you under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. You may obtain a copy +of the License at http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under +the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS +OF ANY KIND, either express or implied. See the License for the specific language +governing permissions and limitations under the License. +*/ + +package client + +import ( + "encoding/json" + "io" + "log" + "os" + "strings" + + registryv1 "github.com/adobe/cluster-registry/pkg/api/registry/v1" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +func ReadFile(patchFilePath string) ([]byte, error) { + patchFile, err := os.Open(patchFilePath) + if err != nil { + log.Fatalf("Error opening patch YAML file: %v", err) + } + defer patchFile.Close() + + patchData, err := io.ReadAll(patchFile) + if err != nil { + log.Fatalf("Error reading patch YAML file: %v", err) + } + return patchData, nil +} + +func UnmarshalYaml(data []byte, cluster *[]registryv1.Cluster) error { + err := yaml.Unmarshal(data, cluster) + if err != nil { + log.Panicf("Error while trying to unmarshal yaml data: %v", err.Error()) + } + + return err +} + +func UnmarshalJSON(data []byte, cluster *[]registryv1.Cluster) error { + err := json.Unmarshal(data, cluster) + if err != nil { + log.Panicf("Error while trying to unmarshal json data: %v", err.Error()) + } + + return err +} + +func MarshalJson(patch map[string]interface{}) ([]byte, error) { + jsonData, err := json.Marshal(patch) + if err != nil { + log.Panicf("Error while trying to marshal json data: %v", err.Error()) + } + + return jsonData, err +} + +// toUnstructured converts a Cluster struct to an unstructured.Unstructured object +func toUnstructured(obj interface{}) (*unstructured.Unstructured, error) { + data, err := json.Marshal(obj) + if err != nil { + return nil, err + } + u := &unstructured.Unstructured{} + if err := u.UnmarshalJSON(data); err != nil { + return nil, err + } + return u, nil +} + +// TODO; check if there is an utils func - see if no need +// unstructuredToJSON converts an unstructured.Unstructured object to a JSON string +func unstructuredToJSON(obj *unstructured.Unstructured) ([]byte, error) { + return obj.MarshalJSON() +} + +func InitLogger(logLevel string, logFormat string) { + + level, err := logrus.ParseLevel(logLevel) + if err != nil { + level = logrus.DebugLevel + } + logrus.SetLevel(level) + + logFormat = strings.ToLower(logFormat) + if logFormat == "text" { + logrus.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + ForceColors: true, + }) + } else { + logrus.SetFormatter(&logrus.JSONFormatter{ + TimestampFormat: "2006-01-02 15:04:05", + }) + } +} From ff10fbb38dfce8cb59f6d198192f7dd32d221965 Mon Sep 17 00:00:00 2001 From: aalexand Date: Wed, 18 Dec 2024 12:38:05 +0200 Subject: [PATCH 2/8] Add missing headers --- cmd/sync/client/client.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cmd/sync/client/client.go b/cmd/sync/client/client.go index 4bd06a26..18575cb2 100644 --- a/cmd/sync/client/client.go +++ b/cmd/sync/client/client.go @@ -1,3 +1,15 @@ +/* +Copyright 2024 Adobe. All rights reserved. +This file is licensed to you under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. You may obtain a copy +of the License at http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under +the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS +OF ANY KIND, either express or implied. See the License for the specific language +governing permissions and limitations under the License. +*/ + package main import ( From f0d5cffea6efcd6dccd66aa89f399c3fe57b0700 Mon Sep 17 00:00:00 2001 From: aalexand Date: Thu, 16 Jan 2025 10:36:02 +0200 Subject: [PATCH 3/8] Integrate sqs client changes --- cmd/sync/client/client.go | 115 ++++++++++++++++++------------- go.mod | 9 +-- go.sum | 130 ++++------------------------------- pkg/config/config.go | 23 +++++++ pkg/sqs/synconsumer.go | 141 -------------------------------------- pkg/sync/event/handler.go | 5 +- 6 files changed, 111 insertions(+), 312 deletions(-) delete mode 100644 pkg/sqs/synconsumer.go diff --git a/cmd/sync/client/client.go b/cmd/sync/client/client.go index 18575cb2..f86d4b5a 100644 --- a/cmd/sync/client/client.go +++ b/cmd/sync/client/client.go @@ -13,96 +13,113 @@ governing permissions and limitations under the License. package main import ( - "fmt" + "github.com/adobe/cluster-registry/pkg/sync/event" "os" "github.com/adobe/cluster-registry/pkg/config" "github.com/adobe/cluster-registry/pkg/sqs" - client "github.com/adobe/cluster-registry/pkg/sync/client" + "github.com/adobe/cluster-registry/pkg/sync/client" awssqs "github.com/aws/aws-sdk-go/service/sqs" - "github.com/davecgh/go-spew/spew" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) var ( + rootCmd = &cobra.Command{ + Use: "cluster-registry-sync-client", + Short: "Cluster Registry Sync Client is a service that keep the Cluster CRD in sync", + Long: "Cluster Registry Sync Client is a service that creates or updates the cluster CRD based on the messages received from the Cluster Registry Sync manager", + PersistentPreRun: loadAppConfig, + Run: run, + } + logLevel, logFormat string appConfig *config.AppConfig namespace string - //clusterName string - //cfgFile string + cfgFile string ) -func InitCLI() *cobra.Command { - - var rootCmd = &cobra.Command{ - Use: "cluster-registry-client-sync", - Short: "Cluster Registry Sync Client is a service that keep in sync the cluster CRD", - Long: "\nCluster Registry Sync Client is a service that creates or updates the cluster CRD based on the messages received from the Cluster Registry Sync manager", - PersistentPreRun: loadAppConfig, - Run: run, +func Execute() { + err := rootCmd.Execute() + if err != nil { + log.Fatalln(err.Error()) } - - initFlags(rootCmd) - - return rootCmd } -func initFlags(rootCmd *cobra.Command) { - +func init() { + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "The path to the yaml configuration file") rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", logrus.DebugLevel.String(), "The verbosity level of the logs, can be [panic|fatal|error|warn|info|debug|trace]") rootCmd.PersistentFlags().StringVar(&logFormat, "log-format", "text", "The output format of the logs, can be [text|json]") - //rootCmd.PersistentFlags().StringVar(&cfgFile, "config-file", "", "The path to the configuration file") rootCmd.PersistentFlags().StringVar(&namespace, "namespace", "cluster-registry", "The namespace where cluster-registry-sync-client will run.") + err := rootCmd.MarkPersistentFlagRequired("config") + if err != nil { + log.Fatalln("No config flag configured") + } } func loadAppConfig(cmd *cobra.Command, args []string) { - client.InitLogger(logLevel, logFormat) - log.Info("Starting the Cluster Registry Sync Client") - log.Info("Loading the configuration") - appConfig, err := config.LoadSyncClientConfig() + + var err error + appConfig, err = config.LoadSyncClientConfig() if err != nil { log.Error("Cannot load the cluster-registry-sync-client configuration:", err.Error()) os.Exit(1) } + log.Info("Config loaded successfully") - log.Info("Cluster (custom resource) to be checked:", appConfig.ClusterName) } func run(cmd *cobra.Command, args []string) { - log.Info("Cluster Registry Sync Client is running") - // Consume the messages from the queue using a sync consumer - sqsInstance := sqs.NewSQS(appConfig) - log.Info("Starting the SQS sync consumer") - - handler := func(m *awssqs.Message) error { - spew.Dump(m) - // TODO - return nil + q, err := sqs.NewSQS(sqs.Config{ + AWSRegion: appConfig.SqsAwsRegion, + Endpoint: appConfig.SqsEndpoint, + QueueName: appConfig.SqsQueueName, + BatchSize: 10, + VisibilityTimeout: 120, + WaitSeconds: 5, + RunInterval: 20, + RunOnce: false, + MaxHandlers: 10, + BusyTimeout: 30, + }) + if err != nil { + log.Panicf("Error while trying to create SQS client: %v", err.Error()) } - syncConsumer := sqs.NewSyncConsumer(sqsInstance, appConfig, handler) - go syncConsumer.Consume() - // Block the thread - select {} -} + handler := event.NewPartialClusterUpdateHandler() + q.RegisterHandler(func(msg *awssqs.Message) { + log.Debugf("Received message: %s", *msg.MessageId) + e, err := sqs.NewEvent(msg) + if err != nil { + log.Errorf("Cannot create event from message: %s", err.Error()) + return + } + if e.Type != sqs.PartialClusterUpdateEvent { + log.Infof("Not interested in event of type %s, skipping", e.Type) + return + } + log.Debugf("Handling event for message: %s", *msg.MessageId) + if err = handler.Handle(e); err != nil { + log.Errorf("Failed to handle event: %s", err.Error()) + return + } + if err = q.Delete(msg); err != nil { + log.Errorf("Failed to delete message: %s", err.Error()) + return + } + }) -func main() { - - rootCmd := InitCLI() - - // Execute the CLI application - if err := rootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } + log.Info("Starting the Cluster Registry Sync Client") - //TODO + q.Poll() +} +func main() { + Execute() } diff --git a/go.mod b/go.mod index c1c0d6ca..ba3844e5 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,8 @@ require ( github.com/onsi/gomega v1.36.1 github.com/prometheus/client_golang v1.20.5 github.com/redis/go-redis/v9 v9.7.0 + github.com/sirupsen/logrus v1.9.3 + github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.10.0 github.com/swaggo/echo-swagger v1.4.1 github.com/swaggo/swag v1.16.4 @@ -85,15 +87,13 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.9 // indirect @@ -116,7 +116,6 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.8 // indirect - github.com/onsi/ginkgo/v2 v2.21.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -127,7 +126,6 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/shirou/gopsutil/v3 v3.23.12 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/swaggo/files/v2 v2.0.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect @@ -151,7 +149,6 @@ require ( golang.org/x/time v0.8.0 // indirect golang.org/x/tools v0.28.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/grpc v1.66.2 // indirect google.golang.org/protobuf v1.35.1 // indirect diff --git a/go.sum b/go.sum index 89b639e0..92271f37 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -55,10 +53,9 @@ github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpS github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/OtI= github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0= -github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= -github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -115,7 +112,6 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= @@ -127,8 +123,6 @@ github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -144,15 +138,11 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -176,8 +166,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= -github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= @@ -186,8 +176,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737 github.com/gusaul/go-dynamock v0.0.0-20210107061312-3e989056e1e6 h1:KxdjsEW5PDmO6zgXUsuokWRlzvYXmb04jV37O8EzuKI= github.com/gusaul/go-dynamock v0.0.0-20210107061312-3e989056e1e6/go.mod h1:EDSgJH1MyCc1x6BzVGei5Bdat3FFZnKy/p0vysyDMCA= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= @@ -214,8 +204,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0= -github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM= github.com/labstack/echo/v4 v4.13.2 h1:9aAt4hstpH54qIcqkuUXRLTf+v7yOTfMPWzDtuqLmtA= github.com/labstack/echo/v4 v4.13.2/go.mod h1:uc9gDtHB8UWt3FfbYx0HyxcCuvR4YuPYOxF/1QjoV/c= github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= @@ -268,13 +256,10 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= -github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= -github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -288,10 +273,6 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4= -github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= -github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= @@ -300,12 +281,11 @@ github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= -github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -314,11 +294,15 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -327,20 +311,14 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/swaggo/echo-swagger v1.4.1 h1:Yf0uPaJWp1uRtDloZALyLnvdBeoEL5Kc7DtnjzO/TUk= github.com/swaggo/echo-swagger v1.4.1/go.mod h1:C8bSi+9yH2FLZsnhqMZLIZddpUxZdBYuNHbtaS1Hljc= github.com/swaggo/files/v2 v2.0.0 h1:hmAt8Dkynw7Ssz46F6pn8ok6YmGZqHSVLZ+HQM7i0kw= github.com/swaggo/files/v2 v2.0.0/go.mod h1:24kk2Y9NYEJ5lHuCra6iVwkMjIekMCaFq/0JQj66kyM= -github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= -github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A= github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg= -github.com/testcontainers/testcontainers-go v0.33.0 h1:zJS9PfXYT5O0ZFXM2xxXfk4J5UMw/kRiISng037Gxdw= -github.com/testcontainers/testcontainers-go v0.33.0/go.mod h1:W80YpTa8D5C3Yy16icheD01UTDu+LmXIA2Keo+jWtT8= github.com/testcontainers/testcontainers-go v0.34.0 h1:5fbgF0vIN5u+nD3IWabQwRybuB4GY8G2HHgCkbMzMHo= github.com/testcontainers/testcontainers-go v0.34.0/go.mod h1:6P/kMkQe8yqPHfPWNulFGdFHTD8HB2vLq/231xY2iPQ= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= @@ -379,8 +357,6 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -392,20 +368,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= -golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 h1:1wqE9dj9NpSm04INVsJhhEUzhuDVjbcyKH91sVyPATw= -golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -413,8 +377,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -432,10 +396,6 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= @@ -447,8 +407,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -479,10 +437,6 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -491,10 +445,6 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= -golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= -golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -504,14 +454,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= -golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -522,10 +466,6 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= -golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -534,8 +474,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f h1:b1Ln/PG8orm0SsBbHZWke8dDp2lrCD4jSmfglFpTZbk= -google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:AHT0dDg3SoMOgZGnZk29b5xTbPHMoEC8qthmBLJCpys= +google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13 h1:vlzZttNJGVqTsRFU9AmdnrcO1Znh8Ew9kCD//yjigk0= +google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= @@ -547,8 +487,6 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -579,64 +517,26 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= -k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= -k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= -k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU= -k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI= -k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0= -k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk= k8s.io/api v0.32.0 h1:OL9JpbvAU5ny9ga2fb24X8H6xQlVp+aJMFlgtQjR9CE= k8s.io/api v0.32.0/go.mod h1:4LEwHZEf6Q/cG96F3dqR965sYOfmPM7rq81BLgsE0p0= k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= -k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= -k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= -k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw= -k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= k8s.io/apimachinery v0.32.0 h1:cFSE7N3rmEEtv4ei5X6DaJPHHX0C+upp+v5lVPiEwpg= k8s.io/apimachinery v0.32.0/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= -k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= -k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= -k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= -k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc= -k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs= k8s.io/client-go v0.32.0 h1:DimtMcnN/JIKZcrSrstiwvvZvLjG0aSxy8PxN8IChp8= k8s.io/client-go v0.32.0/go.mod h1:boDWvdM1Drk4NJj/VddSLnx59X3OPgwrOo0vGbtq9+8= -k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs= -k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo= -k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8= -k8s.io/component-base v0.31.1/go.mod h1:WGeaw7t/kTsqpVTaCoVEtillbqAhF2/JgvO0LDOMa0w= -k8s.io/component-base v0.31.2 h1:Z1J1LIaC0AV+nzcPRFqfK09af6bZ4D1nAOpWsy9owlA= -k8s.io/component-base v0.31.2/go.mod h1:9PeyyFN/drHjtJZMCTkSpQJS3U9OXORnHQqMLDz0sUQ= k8s.io/component-base v0.32.0 h1:d6cWHZkCiiep41ObYQS6IcgzOUQUNpywm39KVYaUqzU= k8s.io/component-base v0.32.0/go.mod h1:JLG2W5TUxUu5uDyKiH2R/7NnxJo1HlPoRIIbVLkK5eM= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= -k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= -k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY9mD9fNT47QO6HI= -k8s.io/utils v0.0.0-20240921022957-49e7df575cb6/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJJI8IUa1AmH/qa0= k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= -sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= -sigs.k8s.io/controller-runtime v0.19.1 h1:Son+Q40+Be3QWb+niBXAg2vFiYWolDjjRfO8hn/cxOk= -sigs.k8s.io/controller-runtime v0.19.1/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/pkg/config/config.go b/pkg/config/config.go index 7b14d82d..9e193b2f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -232,6 +232,29 @@ func LoadClientConfig() (*AppConfig, error) { }, nil } +func LoadSyncClientConfig() (*AppConfig, error) { + sqsEndpoint := getEnv("SQS_ENDPOINT", "") + if sqsEndpoint == "" { + return nil, fmt.Errorf("environment variable SQS_ENDPOINT is not set") + } + + sqsAwsRegion := getEnv("SQS_AWS_REGION", "") + if sqsAwsRegion == "" { + return nil, fmt.Errorf("environment variable SQS_AWS_REGION is not set") + } + + sqsQueueName := getEnv("SQS_QUEUE_NAME", "") + if sqsQueueName == "" { + return nil, fmt.Errorf("environment variable SQS_QUEUE_NAME is not set") + } + + return &AppConfig{ + SqsEndpoint: sqsEndpoint, + SqsAwsRegion: sqsAwsRegion, + SqsQueueName: sqsQueueName, + }, nil +} + func getEnv(varName string, defaultValue string) string { varValue := os.Getenv(varName) if len(varValue) == 0 { diff --git a/pkg/sqs/synconsumer.go b/pkg/sqs/synconsumer.go deleted file mode 100644 index 9e955be3..00000000 --- a/pkg/sqs/synconsumer.go +++ /dev/null @@ -1,141 +0,0 @@ -/* -Copyright 2024 Adobe. All rights reserved. -This file is licensed to you under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. You may obtain a copy -of the License at http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under -the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS -OF ANY KIND, either express or implied. See the License for the specific language -governing permissions and limitations under the License. -*/ - -package sqs - -import ( - "sync" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/sqs" - "github.com/aws/aws-sdk-go/service/sqs/sqsiface" - "github.com/labstack/gommon/log" - - "github.com/adobe/cluster-registry/pkg/config" - monitoring "github.com/adobe/cluster-registry/pkg/monitoring/apiserver" -) - -// consumer struct -type synconsumer struct { - sqs sqsiface.SQSAPI - queueURL string - workerPool int - maxMessages int64 - pollWaitSeconds int64 - retrySeconds int - messageHandler func(*sqs.Message) error -} - -// NewSyncConsumer - creates a new SQS message queue consumer -// used by the sync consumer service -// TODO: add metrics later -func NewSyncConsumer(sqsSvc sqsiface.SQSAPI, appConfig *config.AppConfig, h func(*sqs.Message) error) Consumer { - - urlResult, err := sqsSvc.GetQueueUrl(&sqs.GetQueueUrlInput{ - QueueName: &appConfig.SqsQueueName, - }) - if err != nil { - log.Fatal(err.Error()) - } - - return &synconsumer{ - sqs: sqsSvc, - queueURL: *urlResult.QueueUrl, - workerPool: 10, - maxMessages: 1, - pollWaitSeconds: 1, - retrySeconds: 5, - messageHandler: h, - } -} - -// Status verifies the status/connectivity of the sqs service -func (c *synconsumer) Status(appConfig *config.AppConfig, m monitoring.MetricsI) error { - _, err := c.sqs.GetQueueUrl(&sqs.GetQueueUrlInput{ - QueueName: &appConfig.SqsQueueName, - }) - - if err != nil { - log.Error(err.Error()) - } - - return err -} - -// Consume - long pooling -func (c *synconsumer) Consume() { - var wg sync.WaitGroup - - for w := 1; w <= c.workerPool; w++ { - wg.Add(1) - go func(w int) { - defer wg.Done() - c.worker(w) - }(w) - } - wg.Wait() -} - -func (c *synconsumer) worker(id int) { - for { - output, err := c.sqs.ReceiveMessage((&sqs.ReceiveMessageInput{ - QueueUrl: &c.queueURL, - AttributeNames: aws.StringSlice([]string{ - "ClusterName", "SentTimestamp", - }), - MaxNumberOfMessages: aws.Int64(c.maxMessages), - WaitTimeSeconds: aws.Int64(c.pollWaitSeconds), - })) - - if err != nil { - log.Error(err.Error()) - log.Info("Retrying in", c.retrySeconds, " seconds") - time.Sleep(time.Duration(c.retrySeconds) * time.Second) - continue - } - - for _, m := range output.Messages { - log.Debug("Messsage ID: ", *m.MessageId) - log.Debug("Message Body: ", *m.Body) - - err := c.processMessage(m) - - if err != nil { - log.Error(err.Error()) - continue - } - err = c.delete(m) - if err != nil { - log.Error(err.Error()) - } - } - } -} - -// processMessage - process the recieved message -func (c *synconsumer) processMessage(m *sqs.Message) error { - - err := c.messageHandler(m) - return err -} - -func (c *synconsumer) delete(m *sqs.Message) error { - - _, err := c.sqs.DeleteMessage( - &sqs.DeleteMessageInput{QueueUrl: &c.queueURL, ReceiptHandle: m.ReceiptHandle}) - - if err != nil { - log.Error(err.Error()) - } - return err -} diff --git a/pkg/sync/event/handler.go b/pkg/sync/event/handler.go index 058259e5..1ab52bce 100644 --- a/pkg/sync/event/handler.go +++ b/pkg/sync/event/handler.go @@ -15,6 +15,8 @@ package event import ( "errors" "github.com/adobe/cluster-registry/pkg/sqs" + "github.com/davecgh/go-spew/spew" + log "github.com/sirupsen/logrus" ) type PartialClusterUpdateHandler struct { @@ -38,7 +40,8 @@ func (h *PartialClusterUpdateHandler) Handle(event *sqs.Event) error { return errors.New("event type does not match handler type") } - // TODO + log.Info("Handling partial cluster update event") + log.Info(spew.Sdump(event)) return nil } From f5d840d5254bcae253108d8725b62eb1196f9867 Mon Sep 17 00:00:00 2001 From: aalexand Date: Thu, 13 Feb 2025 10:09:17 +0200 Subject: [PATCH 4/8] Minor refactoring; added controller rate limiting options --- cmd/apiserver/apiserver.go | 2 +- cmd/sync/client/client.go | 2 +- cmd/sync/manager/manager.go | 57 ++++++++++++++++++++++++++++------ pkg/sync/manager/controller.go | 17 +++++----- pkg/sync/manager/util.go | 18 +++++++++++ 5 files changed, 78 insertions(+), 18 deletions(-) diff --git a/cmd/apiserver/apiserver.go b/cmd/apiserver/apiserver.go index 554724bd..34e940ca 100644 --- a/cmd/apiserver/apiserver.go +++ b/cmd/apiserver/apiserver.go @@ -72,7 +72,7 @@ func main() { Endpoint: appConfig.SqsEndpoint, QueueName: appConfig.SqsQueueName, BatchSize: appConfig.SqsBatchSize, - VisibilityTimeout: 120, + VisibilityTimeout: 0, WaitSeconds: appConfig.SqsWaitSeconds, RunInterval: appConfig.SqsRunInterval, RunOnce: false, diff --git a/cmd/sync/client/client.go b/cmd/sync/client/client.go index f86d4b5a..784e30da 100644 --- a/cmd/sync/client/client.go +++ b/cmd/sync/client/client.go @@ -81,7 +81,7 @@ func run(cmd *cobra.Command, args []string) { Endpoint: appConfig.SqsEndpoint, QueueName: appConfig.SqsQueueName, BatchSize: 10, - VisibilityTimeout: 120, + VisibilityTimeout: 0, WaitSeconds: 5, RunInterval: 20, RunOnce: false, diff --git a/cmd/sync/manager/manager.go b/cmd/sync/manager/manager.go index abaf61e7..21fad553 100644 --- a/cmd/sync/manager/manager.go +++ b/cmd/sync/manager/manager.go @@ -35,9 +35,11 @@ import ( "net/http" "os" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "time" ) var ( @@ -55,11 +57,19 @@ func init() { func main() { ctx := ctrl.SetupSignalHandler() - var configFile string - var metricsAddr string - var probeAddr string - var namespace string - var enableLeaderElection bool + var ( + configFile string + metricsAddr string + probeAddr string + namespace string + enableLeaderElection bool + maxConcurrentReconciles int + workqueueDefaultSyncBackoffStr string + workqueueMaxSyncBackoffStr string + workqueueQPS int + workqueueBurst int + cacheSyncTimeoutStr string + ) flag.StringVar(&configFile, "config", "", "The controller will load its initial configuration from this file. "+ @@ -68,9 +78,13 @@ func main() { flag.StringVar(&metricsAddr, "metrics-bind-address", ":9090", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":9091", "The address the probe endpoint binds to.") flag.StringVar(&namespace, "namespace", "cluster-registry", "The namespace where cluster-registry-sync-manager will run.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + flag.IntVar(&maxConcurrentReconciles, "max-concurrent-reconciles", 10, "The maximum number of concurrent reconciles which can be run.") + flag.StringVar(&workqueueDefaultSyncBackoffStr, "workqueue-default-sync-backoff", "5ms", "Base backoff period for failed reconciliation in controller's workqueue.") + flag.StringVar(&workqueueMaxSyncBackoffStr, "workqueue-max-sync-backoff", "1000s", "Maximum backoff period for failed reconciliation in controller's workqueue.") + flag.IntVar(&workqueueQPS, "workqueue-qps", 10, "QPS limit value for controller's workqueue.") + flag.IntVar(&workqueueBurst, "workqueue-burst", 100, "Burst limit value for controller's workqueue.") + flag.StringVar(&cacheSyncTimeoutStr, "cache-sync-timeout", "5m", "Time limit set to wait for syncing caches.") opts := zap.Options{ // TODO: change this to false @@ -144,6 +158,31 @@ func main() { } client := mgr.GetClient() + + defaultSyncBackoff, err := time.ParseDuration(workqueueDefaultSyncBackoffStr) + if err != nil { + setupLog.Error(err, "workqueue-default-sync-backoff is not a valid duration") + os.Exit(1) + } + + maxSyncBackoff, err := time.ParseDuration(workqueueMaxSyncBackoffStr) + if err != nil { + setupLog.Error(err, "workqueue-max-sync-backoff is not a valid duration") + os.Exit(1) + } + + cacheSyncTimeout, err := time.ParseDuration(cacheSyncTimeoutStr) + if err != nil { + setupLog.Error(err, "cache-sync-timeout is not a valid duration") + os.Exit(1) + } + + ctrlOpts := controller.Options{ + MaxConcurrentReconciles: maxConcurrentReconciles, + RateLimiter: manager.NewControllerRateLimiter(defaultSyncBackoff, maxSyncBackoff, workqueueQPS, workqueueBurst), + CacheSyncTimeout: cacheSyncTimeout, + } + ctrlLog := ctrl.Log.WithName("controllers").WithName("SyncController") rp := parser.New(client, ctrlLog) @@ -162,7 +201,7 @@ func main() { Queue: q, ResourceParser: rp, Metrics: m, - }).SetupWithManager(ctx, mgr); err != nil { + }).SetupWithManager(ctx, mgr, ctrlOpts); err != nil { setupLog.Error(err, "unable to create controller", "controller", "SyncController") os.Exit(1) } diff --git a/pkg/sync/manager/controller.go b/pkg/sync/manager/controller.go index bcdec860..4e05e404 100644 --- a/pkg/sync/manager/controller.go +++ b/pkg/sync/manager/controller.go @@ -145,8 +145,7 @@ func (c *SyncController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. return noRequeue() } -func (c *SyncController) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { - options := controller.Options{MaxConcurrentReconciles: 10} +func (c *SyncController) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { b := ctrl.NewControllerManagedBy(mgr).For(®istryv1alpha1.ClusterSync{}, builder.WithPredicates(c.eventFilters())) for _, gvk := range c.WatchedGVKs { obj := new(unstructured.Unstructured) @@ -219,11 +218,9 @@ func (c *SyncController) enqueueRequestsFromMapFunc(gvk schema.GroupVersionKind) return requests } - // limit the search to the cluster sync namespace list := ®istryv1alpha1.ClusterSyncList{} - if err := c.List(ctx, list, &client.ListOptions{Namespace: obj.GetNamespace()}); err != nil { - c.Log.Error(err, "failed to list ClusterSync objects", - "namespace", obj.GetNamespace()) + if err := c.List(ctx, list, &client.ListOptions{}); err != nil { + c.Log.Error(err, "failed to list ClusterSync objects") return requests } @@ -234,7 +231,13 @@ func (c *SyncController) enqueueRequestsFromMapFunc(gvk schema.GroupVersionKind) c.Log.Error(err, "failed to parse resource API version") return requests } - if gv != gvk.GroupVersion() || res.Kind != gvk.Kind || clusterSync.Namespace != obj.GetNamespace() { + // if the namespace is specified, only enqueue if the object namespace matches + if res.Namespace != "" { + if res.Namespace != obj.GetNamespace() { + continue + } + } + if gv != gvk.GroupVersion() || res.Kind != gvk.Kind { continue } // if the name is specified, only enqueue if the object name matches diff --git a/pkg/sync/manager/util.go b/pkg/sync/manager/util.go index 3ac29a19..650fe41e 100644 --- a/pkg/sync/manager/util.go +++ b/pkg/sync/manager/util.go @@ -16,6 +16,10 @@ import ( "crypto/sha256" "fmt" jsoniter "github.com/json-iterator/go" + "golang.org/x/time/rate" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "time" ) var json = jsoniter.ConfigCompatibleWithStandardLibrary @@ -26,3 +30,17 @@ func hash(obj interface{}) string { h.Write([]byte(fmt.Sprintf("%v", b))) return fmt.Sprintf("%x", h.Sum(nil)) } + +func NewControllerRateLimiter( + exponentialFailureBaseDelay time.Duration, + exponentialFailureMaxDelay time.Duration, + overallBucketQPS int, + overallBucketBurst int) workqueue.TypedRateLimiter[reconcile.Request] { + return workqueue.NewTypedMaxOfRateLimiter[reconcile.Request]( + workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request]( + exponentialFailureBaseDelay, + exponentialFailureMaxDelay), + &workqueue.TypedBucketRateLimiter[reconcile.Request]{ + Limiter: rate.NewLimiter(rate.Limit(overallBucketQPS), overallBucketBurst)}, + ) +} From bbb73e673ae8ea56bfb56386b30789fc559db186 Mon Sep 17 00:00:00 2001 From: aalexand Date: Wed, 12 Mar 2025 15:26:19 +0200 Subject: [PATCH 5/8] Add logic for creating/updating cluster object --- ...registry.ethos.adobe.com_clustersyncs.yaml | 3 + cmd/sync/client/client.go | 10 +- cmd/sync/manager/manager.go | 2 +- ...registry.ethos.adobe.com_clustersyncs.yaml | 3 + go.mod | 24 ++-- go.sum | 46 +++--- pkg/api/registry/v1/cluster_types.go | 132 +++++++++--------- .../registry/v1alpha1/clustersync_types.go | 2 + pkg/sqs/event.go | 8 ++ pkg/sync/client/k8s.go | 17 +-- pkg/sync/client/utils.go | 78 ++--------- pkg/sync/event/handler.go | 80 ++++++++++- pkg/sync/manager/controller.go | 4 + 13 files changed, 220 insertions(+), 189 deletions(-) diff --git a/charts/cluster-registry-sync-manager/crds/registry.ethos.adobe.com_clustersyncs.yaml b/charts/cluster-registry-sync-manager/crds/registry.ethos.adobe.com_clustersyncs.yaml index afe6d8b1..f3ccef79 100644 --- a/charts/cluster-registry-sync-manager/crds/registry.ethos.adobe.com_clustersyncs.yaml +++ b/charts/cluster-registry-sync-manager/crds/registry.ethos.adobe.com_clustersyncs.yaml @@ -39,6 +39,8 @@ spec: spec: description: ClusterSyncSpec defines the desired state of ClusterSync properties: + clusterName: + type: string initialData: type: string watchedResources: @@ -109,6 +111,7 @@ spec: type: object type: array required: + - clusterName - watchedResources type: object status: diff --git a/cmd/sync/client/client.go b/cmd/sync/client/client.go index 784e30da..830e0b88 100644 --- a/cmd/sync/client/client.go +++ b/cmd/sync/client/client.go @@ -66,7 +66,7 @@ func loadAppConfig(cmd *cobra.Command, args []string) { var err error appConfig, err = config.LoadSyncClientConfig() if err != nil { - log.Error("Cannot load the cluster-registry-sync-client configuration:", err.Error()) + log.Error("Cannot load the cluster-registry-sync-client configuration: ", err.Error()) os.Exit(1) } @@ -92,7 +92,13 @@ func run(cmd *cobra.Command, args []string) { log.Panicf("Error while trying to create SQS client: %v", err.Error()) } - handler := event.NewPartialClusterUpdateHandler() + dynamicClient, err := client.GetDynamicClientSet() + if err != nil { + log.Error("Error while trying to create dynamic client: ", err.Error()) + os.Exit(1) + } + + handler := event.NewPartialClusterUpdateHandler(dynamicClient, namespace) q.RegisterHandler(func(msg *awssqs.Message) { log.Debugf("Received message: %s", *msg.MessageId) e, err := sqs.NewEvent(msg) diff --git a/cmd/sync/manager/manager.go b/cmd/sync/manager/manager.go index 21fad553..d3f45969 100644 --- a/cmd/sync/manager/manager.go +++ b/cmd/sync/manager/manager.go @@ -145,7 +145,7 @@ func main() { Endpoint: appConfig.SqsEndpoint, QueueName: appConfig.SqsQueueName, BatchSize: 10, - VisibilityTimeout: 120, + VisibilityTimeout: 0, WaitSeconds: 5, RunInterval: 20, RunOnce: false, diff --git a/config/crd/bases/registry.ethos.adobe.com_clustersyncs.yaml b/config/crd/bases/registry.ethos.adobe.com_clustersyncs.yaml index afe6d8b1..f3ccef79 100644 --- a/config/crd/bases/registry.ethos.adobe.com_clustersyncs.yaml +++ b/config/crd/bases/registry.ethos.adobe.com_clustersyncs.yaml @@ -39,6 +39,8 @@ spec: spec: description: ClusterSyncSpec defines the desired state of ClusterSync properties: + clusterName: + type: string initialData: type: string watchedResources: @@ -109,6 +111,7 @@ spec: type: object type: array required: + - clusterName - watchedResources type: object status: diff --git a/go.mod b/go.mod index ba3844e5..ba4a5342 100644 --- a/go.mod +++ b/go.mod @@ -32,14 +32,15 @@ require ( github.com/swaggo/swag v1.16.4 github.com/testcontainers/testcontainers-go v0.34.0 golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 - golang.org/x/net v0.32.0 + golang.org/x/net v0.35.0 + golang.org/x/time v0.8.0 gopkg.in/go-playground/validator.v9 v9.31.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.32.0 - k8s.io/apimachinery v0.32.0 - k8s.io/client-go v0.32.0 - k8s.io/component-base v0.32.0 + k8s.io/api v0.32.3 + k8s.io/apimachinery v0.32.3 + k8s.io/client-go v0.32.3 + k8s.io/component-base v0.32.3 k8s.io/utils v0.0.0-20241210054802-24370beab758 sigs.k8s.io/controller-runtime v0.19.3 sigs.k8s.io/yaml v1.4.0 @@ -140,13 +141,12 @@ require ( go.opentelemetry.io/otel/trace v1.29.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/crypto v0.35.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.8.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/term v0.29.0 // indirect + golang.org/x/text v0.22.0 // indirect golang.org/x/tools v0.28.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect @@ -156,7 +156,7 @@ require ( gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - k8s.io/apiextensions-apiserver v0.31.0 // indirect + k8s.io/apiextensions-apiserver v0.32.3 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect diff --git a/go.sum b/go.sum index 92271f37..97b9bda7 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -396,8 +396,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= +golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -407,8 +407,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -437,16 +437,16 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -454,8 +454,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -474,7 +474,7 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13 h1:vlzZttNJGVqTsRFU9AmdnrcO1Znh8Ew9kCD//yjigk0= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= @@ -517,16 +517,16 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= -k8s.io/api v0.32.0 h1:OL9JpbvAU5ny9ga2fb24X8H6xQlVp+aJMFlgtQjR9CE= -k8s.io/api v0.32.0/go.mod h1:4LEwHZEf6Q/cG96F3dqR965sYOfmPM7rq81BLgsE0p0= -k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= -k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= -k8s.io/apimachinery v0.32.0 h1:cFSE7N3rmEEtv4ei5X6DaJPHHX0C+upp+v5lVPiEwpg= -k8s.io/apimachinery v0.32.0/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/client-go v0.32.0 h1:DimtMcnN/JIKZcrSrstiwvvZvLjG0aSxy8PxN8IChp8= -k8s.io/client-go v0.32.0/go.mod h1:boDWvdM1Drk4NJj/VddSLnx59X3OPgwrOo0vGbtq9+8= -k8s.io/component-base v0.32.0 h1:d6cWHZkCiiep41ObYQS6IcgzOUQUNpywm39KVYaUqzU= -k8s.io/component-base v0.32.0/go.mod h1:JLG2W5TUxUu5uDyKiH2R/7NnxJo1HlPoRIIbVLkK5eM= +k8s.io/api v0.32.3 h1:Hw7KqxRusq+6QSplE3NYG4MBxZw1BZnq4aP4cJVINls= +k8s.io/api v0.32.3/go.mod h1:2wEDTXADtm/HA7CCMD8D8bK4yuBUptzaRhYcYEEYA3k= +k8s.io/apiextensions-apiserver v0.32.3 h1:4D8vy+9GWerlErCwVIbcQjsWunF9SUGNu7O7hiQTyPY= +k8s.io/apiextensions-apiserver v0.32.3/go.mod h1:8YwcvVRMVzw0r1Stc7XfGAzB/SIVLunqApySV5V7Dss= +k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U= +k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU= +k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY= +k8s.io/component-base v0.32.3 h1:98WJvvMs3QZ2LYHBzvltFSeJjEx7t5+8s71P7M74u8k= +k8s.io/component-base v0.32.3/go.mod h1:LWi9cR+yPAv7cu2X9rZanTiFKB2kHA+JjmhkKjCZRpI= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= diff --git a/pkg/api/registry/v1/cluster_types.go b/pkg/api/registry/v1/cluster_types.go index 0342f9ee..38e08096 100644 --- a/pkg/api/registry/v1/cluster_types.go +++ b/pkg/api/registry/v1/cluster_types.go @@ -23,115 +23,115 @@ type ClusterSpec struct { // +kubebuilder:validation:Required // +kubebuilder:validation:MaxLength=64 // +kubebuilder:validation:MinLength=3 - Name string `json:"name"` + Name string `json:"name" yaml:"name"` // Cluster name, without dash // +kubebuilder:validation:Required // +kubebuilder:validation:MaxLength=64 // +kubebuilder:validation:MinLength=3 - ShortName string `json:"shortName"` + ShortName string `json:"shortName" yaml:"shortName"` // Information about K8s API endpoint and CA cert // +kubebuilder:validation:Required - APIServer APIServer `json:"apiServer"` + APIServer APIServer `json:"apiServer" yaml:"apiServer"` // Cluster internal region name // +kubebuilder:validation:Required - Region string `json:"region"` + Region string `json:"region" yaml:"region"` // The cloud provider // +kubebuilder:validation:Required - CloudType string `json:"cloudType"` + CloudType string `json:"cloudType" yaml:"cloudType"` // The cloud provider standard region // +kubebuilder:validation:Required - CloudProviderRegion string `json:"cloudProviderRegion"` + CloudProviderRegion string `json:"cloudProviderRegion" yaml:"cloudProviderRegion"` // Cluster environment // +kubebuilder:validation:Required - Environment string `json:"environment"` + Environment string `json:"environment" yaml:"environment"` // The BU that owns the cluster // +kubebuilder:validation:Required - BusinessUnit string `json:"businessUnit"` + BusinessUnit string `json:"businessUnit" yaml:"businessUnit"` // The BU responsible for paying for the cluster. - ChargebackBusinessUnit string `json:"chargebackBusinessUnit,omitempty"` + ChargebackBusinessUnit string `json:"chargebackBusinessUnit,omitempty" yaml:"chargebackBusinessUnit,omitempty"` // Whether the cluster is charged back to the chargebackBusinessUnit - ChargedBack *bool `json:"chargedBack,omitempty"` + ChargedBack *bool `json:"chargedBack,omitempty" yaml:"chargedBack,omitempty"` // The Org that is responsible for the cluster operations // +kubebuilder:validation:Required - ManagingOrg string `json:"managingOrg"` + ManagingOrg string `json:"managingOrg" yaml:"managingOrg"` // The Offering that the cluster is meant for // +kubebuilder:validation:Required - Offering []Offering `json:"offering"` + Offering []Offering `json:"offering" yaml:"offering"` // The cloud account associated with the cluster // +kubebuilder:validation:Required - AccountID string `json:"accountId"` + AccountID string `json:"accountId" yaml:"accountID"` // List of tiers with their associated information // +kubebuilder:validation:Required - Tiers []Tier `json:"tiers"` + Tiers []Tier `json:"tiers" yaml:"tiers"` // Virtual Private Networks information // +kubebuilder:validation:Required - VirtualNetworks []VirtualNetwork `json:"virtualNetworks"` + VirtualNetworks []VirtualNetwork `json:"virtualNetworks" yaml:"virtualNetworks"` // Timestamp when cluster was registered in Cluster Registry // +kubebuilder:validation:Required - RegisteredAt string `json:"registeredAt"` + RegisteredAt string `json:"registeredAt" yaml:"registeredAt"` // Cluster status // +kubebuilder:validation:Required // +kubebuilder:validation:Enum=Inactive;Active;Deprecated;Deleted - Status string `json:"status"` + Status string `json:"status" yaml:"status"` // Cluster phase // +kubebuilder:validation:Required // +kubebuilder:validation:Enum=Building;Testing;Running;Upgrading - Phase string `json:"phase"` + Phase string `json:"phase" yaml:"phase"` // Cluster maintenance group // +kubebuilder:validation:Required - MaintenanceGroup string `json:"maintenanceGroup"` + MaintenanceGroup string `json:"maintenanceGroup" yaml:"maintenanceGroup"` // The corresponding Argo instance of the cluster // +kubebuilder:validation:Required - ArgoInstance string `json:"argoInstance"` + ArgoInstance string `json:"argoInstance" yaml:"argoInstance"` // The type of the cluster - Type string `json:"type,omitempty"` + Type string `json:"type,omitempty" yaml:"type,omitempty"` // Extra information, not necessary related to the cluster - Extra Extra `json:"extra,omitempty"` + Extra Extra `json:"extra,omitempty" yaml:"extra,omitempty"` // Git teams and/or LDAP groups that are allowed to onboard and deploy on the cluster - AllowedOnboardingTeams []AllowedOnboardingTeam `json:"allowedOnboardingTeams,omitempty"` + AllowedOnboardingTeams []AllowedOnboardingTeam `json:"allowedOnboardingTeams,omitempty" yaml:"allowedOnboardingTeams,omitempty"` // List of cluster capabilities - Capabilities []string `json:"capabilities,omitempty"` + Capabilities []string `json:"capabilities,omitempty" yaml:"capabilities,omitempty"` // Information about Virtual Networks manual peered with the cluster - PeerVirtualNetworks []PeerVirtualNetwork `json:"peerVirtualNetworks,omitempty"` + PeerVirtualNetworks []PeerVirtualNetwork `json:"peerVirtualNetworks,omitempty" yaml:"peerVirtualNetworks,omitempty"` // Timestamp when cluster information was updated - LastUpdated string `json:"lastUpdated"` + LastUpdated string `json:"lastUpdated" yaml:"lastUpdated"` // Cluster tags that were applied - Tags map[string]string `json:"tags,omitempty"` + Tags map[string]string `json:"tags,omitempty" yaml:"tags,omitempty"` // Capacity cluster information - Capacity Capacity `json:"capacity,omitempty"` + Capacity Capacity `json:"capacity,omitempty" yaml:"capacity,omitempty"` // ServiceMetadata service specific metadata - ServiceMetadata ServiceMetadata `json:"services,omitempty"` + ServiceMetadata ServiceMetadata `json:"services,omitempty" yaml:"serviceMetadata,omitempty"` // AvailabilityZones cluster availability zones - AvailabilityZones []AvailabilityZone `json:"availabilityZones,omitempty"` + AvailabilityZones []AvailabilityZone `json:"availabilityZones,omitempty" yaml:"availabilityZones,omitempty"` } // Offering the cluster is meant for @@ -143,10 +143,10 @@ type APIServer struct { // Information about K8s Api Endpoint // +kubebuilder:validation:Required - Endpoint string `json:"endpoint"` + Endpoint string `json:"endpoint" yaml:"endpoint"` // Information about K8s Api CA Cert - CertificateAuthorityData string `json:"certificateAuthorityData"` + CertificateAuthorityData string `json:"certificateAuthorityData" yaml:"certificateAuthorityData"` } // AllowedOnboardingTeam represents the Git teams and/or LDAP groups that are allowed to onboard @@ -154,43 +154,43 @@ type AllowedOnboardingTeam struct { // Name of the team // +kubebuilder:validation:Required - Name string `json:"name"` + Name string `json:"name" yaml:"name"` // List of git teams - GitTeams []string `json:"gitTeams,omitempty"` + GitTeams []string `json:"gitTeams,omitempty" yaml:"gitTeams,omitempty"` // List of ldap groups - LdapGroups []string `json:"ldapGroups,omitempty"` + LdapGroups []string `json:"ldapGroups,omitempty" yaml:"ldapGroups,omitempty"` } // Extra information type Extra struct { // Name of the domain - DomainName string `json:"domainName"` + DomainName string `json:"domainName" yaml:"domainName"` // Load balancer endpoints - LbEndpoints map[string]string `json:"lbEndpoints"` + LbEndpoints map[string]string `json:"lbEndpoints" yaml:"lbEndpoints"` // Logging endpoints - LoggingEndpoints []map[string]string `json:"loggingEndpoints,omitempty"` + LoggingEndpoints []map[string]string `json:"loggingEndpoints,omitempty" yaml:"loggingEndpoints,omitempty"` // List of IAM Arns - EcrIamArns map[string][]string `json:"ecrIamArns,omitempty"` + EcrIamArns map[string][]string `json:"ecrIamArns,omitempty" yaml:"ecrIamArns,omitempty"` // Egress ports allowed outside of the namespace - EgressPorts string `json:"egressPorts,omitempty"` + EgressPorts string `json:"egressPorts,omitempty" yaml:"egressPorts,omitempty"` // NFS information - NFSInfo []map[string]string `json:"nfsInfo,omitempty"` + NFSInfo []map[string]string `json:"nfsInfo,omitempty" yaml:"nfsInfo,omitempty"` // ExtendedRegion information - ExtendedRegion string `json:"extendedRegion,omitempty"` + ExtendedRegion string `json:"extendedRegion,omitempty" yaml:"extendedRegion,omitempty"` // OIDC Issuer URL - OidcIssuer string `json:"oidcIssuer,omitempty"` + OidcIssuer string `json:"oidcIssuer,omitempty" yaml:"oidcIssuer,omitempty"` // Namespace Profile Infrastructure Type - NamespaceProfileInfraType string `json:"namespaceProfileInfraType,omitempty"` + NamespaceProfileInfraType string `json:"namespaceProfileInfraType,omitempty" yaml:"namespaceProfileInfraType,omitempty"` } // Tier details @@ -198,36 +198,36 @@ type Tier struct { // Name of the tier // +kubebuilder:validation:Required - Name string `json:"name"` + Name string `json:"name" yaml:"name"` // Type of the instances // +kubebuilder:validation:Required - InstanceType string `json:"instanceType"` + InstanceType string `json:"instanceType" yaml:"instanceType"` // Container runtime // +kubebuilder:validation:Required // +kubebuilder:validation:Enum=docker;cri-o - ContainerRuntime string `json:"containerRuntime"` + ContainerRuntime string `json:"containerRuntime" yaml:"containerRuntime"` // Min number of instances // +kubebuilder:validation:Required - MinCapacity int `json:"minCapacity"` + MinCapacity int `json:"minCapacity" yaml:"minCapacity"` // Max number of instances // +kubebuilder:validation:Required - MaxCapacity int `json:"maxCapacity"` + MaxCapacity int `json:"maxCapacity" yaml:"maxCapacity"` // Instance K8s labels - Labels map[string]string `json:"labels,omitempty"` + Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` // Instance K8s taints - Taints []string `json:"taints,omitempty"` + Taints []string `json:"taints,omitempty" yaml:"taints,omitempty"` // EnableKataSupport - EnableKataSupport bool `json:"enableKataSupport,omitempty"` + EnableKataSupport bool `json:"enableKataSupport,omitempty" yaml:"enableKataSupport,omitempty"` // KernelParameters - KernelParameters map[string]string `json:"kernelParameters,omitempty"` + KernelParameters map[string]string `json:"kernelParameters,omitempty" yaml:"kernelParameters,omitempty"` } // VirtualNetwork information @@ -235,34 +235,34 @@ type VirtualNetwork struct { // Virtual private network Id // +kubebuilder:validation:Required - ID string `json:"id"` + ID string `json:"id" yaml:"id"` // CIDRs used in this VirtualNetwork // +kubebuilder:validation:Required - Cidrs []string `json:"cidrs"` + Cidrs []string `json:"cidrs" yaml:"cidrs"` } // PeerVirtualNetwork - peering information done at cluster onboarding type PeerVirtualNetwork struct { // Remote Virtual Netowrk ID - ID string `json:"id,omitempty"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` // Remote Virtual Netowrk CIDRs - Cidrs []string `json:"cidrs,omitempty"` + Cidrs []string `json:"cidrs,omitempty" yaml:"cidrs,omitempty"` // Cloud account of the owner - OwnerID string `json:"ownerID,omitempty"` + OwnerID string `json:"ownerID,omitempty" yaml:"ownerID,omitempty"` } // Capacity cluster information type Capacity struct { - LastUpdated string `json:"lastUpdated"` - ClusterCapacity int `json:"clusterCapacity"` - ClusterProvisioning int `json:"clusterProvisioning"` - MaxBQUPerRequest int `json:"maxBquPerRequest"` - ClusterMaxBQU int `json:"clusterMaxBqu"` - ClusterCurrentBQU int `json:"clusterCurrentBqu"` + LastUpdated string `json:"lastUpdated" yaml:"lastUpdated"` + ClusterCapacity int `json:"clusterCapacity" yaml:"clusterCapacity"` + ClusterProvisioning int `json:"clusterProvisioning" yaml:"clusterProvisioning"` + MaxBQUPerRequest int `json:"maxBquPerRequest" yaml:"maxBquPerRequest"` + ClusterMaxBQU int `json:"clusterMaxBqu" yaml:"clusterMaxBqu"` + ClusterCurrentBQU int `json:"clusterCurrentBqu" yaml:"clusterCurrentBqu"` } type ServiceMetadata map[string]ServiceMetadataItem @@ -272,8 +272,8 @@ type ServiceMetadataItem map[string]ServiceMetadataMap type ServiceMetadataMap map[string]string type AvailabilityZone struct { - Name string `json:"name"` - ID string `json:"id,omitempty"` + Name string `json:"name" yaml:"name"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` } // ClusterStatus defines the observed state of Cluster diff --git a/pkg/api/registry/v1alpha1/clustersync_types.go b/pkg/api/registry/v1alpha1/clustersync_types.go index a8f64b18..625468ff 100644 --- a/pkg/api/registry/v1alpha1/clustersync_types.go +++ b/pkg/api/registry/v1alpha1/clustersync_types.go @@ -34,6 +34,8 @@ type WatchedResource struct { // ClusterSyncSpec defines the desired state of ClusterSync type ClusterSyncSpec struct { + // +required + ClusterName string `json:"clusterName"` // +required // +kubebuilder:validation:Required WatchedResources []WatchedResource `json:"watchedResources"` diff --git a/pkg/sqs/event.go b/pkg/sqs/event.go index 61007700..5a461422 100644 --- a/pkg/sqs/event.go +++ b/pkg/sqs/event.go @@ -63,3 +63,11 @@ type EventHandler interface { Type() string Handle(event *Event) error } + +// GetClusterName returns the cluster name from the message +func (e *Event) GetClusterName() (string, error) { + if e.Message.MessageAttributes[MessageAttributeClusterName] == nil { + return "", errors.New("missing cluster name") + } + return *e.Message.MessageAttributes[MessageAttributeClusterName].StringValue, nil +} diff --git a/pkg/sync/client/k8s.go b/pkg/sync/client/k8s.go index 097ffce7..c17b60cb 100644 --- a/pkg/sync/client/k8s.go +++ b/pkg/sync/client/k8s.go @@ -14,25 +14,10 @@ package client import ( "k8s.io/client-go/dynamic" - "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client/config" ) -func getClientSet() (*kubernetes.Clientset, error) { - cfg, err := config.GetConfig() - if err != nil { - return nil, err - } - - clientSet, err := kubernetes.NewForConfig(cfg) - if err != nil { - return nil, err - } - - return clientSet, nil -} - -func getDynamicClientSet() (*dynamic.DynamicClient, error) { +func GetDynamicClientSet() (*dynamic.DynamicClient, error) { cfg, err := config.GetConfig() if err != nil { return nil, err diff --git a/pkg/sync/client/utils.go b/pkg/sync/client/utils.go index d923b7e3..40f7bcac 100644 --- a/pkg/sync/client/utils.go +++ b/pkg/sync/client/utils.go @@ -14,77 +14,12 @@ package client import ( "encoding/json" - "io" - "log" - "os" "strings" - registryv1 "github.com/adobe/cluster-registry/pkg/api/registry/v1" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "sigs.k8s.io/yaml" ) -func ReadFile(patchFilePath string) ([]byte, error) { - patchFile, err := os.Open(patchFilePath) - if err != nil { - log.Fatalf("Error opening patch YAML file: %v", err) - } - defer patchFile.Close() - - patchData, err := io.ReadAll(patchFile) - if err != nil { - log.Fatalf("Error reading patch YAML file: %v", err) - } - return patchData, nil -} - -func UnmarshalYaml(data []byte, cluster *[]registryv1.Cluster) error { - err := yaml.Unmarshal(data, cluster) - if err != nil { - log.Panicf("Error while trying to unmarshal yaml data: %v", err.Error()) - } - - return err -} - -func UnmarshalJSON(data []byte, cluster *[]registryv1.Cluster) error { - err := json.Unmarshal(data, cluster) - if err != nil { - log.Panicf("Error while trying to unmarshal json data: %v", err.Error()) - } - - return err -} - -func MarshalJson(patch map[string]interface{}) ([]byte, error) { - jsonData, err := json.Marshal(patch) - if err != nil { - log.Panicf("Error while trying to marshal json data: %v", err.Error()) - } - - return jsonData, err -} - -// toUnstructured converts a Cluster struct to an unstructured.Unstructured object -func toUnstructured(obj interface{}) (*unstructured.Unstructured, error) { - data, err := json.Marshal(obj) - if err != nil { - return nil, err - } - u := &unstructured.Unstructured{} - if err := u.UnmarshalJSON(data); err != nil { - return nil, err - } - return u, nil -} - -// TODO; check if there is an utils func - see if no need -// unstructuredToJSON converts an unstructured.Unstructured object to a JSON string -func unstructuredToJSON(obj *unstructured.Unstructured) ([]byte, error) { - return obj.MarshalJSON() -} - func InitLogger(logLevel string, logFormat string) { level, err := logrus.ParseLevel(logLevel) @@ -105,3 +40,16 @@ func InitLogger(logLevel string, logFormat string) { }) } } + +// ToUnstructured converts a Kubernetes object to an unstructured.Unstructured +func ToUnstructured(obj interface{}) (*unstructured.Unstructured, error) { + data, err := json.Marshal(obj) + if err != nil { + return nil, err + } + u := &unstructured.Unstructured{} + if err := u.UnmarshalJSON(data); err != nil { + return nil, err + } + return u, nil +} diff --git a/pkg/sync/event/handler.go b/pkg/sync/event/handler.go index 1ab52bce..1567c24d 100644 --- a/pkg/sync/event/handler.go +++ b/pkg/sync/event/handler.go @@ -13,18 +13,31 @@ governing permissions and limitations under the License. package event import ( + "context" "errors" + v1 "github.com/adobe/cluster-registry/pkg/api/registry/v1" "github.com/adobe/cluster-registry/pkg/sqs" - "github.com/davecgh/go-spew/spew" + "github.com/adobe/cluster-registry/pkg/sync/client" log "github.com/sirupsen/logrus" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/client-go/dynamic" + "strconv" ) type PartialClusterUpdateHandler struct { sqs.EventHandler + client *dynamic.DynamicClient + namespace string } -func NewPartialClusterUpdateHandler() *PartialClusterUpdateHandler { - return &PartialClusterUpdateHandler{} +func NewPartialClusterUpdateHandler( + client *dynamic.DynamicClient, + namespace string, +) *PartialClusterUpdateHandler { + return &PartialClusterUpdateHandler{client: client, namespace: namespace} } func (h *PartialClusterUpdateHandler) Type() string { @@ -41,7 +54,66 @@ func (h *PartialClusterUpdateHandler) Handle(event *sqs.Event) error { } log.Info("Handling partial cluster update event") - log.Info(spew.Sdump(event)) + + // try to get cluster name from message + clusterName, err := event.GetClusterName() + if err != nil { + log.Error("Failed to get cluster name from message") + return err + } + + msg, _ := strconv.Unquote(*event.Message.Body) + data := []byte(msg) + + clusterResource := v1.GroupVersion.WithResource("clusters") + + // try to patch Cluster object if it exists + _, err = h.client.Resource(clusterResource).Namespace(h.namespace).Patch(context.TODO(), clusterName, types.MergePatchType, data, metav1.PatchOptions{}) + if err != nil { + + // if Cluster object does not exist, attempt to create it + if kerrors.IsNotFound(err) { + log.Info("Cluster object not found, checking if it is a new cluster") + + clusterSpec := v1.ClusterSpec{} + err = json.Unmarshal(data, &clusterSpec) + if err != nil { + log.Error("Failed to unmarshal cluster spec from message") + return err + } + + cluster := v1.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: v1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: h.namespace, + }, + Spec: clusterSpec, + } + + obj, err := client.ToUnstructured(cluster) + if err != nil { + log.Error("Failed to convert cluster to unstructured.") + } + + create, err := h.client.Resource(clusterResource).Namespace(h.namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) + if err != nil { + // if Cluster object is invalid, log and return; + // this most likely means that the sync-manager has not yet gathered all the required data for this cluster + if kerrors.IsInvalid(err) { + log.Error("Invalid Cluster object: ", err) + return nil + } + log.Error("Failed to create Cluster object: ", err) + return err + } + log.Info("Cluster object created: ", create.GetName()) + } + return err + } return nil } diff --git a/pkg/sync/manager/controller.go b/pkg/sync/manager/controller.go index 4e05e404..190b2312 100644 --- a/pkg/sync/manager/controller.go +++ b/pkg/sync/manager/controller.go @@ -304,6 +304,10 @@ func (c *SyncController) enqueueData(instance *registryv1alpha1.ClusterSync) err DataType: aws.String("String"), StringValue: aws.String(sqs.PartialClusterUpdateEvent), }, + "ClusterName": { + DataType: aws.String("String"), + StringValue: aws.String(instance.Spec.ClusterName), + }, }, MessageBody: aws.String(string(obj)), }, From 82ba8d4556e0648a002155e6cb11c30771c6b8b4 Mon Sep 17 00:00:00 2001 From: aalexand Date: Wed, 12 Mar 2025 16:27:31 +0200 Subject: [PATCH 6/8] Fix partial cluster patching --- pkg/sync/client/utils.go | 26 ++++++++++++++++++++++++++ pkg/sync/event/handler.go | 9 +++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/pkg/sync/client/utils.go b/pkg/sync/client/utils.go index 40f7bcac..8c2b3d14 100644 --- a/pkg/sync/client/utils.go +++ b/pkg/sync/client/utils.go @@ -14,6 +14,8 @@ package client import ( "encoding/json" + v1 "github.com/adobe/cluster-registry/pkg/api/registry/v1" + jsonpatch "github.com/evanphx/json-patch/v5" "strings" "github.com/sirupsen/logrus" @@ -53,3 +55,27 @@ func ToUnstructured(obj interface{}) (*unstructured.Unstructured, error) { } return u, nil } + +func PartialClusterMergePatch(data []byte) ([]byte, error) { + original, err := json.Marshal(v1.Cluster{ + Spec: v1.ClusterSpec{}, + }) + if err != nil { + return nil, err + } + + clusterSpec := v1.ClusterSpec{} + err = json.Unmarshal(data, &clusterSpec) + if err != nil { + return nil, err + } + + modified, err := json.Marshal(v1.Cluster{ + Spec: clusterSpec, + }) + if err != nil { + return nil, err + } + + return jsonpatch.CreateMergePatch(original, modified) +} diff --git a/pkg/sync/event/handler.go b/pkg/sync/event/handler.go index 1567c24d..394573a0 100644 --- a/pkg/sync/event/handler.go +++ b/pkg/sync/event/handler.go @@ -63,7 +63,12 @@ func (h *PartialClusterUpdateHandler) Handle(event *sqs.Event) error { } msg, _ := strconv.Unquote(*event.Message.Body) - data := []byte(msg) + + data, err := client.PartialClusterMergePatch([]byte(msg)) + if err != nil { + log.Error("Failed to create partial merge patch: ", err) + return err + } clusterResource := v1.GroupVersion.WithResource("clusters") @@ -76,7 +81,7 @@ func (h *PartialClusterUpdateHandler) Handle(event *sqs.Event) error { log.Info("Cluster object not found, checking if it is a new cluster") clusterSpec := v1.ClusterSpec{} - err = json.Unmarshal(data, &clusterSpec) + err = json.Unmarshal([]byte(msg), &clusterSpec) if err != nil { log.Error("Failed to unmarshal cluster spec from message") return err From 23f1c1a696be5f7b1a2c26f926b6f0cf3d403380 Mon Sep 17 00:00:00 2001 From: aalexand Date: Wed, 12 Mar 2025 17:51:14 +0200 Subject: [PATCH 7/8] Fix sqs/db tests --- local/sqs/sqs.go | 2 +- pkg/database/suite_test.go | 8 +++++--- pkg/sqs/sqs_test.go | 2 +- pkg/sqs/suite_test.go | 8 +++++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/local/sqs/sqs.go b/local/sqs/sqs.go index 1aff47c9..be109330 100644 --- a/local/sqs/sqs.go +++ b/local/sqs/sqs.go @@ -49,7 +49,7 @@ func main() { Endpoint: appConfig.SqsEndpoint, QueueName: appConfig.SqsQueueName, BatchSize: 10, - VisibilityTimeout: 120, + VisibilityTimeout: 0, WaitSeconds: 5, RunInterval: 20, RunOnce: false, diff --git a/pkg/database/suite_test.go b/pkg/database/suite_test.go index 215c7218..46e3f964 100644 --- a/pkg/database/suite_test.go +++ b/pkg/database/suite_test.go @@ -77,9 +77,11 @@ var _ = AfterSuite(func() { By("tearing down the test environment") gexec.KillAndWait(5 * time.Second) - err := dbContainer.Terminate(ctx) - if err != nil { - log.Fatalf("Error while creating the database container: %v", err.Error()) + if dbContainer != nil { + err := dbContainer.Terminate(ctx) + if err != nil { + log.Fatalf("Error while creating the database container: %v", err.Error()) + } } for k := range dbTestConfig { diff --git a/pkg/sqs/sqs_test.go b/pkg/sqs/sqs_test.go index a2c25c5a..efa05f18 100644 --- a/pkg/sqs/sqs_test.go +++ b/pkg/sqs/sqs_test.go @@ -48,7 +48,7 @@ var _ = Describe("SQS suite", func() { QueueName: appConfig.SqsQueueName, QueueURL: fmt.Sprintf("%s/%s/%s", container.Endpoint, "1234567890", appConfig.SqsQueueName), BatchSize: 1, - VisibilityTimeout: 120, + VisibilityTimeout: 0, WaitSeconds: 10, RunInterval: 5, RunOnce: true, diff --git a/pkg/sqs/suite_test.go b/pkg/sqs/suite_test.go index 24af5f25..5ab84a01 100644 --- a/pkg/sqs/suite_test.go +++ b/pkg/sqs/suite_test.go @@ -75,9 +75,11 @@ var _ = AfterSuite(func() { By("tearing down the test environment") gexec.KillAndWait(5 * time.Second) - err := container.Terminate(ctx) - if err != nil { - Fail(fmt.Sprintf("Error while terminating the SQS container: %v", err)) + if container != nil { + err := container.Terminate(ctx) + if err != nil { + Fail(fmt.Sprintf("Error while terminating the SQS container: %v", err)) + } } for k := range sqsTestConfig { From 5bc1291b170b3071c1b30060f52582db53e5835d Mon Sep 17 00:00:00 2001 From: aalexand Date: Mon, 7 Apr 2025 13:56:40 +0300 Subject: [PATCH 8/8] Add cluster-registry-sync-client chart --- Makefile | 17 ++- .../cluster-registry-sync-client/.helmignore | 22 +++ .../cluster-registry-sync-client/Chart.yaml | 19 +++ charts/cluster-registry-sync-client/README.md | 57 +++++++ .../templates/_helpers.tpl | 63 ++++++++ .../templates/clusterrole.yaml | 36 +++++ .../templates/clusterrolebinding.yaml | 14 ++ .../templates/deployment.yaml | 87 +++++++++++ .../templates/poddisruptionbudget.yaml | 19 +++ .../templates/podmonitor.yaml | 27 ++++ .../templates/service.yaml | 18 +++ .../templates/serviceaccount.yaml | 9 ++ .../cluster-registry-sync-client/values.yaml | 52 +++++++ cmd/apiserver/apiserver.go | 6 +- cmd/sync/client/Dockerfile | 5 + cmd/sync/client/client.go | 80 +++++++--- cmd/sync/manager/manager.go | 14 +- local/.env.local | 4 +- local/setup.sh | 20 +++ pkg/config/config.go | 139 +++++++++--------- 20 files changed, 606 insertions(+), 102 deletions(-) create mode 100644 charts/cluster-registry-sync-client/.helmignore create mode 100644 charts/cluster-registry-sync-client/Chart.yaml create mode 100644 charts/cluster-registry-sync-client/README.md create mode 100644 charts/cluster-registry-sync-client/templates/_helpers.tpl create mode 100644 charts/cluster-registry-sync-client/templates/clusterrole.yaml create mode 100644 charts/cluster-registry-sync-client/templates/clusterrolebinding.yaml create mode 100644 charts/cluster-registry-sync-client/templates/deployment.yaml create mode 100644 charts/cluster-registry-sync-client/templates/poddisruptionbudget.yaml create mode 100644 charts/cluster-registry-sync-client/templates/podmonitor.yaml create mode 100644 charts/cluster-registry-sync-client/templates/service.yaml create mode 100644 charts/cluster-registry-sync-client/templates/serviceaccount.yaml create mode 100644 charts/cluster-registry-sync-client/values.yaml create mode 100644 cmd/sync/client/Dockerfile diff --git a/Makefile b/Makefile index 4d7e4ccb..540bb16c 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,11 @@ ifeq ($(SYNC_MANAGER),false) else SETUP_CMD += "1" endif +ifeq ($(SYNC_CLIENT),false) + SETUP_CMD += "0" +else + SETUP_CMD += "1" +endif .PHONY: clean clean: @@ -71,7 +76,7 @@ setup: ############ .PHONY: build -build: build-apiserver build-client build-sync-manager +build: build-apiserver build-client build-sync-manager build-sync-client .PHONY: build-apiserver build-apiserver: @@ -85,13 +90,17 @@ build-client: build-sync-manager: $(GO_BUILD_RECIPE) -o cluster-registry-sync-manager cmd/sync/manager/manager.go +.PHONY: build-sync-client +build-sync-client: + $(GO_BUILD_RECIPE) -o cluster-registry-sync-client cmd/sync/client/client.go + .PHONY: release release: ./hack/release.sh .PHONY: image image: GOOS := linux -image: .hack-apiserver-image .hack-client-image .hack-sync-manager-image +image: .hack-apiserver-image .hack-client-image .hack-sync-manager-image .hack-sync-client-image .hack-apiserver-image: cmd/apiserver/Dockerfile build-apiserver docker build -t $(IMAGE_APISERVER):$(TAG) -f cmd/apiserver/Dockerfile . @@ -105,6 +114,10 @@ image: .hack-apiserver-image .hack-client-image .hack-sync-manager-image docker build -t $(IMAGE_SYNC_MANAGER):$(TAG) -f cmd/sync/manager/Dockerfile . touch $@ +.hack-sync-client-image: cmd/sync/client/Dockerfile build-sync-client + docker build -t $(IMAGE_SYNC_CLIENT):$(TAG) -f cmd/sync/client/Dockerfile . + touch $@ + .PHONY: update-go-deps update-go-deps: for m in $$(go list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \ diff --git a/charts/cluster-registry-sync-client/.helmignore b/charts/cluster-registry-sync-client/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/cluster-registry-sync-client/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/cluster-registry-sync-client/Chart.yaml b/charts/cluster-registry-sync-client/Chart.yaml new file mode 100644 index 00000000..1b47780f --- /dev/null +++ b/charts/cluster-registry-sync-client/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +name: cluster-registry-sync-client + +description: Cluster Registry is a Rest API representing the source of record for + all Kubernetes clusters in the infrastructure fleet. All clusters are automatically + registered, and the information is accurately reflected in the Cluster Registry + using a client-server architecture. + +type: application +home: https://github.com/adobe/cluster-registry + +maintainers: + - name: aalexandru + email: aalexand@adobe.com + - name: radu-catalina + email: caradu@adobe.com + +version: 0.0.1 +appVersion: v1.6.5 diff --git a/charts/cluster-registry-sync-client/README.md b/charts/cluster-registry-sync-client/README.md new file mode 100644 index 00000000..fd091b78 --- /dev/null +++ b/charts/cluster-registry-sync-client/README.md @@ -0,0 +1,57 @@ +# cluster-registry-sync-client + +![Version: 0.0.1](https://img.shields.io/badge/Version-0.0.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.6.5](https://img.shields.io/badge/AppVersion-v1.6.5-informational?style=flat-square) + +Cluster Registry is a Rest API representing the source of record for all Kubernetes clusters in the infrastructure fleet. All clusters are automatically registered, and the information is accurately reflected in the Cluster Registry using a client-server architecture. + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| aalexandru | | | +| radu-catalina | | | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| clusterRegistrySyncManager.health.healthProbeBindAddress | string | `":8081"` | | +| clusterRegistrySyncManager.leaderElection.leaderElect | bool | `false` | | +| clusterRegistrySyncManager.leaderElection.resourceLock | string | `"leases"` | | +| clusterRegistrySyncManager.leaderElection.resourceName | string | `"sync.registry.ethos.adobe.com"` | | +| clusterRegistrySyncManager.metrics.bindAddress | string | `"0.0.0.0:9090"` | | +| clusterRegistrySyncManager.watchedGVKs | object | `{}` | | +| clusterRegistrySyncManager.webhook.port | int | `9443` | | +| fullnameOverride | string | `"cluster-registry-sync-manager"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.registry | string | `"ghcr.io/adobe/cluster-registry-sync-manager"` | | +| imagePullSecrets | list | `[]` | | +| livenessProbe.httpGet.path | string | `"/healthz"` | | +| livenessProbe.httpGet.port | int | `9091` | | +| livenessProbe.initialDelaySeconds | int | `15` | | +| livenessProbe.periodSeconds | int | `20` | | +| nameOverride | string | `"cluster-registry-sync-manager"` | | +| podDisruptionBudget.enabled | bool | `true` | | +| podDisruptionBudget.minAvailable | string | `"50%"` | | +| podMonitor.enabled | bool | `false` | | +| podMonitor.extraLabels | object | `{}` | | +| ports[0].containerPort | int | `9090` | | +| ports[0].name | string | `"metrics"` | | +| rbac.create | bool | `true` | | +| readinessProbe.httpGet.path | string | `"/readyz"` | | +| readinessProbe.httpGet.port | int | `9091` | | +| readinessProbe.initialDelaySeconds | int | `5` | | +| readinessProbe.periodSeconds | int | `10` | | +| replicaCount | int | `2` | | +| resources.limits.cpu | string | `"200m"` | | +| resources.limits.memory | string | `"400Mi"` | | +| resources.requests.cpu | string | `"100m"` | | +| resources.requests.memory | string | `"200Mi"` | | +| serviceAccount.create | bool | `true` | | +| serviceAccount.name | string | `"cluster-registry-sync-manager"` | | +| terminationGracePeriodSeconds | int | `10` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/charts/cluster-registry-sync-client/templates/_helpers.tpl b/charts/cluster-registry-sync-client/templates/_helpers.tpl new file mode 100644 index 00000000..e3eccfff --- /dev/null +++ b/charts/cluster-registry-sync-client/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cluster-registry-sync-client.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cluster-registry-sync-client.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cluster-registry-sync-client.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cluster-registry-sync-client.labels" -}} +helm.sh/chart: {{ include "cluster-registry-sync-client.chart" . }} +{{ include "cluster-registry-sync-client.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +component: cluster-registry +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cluster-registry-sync-client.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cluster-registry-sync-client.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "cluster-registry-sync-client.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "cluster-registry-sync-client.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/cluster-registry-sync-client/templates/clusterrole.yaml b/charts/cluster-registry-sync-client/templates/clusterrole.yaml new file mode 100644 index 00000000..ec1ae349 --- /dev/null +++ b/charts/cluster-registry-sync-client/templates/clusterrole.yaml @@ -0,0 +1,36 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "cluster-registry-sync-client.fullname" . }} + labels: + {{- include "cluster-registry-sync-manager.labels" . | nindent 4 }} +rules: + - apiGroups: + - registry.ethos.adobe.com + resources: + - clusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - registry.ethos.adobe.com + resources: + - clusters/finalizers + verbs: + - update + - apiGroups: + - registry.ethos.adobe.com + resources: + - clusters/status + verbs: + - get + - patch + - update + {{- with .Values.extraRBAC }} + {{- toYaml . | nindent 2 }} + {{- end }} diff --git a/charts/cluster-registry-sync-client/templates/clusterrolebinding.yaml b/charts/cluster-registry-sync-client/templates/clusterrolebinding.yaml new file mode 100644 index 00000000..08ad0c19 --- /dev/null +++ b/charts/cluster-registry-sync-client/templates/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "cluster-registry-sync-client.fullname" . }} + labels: + {{- include "cluster-registry-sync-client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-registry-sync-client +subjects: + - kind: ServiceAccount + name: {{ include "cluster-registry-sync-client.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/charts/cluster-registry-sync-client/templates/deployment.yaml b/charts/cluster-registry-sync-client/templates/deployment.yaml new file mode 100644 index 00000000..2e6eab83 --- /dev/null +++ b/charts/cluster-registry-sync-client/templates/deployment.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + {{- include "cluster-registry-sync-client.labels" . | nindent 4 }} + name: {{ include "cluster-registry-sync-client.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + replicas: {{ .Values.replicaCount | required ".Values.replicaCount is required" }} + selector: + matchLabels: + {{- include "cluster-registry-sync-client.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "cluster-registry-sync-client.selectorLabels" . | nindent 8 }} + annotations: + kubectl.kubernetes.io/default-container: sync-client + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - args: + - --namespace={{ .Release.Namespace }} + {{- if .Values.clusterRegistrySyncClient.health.bindAddress }} + - --health-probe-bind-address={{ .Values.clusterRegistrySyncClient.health.bindAddress }} + {{- end }} + command: + - /bin/sync-client + image: "{{ .Values.image.registry }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + {{- toYaml .Values.ports | nindent 12 }} + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: AWS_ACCESS_KEY_ID + name: cluster-registry-aws + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: AWS_SECRET_ACCESS_KEY + name: cluster-registry-aws + - name: SQS_ENDPOINT + valueFrom: + secretKeyRef: + key: SQS_ENDPOINT + name: cluster-registry-aws + - name: SQS_QUEUE_NAME + valueFrom: + secretKeyRef: + key: SQS_QUEUE_NAME + name: cluster-registry-aws + - name: AWS_REGION + valueFrom: + secretKeyRef: + key: SQS_AWS_REGION + name: cluster-registry-aws + - name: SQS_AWS_REGION + valueFrom: + secretKeyRef: + key: SQS_AWS_REGION + name: cluster-registry-aws + {{- if .Values.livenessProbe }} + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + {{- end }} + {{- if .Values.readinessProbe }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + {{- end }} + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- end }} + name: sync-client + securityContext: + allowPrivilegeEscalation: false + volumes: + - name: {{ include "cluster-registry-sync-client.fullname" . }}-config + configMap: + name: {{ include "cluster-registry-sync-client.fullname" . }}-config + serviceAccountName: {{ include "cluster-registry-sync-client.serviceAccountName" . }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds | required ".Values.terminationGracePeriodSeconds is required" }} diff --git a/charts/cluster-registry-sync-client/templates/poddisruptionbudget.yaml b/charts/cluster-registry-sync-client/templates/poddisruptionbudget.yaml new file mode 100644 index 00000000..46728e9a --- /dev/null +++ b/charts/cluster-registry-sync-client/templates/poddisruptionbudget.yaml @@ -0,0 +1,19 @@ +{{- if .Values.podDisruptionBudget.enabled }} +{{- $isPercentage := regexMatch "^[0-9]+%$" (.Values.podDisruptionBudget.minAvailable | quote) }} +{{- if and (not $isPercentage) (le (.Values.replicaCount | int) (.Values.podDisruptionBudget.minAvailable | int)) }} +{{- fail ".Values.replicaCount should be greater than .Values.podDisruptionBudget.minAvailable" }} +{{- else }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + {{- include "cluster-registry-sync-client.labels" . | nindent 4 }} + name: {{ include "cluster-registry-sync-client.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + {{- include "cluster-registry-sync-client.selectorLabels" . | nindent 6 }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} +{{- end }} +{{- end }} diff --git a/charts/cluster-registry-sync-client/templates/podmonitor.yaml b/charts/cluster-registry-sync-client/templates/podmonitor.yaml new file mode 100644 index 00000000..d5a66125 --- /dev/null +++ b/charts/cluster-registry-sync-client/templates/podmonitor.yaml @@ -0,0 +1,27 @@ +{{- if .Values.podMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + labels: + {{- include "cluster-registry-sync-client.labels" . | nindent 4 }} + {{- if .Values.podMonitor.extraLabels }} + {{- toYaml .Values.podMonitor.extraLabels | nindent 4 }} + {{- end }} + name: {{ include "cluster-registry-sync-client.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + jobLabel: app + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + podMetricsEndpoints: + - interval: 60s + path: /metrics + port: metrics + - interval: 60s + path: /metrics/extra + port: metrics + selector: + matchLabels: + {{- include "cluster-registry-sync-client.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/cluster-registry-sync-client/templates/service.yaml b/charts/cluster-registry-sync-client/templates/service.yaml new file mode 100644 index 00000000..226a319a --- /dev/null +++ b/charts/cluster-registry-sync-client/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + {{- include "cluster-registry-sync-client.labels" . | nindent 4 }} + name: {{ include "cluster-registry-sync-client.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + ports: + {{- range $_, $port := .Values.ports }} + - name: {{ $port.name }} + port: {{ $port.containerPort }} + targetPort: {{ $port.name }} + {{- else }} + {{ fail "No ports defined" }} + {{- end }} + selector: + {{- include "cluster-registry-sync-client.selectorLabels" . | nindent 4 }} diff --git a/charts/cluster-registry-sync-client/templates/serviceaccount.yaml b/charts/cluster-registry-sync-client/templates/serviceaccount.yaml new file mode 100644 index 00000000..e1057cf6 --- /dev/null +++ b/charts/cluster-registry-sync-client/templates/serviceaccount.yaml @@ -0,0 +1,9 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "cluster-registry-sync-client.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cluster-registry-sync-client.labels" . | nindent 4 }} +{{- end }} diff --git a/charts/cluster-registry-sync-client/values.yaml b/charts/cluster-registry-sync-client/values.yaml new file mode 100644 index 00000000..77a81503 --- /dev/null +++ b/charts/cluster-registry-sync-client/values.yaml @@ -0,0 +1,52 @@ +nameOverride: cluster-registry-sync-client +fullnameOverride: cluster-registry-sync-client + +replicaCount: 1 + +imagePullSecrets: [] +image: + pullPolicy: IfNotPresent + registry: ghcr.io/adobe/cluster-registry-sync-client + +ports: [] + +resources: + limits: + cpu: 200m + memory: 400Mi + requests: + cpu: 100m + memory: 200Mi + +livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 20 +readinessProbe: + httpGet: + path: /readyz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 +terminationGracePeriodSeconds: 10 + +clusterRegistrySyncClient: + health: + bindAddress: :8080 + +rbac: + create: true + +serviceAccount: + create: true + name: cluster-registry-sync-client + +podDisruptionBudget: + enabled: false + minAvailable: 50% + +podMonitor: + enabled: false + extraLabels: {} diff --git a/cmd/apiserver/apiserver.go b/cmd/apiserver/apiserver.go index 34e940ca..a44ed3a9 100644 --- a/cmd/apiserver/apiserver.go +++ b/cmd/apiserver/apiserver.go @@ -72,12 +72,12 @@ func main() { Endpoint: appConfig.SqsEndpoint, QueueName: appConfig.SqsQueueName, BatchSize: appConfig.SqsBatchSize, - VisibilityTimeout: 0, + VisibilityTimeout: appConfig.SqsVisibilityTimeout, WaitSeconds: appConfig.SqsWaitSeconds, RunInterval: appConfig.SqsRunInterval, RunOnce: false, - MaxHandlers: 10, - BusyTimeout: 30, + MaxHandlers: appConfig.SqsMaxHandlers, + BusyTimeout: appConfig.SqsBusyTimeout, }) if err != nil { diff --git a/cmd/sync/client/Dockerfile b/cmd/sync/client/Dockerfile new file mode 100644 index 00000000..baabc1b2 --- /dev/null +++ b/cmd/sync/client/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine +RUN apk add --update --no-cache ca-certificates +ADD cluster-registry-sync-client /bin/sync-client +USER nobody +ENTRYPOINT ["/bin/sync-client"] diff --git a/cmd/sync/client/client.go b/cmd/sync/client/client.go index 830e0b88..33d7df4d 100644 --- a/cmd/sync/client/client.go +++ b/cmd/sync/client/client.go @@ -13,16 +13,22 @@ governing permissions and limitations under the License. package main import ( - "github.com/adobe/cluster-registry/pkg/sync/event" - "os" - + "context" + "errors" "github.com/adobe/cluster-registry/pkg/config" "github.com/adobe/cluster-registry/pkg/sqs" "github.com/adobe/cluster-registry/pkg/sync/client" + "github.com/adobe/cluster-registry/pkg/sync/event" awssqs "github.com/aws/aws-sdk-go/service/sqs" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "k8s.io/klog/v2" + "net" + "net/http" + "os" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" ) var ( @@ -34,10 +40,10 @@ var ( Run: run, } - logLevel, logFormat string - appConfig *config.AppConfig - namespace string - cfgFile string + logLevel, logFormat string + appConfig *config.AppConfig + namespace string + healthProbeBindAddress string ) func Execute() { @@ -48,14 +54,10 @@ func Execute() { } func init() { - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "The path to the yaml configuration file") rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", logrus.DebugLevel.String(), "The verbosity level of the logs, can be [panic|fatal|error|warn|info|debug|trace]") rootCmd.PersistentFlags().StringVar(&logFormat, "log-format", "text", "The output format of the logs, can be [text|json]") rootCmd.PersistentFlags().StringVar(&namespace, "namespace", "cluster-registry", "The namespace where cluster-registry-sync-client will run.") - err := rootCmd.MarkPersistentFlagRequired("config") - if err != nil { - log.Fatalln("No config flag configured") - } + rootCmd.PersistentFlags().StringVar(&healthProbeBindAddress, "health-probe-bind-address", ":8080", "The address the health probes will bind to.") } func loadAppConfig(cmd *cobra.Command, args []string) { @@ -64,7 +66,7 @@ func loadAppConfig(cmd *cobra.Command, args []string) { log.Info("Loading the configuration") var err error - appConfig, err = config.LoadSyncClientConfig() + appConfig, err = config.LoadSQSConfig() if err != nil { log.Error("Cannot load the cluster-registry-sync-client configuration: ", err.Error()) os.Exit(1) @@ -74,19 +76,19 @@ func loadAppConfig(cmd *cobra.Command, args []string) { } func run(cmd *cobra.Command, args []string) { - log.Info("Cluster Registry Sync Client is running") + ctx := signals.SetupSignalHandler() q, err := sqs.NewSQS(sqs.Config{ AWSRegion: appConfig.SqsAwsRegion, Endpoint: appConfig.SqsEndpoint, QueueName: appConfig.SqsQueueName, - BatchSize: 10, - VisibilityTimeout: 0, - WaitSeconds: 5, - RunInterval: 20, + BatchSize: appConfig.SqsBatchSize, + VisibilityTimeout: appConfig.SqsVisibilityTimeout, + WaitSeconds: appConfig.SqsWaitSeconds, + RunInterval: appConfig.SqsRunInterval, RunOnce: false, - MaxHandlers: 10, - BusyTimeout: 30, + MaxHandlers: appConfig.SqsMaxHandlers, + BusyTimeout: appConfig.SqsBusyTimeout, }) if err != nil { log.Panicf("Error while trying to create SQS client: %v", err.Error()) @@ -121,11 +123,47 @@ func run(cmd *cobra.Command, args []string) { } }) - log.Info("Starting the Cluster Registry Sync Client") + go serveHealthProbes(ctx.Done(), healthProbeBindAddress) + log.Info("Starting the Cluster Registry Sync Client") q.Poll() } +func serveHealthProbes(stop <-chan struct{}, healthProbeBindAddress string) { + healthzHandler := &healthz.Handler{Checks: map[string]healthz.Checker{ + "healthz": healthz.Ping, + }} + readyzHandler := &healthz.Handler{Checks: map[string]healthz.Checker{ + "readyz": healthz.Ping, + }} + + mux := http.NewServeMux() + mux.Handle("/readyz", http.StripPrefix("/readyz", readyzHandler)) + mux.Handle("/healthz", http.StripPrefix("/healthz", healthzHandler)) + + server := http.Server{ + Handler: mux, + } + + ln, err := net.Listen("tcp", healthProbeBindAddress) + if err != nil { + log.Errorf("error listening on %s: %v", healthProbeBindAddress, err) + return + } + + log.Infof("Health probes listening on %s", healthProbeBindAddress) + go func() { + if err := server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatal(err) + } + }() + + <-stop + if err := server.Shutdown(context.Background()); err != nil { + klog.Fatal(err) + } +} + func main() { Execute() } diff --git a/cmd/sync/manager/manager.go b/cmd/sync/manager/manager.go index d3f45969..16af9111 100644 --- a/cmd/sync/manager/manager.go +++ b/cmd/sync/manager/manager.go @@ -133,7 +133,7 @@ func main() { m := monitoring.NewMetrics() m.Init(false) - appConfig, err := config.LoadClientConfig() + appConfig, err := config.LoadSQSConfig() if err != nil { setupLog.Error(err, "failed to load client configuration") @@ -144,13 +144,13 @@ func main() { AWSRegion: appConfig.SqsAwsRegion, Endpoint: appConfig.SqsEndpoint, QueueName: appConfig.SqsQueueName, - BatchSize: 10, - VisibilityTimeout: 0, - WaitSeconds: 5, - RunInterval: 20, + BatchSize: appConfig.SqsBatchSize, + VisibilityTimeout: appConfig.SqsVisibilityTimeout, + WaitSeconds: appConfig.SqsWaitSeconds, + RunInterval: appConfig.SqsRunInterval, RunOnce: false, - MaxHandlers: 10, - BusyTimeout: 30, + MaxHandlers: appConfig.SqsMaxHandlers, + BusyTimeout: appConfig.SqsBusyTimeout, }) if err != nil { setupLog.Error(err, "cannot create SQS client") diff --git a/local/.env.local b/local/.env.local index 5455d516..291fe92d 100644 --- a/local/.env.local +++ b/local/.env.local @@ -48,4 +48,6 @@ export API_CACHE_TTL=1h export API_CACHE_REDIS_HOST="localhost:6379" export API_CACHE_REDIS_TLS_ENABLED="false" export CONTAINER_SYNC_MANAGER="cluster-registry-sync-manager" -export IMAGE_SYNC_MANAGER="ghcr.io/adobe/cluster-registry-sync-manager" \ No newline at end of file +export CONTAINER_SYNC_CLIENT="cluster-registry-sync-client" +export IMAGE_SYNC_MANAGER="ghcr.io/adobe/cluster-registry-sync-manager" +export IMAGE_SYNC_CLIENT="ghcr.io/adobe/cluster-registry-sync-client" \ No newline at end of file diff --git a/local/setup.sh b/local/setup.sh index 08b7afdb..d013876b 100755 --- a/local/setup.sh +++ b/local/setup.sh @@ -45,6 +45,7 @@ docker info -f json > /dev/null || die 'Cannot talk to the docker daemon. Ensure RUN_APISERVER="${1:-1}" RUN_CLIENT="${2:-1}" RUN_SYNC_MANAGER="${3:-1}" +RUN_SYNC_CLIENT="${4:-1}" ROOT_DIR="$(cd "$(dirname "$0")/.."; pwd)" @@ -222,4 +223,23 @@ if [[ "${RUN_SYNC_MANAGER}" == 1 ]]; then "${IMAGE_SYNC_MANAGER}":"${TAG}" || die "Failed to create $CONTAINER_SYNC_MANAGER container." fi +if [[ "${RUN_SYNC_CLIENT}" == 1 ]]; then + echo 'Running cluster-registry-sync-client' + if container_exists "${CONTAINER_SYNC_CLIENT}"; then + container_running "${CONTAINER_SYNC_CLIENT}" && { docker stop "CONTAINER_SYNC_CLIENT" || die "Failed to stop cluster-registry-sync-client container $CONTAINER_SYNC_CLIENT"; } + docker rm "${CONTAINER_SYNC_CLIENT}" || die "Failed to remove cluster-registry-sync-client container CONTAINER_SYNC_CLIENT" + fi + docker run -d \ + --name "${CONTAINER_SYNC_CLIENT}" \ + -v "${ROOT_DIR}/kubeconfig_client":/kubeconfig \ + -e AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY \ + -e KUBECONFIG=/kubeconfig \ + -e SQS_AWS_REGION \ + -e SQS_ENDPOINT=http://"${CONTAINER_SQS}":9324 \ + -e SQS_QUEUE_NAME="${SQS_QUEUE_NAME}" \ + --network "${NETWORK}" \ + "${IMAGE_SYNC_CLIENT}":"${TAG}" || die "Failed to create $CONTAINER_SYNC_CLIENT container." +fi + echo 'Local stack was set up successfully.' diff --git a/pkg/config/config.go b/pkg/config/config.go index 9e193b2f..776af47d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -38,6 +38,9 @@ type AppConfig struct { SqsBatchSize int64 SqsWaitSeconds int64 SqsRunInterval int + SqsVisibilityTimeout int64 + SqsMaxHandlers int + SqsBusyTimeout int K8sResourceId string ApiTenantId string ApiClientId string @@ -71,48 +74,6 @@ func LoadApiConfig() (*AppConfig, error) { dbIndexName := getEnv("DB_INDEX_NAME", "") - sqsEndpoint := getEnv("SQS_ENDPOINT", "") - if sqsEndpoint == "" { - return nil, fmt.Errorf("environment variable SQS_ENDPOINT is not set") - } - - sqsAwsRegion := getEnv("SQS_AWS_REGION", "") - if sqsAwsRegion == "" { - return nil, fmt.Errorf("environment variable SQS_AWS_REGION is not set") - } - - sqsQueueName := getEnv("SQS_QUEUE_NAME", "") - if sqsQueueName == "" { - return nil, fmt.Errorf("environment variable SQS_QUEUE_NAME is not set") - } - - sqsBatchSize := getEnv("SQS_BATCH_SIZE", "") - if sqsBatchSize == "" { - return nil, fmt.Errorf("environment variable SQS_BATCH_SIZE is not set") - } - sqsBatchSizeInt, err := strconv.ParseInt(sqsBatchSize, 10, 64) - if err != nil { - return nil, fmt.Errorf("error parsing SQS_BATCH_SIZE: %v", err) - } - - sqsWaitSeconds := getEnv("SQS_WAIT_SECONDS", "") - if sqsWaitSeconds == "" { - return nil, fmt.Errorf("environment variable SQS_WAIT_SECONDS is not set") - } - sqsWaitSecondsInt, err := strconv.ParseInt(sqsWaitSeconds, 10, 64) - if err != nil { - return nil, fmt.Errorf("error parsing SQS_WAIT_SECONDS: %v", err) - } - - sqsRunInterval := getEnv("SQS_RUN_INTERVAL", "") - if sqsRunInterval == "" { - return nil, fmt.Errorf("environment variable SQS_RUN_INTERVAL is not set") - } - sqsRunIntervalInt, err := strconv.Atoi(sqsRunInterval) - if err != nil { - return nil, fmt.Errorf("error parsing SQS_RUN_INTERVAL: %v", err) - } - oidcClientId := getEnv("OIDC_CLIENT_ID", "") if oidcClientId == "" { return nil, fmt.Errorf("environment variable OIDC_CLIENT_ID is not set") @@ -181,18 +142,17 @@ func LoadApiConfig() (*AppConfig, error) { return nil, fmt.Errorf("error parsing API_CACHE_REDIS_TLS_ENABLED: %v", err) } + sqsConfig, err := LoadSQSConfig() + if err != nil { + return nil, fmt.Errorf("cannot load SQS configuration: %v", err) + } + return &AppConfig{ AwsRegion: awsRegion, DbEndpoint: dbEndpoint, DbAwsRegion: dbAwsRegion, DbTableName: dbTableName, DbIndexName: dbIndexName, - SqsEndpoint: sqsEndpoint, - SqsAwsRegion: sqsAwsRegion, - SqsQueueName: sqsQueueName, - SqsBatchSize: sqsBatchSizeInt, - SqsWaitSeconds: sqsWaitSecondsInt, - SqsRunInterval: sqsRunIntervalInt, OidcClientId: oidcClientId, OidcIssuerUrl: oidcIssuerUrl, ApiRateLimiterEnabled: apiRateLimiterEnabled, @@ -206,10 +166,16 @@ func LoadApiConfig() (*AppConfig, error) { ApiCacheTTL: apiCacheTTL, ApiCacheRedisHost: apiCacheRedisHost, ApiCacheRedisTLSEnabled: apiCacheRedisTLSEnabledBool, + SqsEndpoint: sqsConfig.SqsEndpoint, + SqsAwsRegion: sqsConfig.SqsAwsRegion, + SqsQueueName: sqsConfig.SqsQueueName, + SqsBatchSize: sqsConfig.SqsBatchSize, + SqsWaitSeconds: sqsConfig.SqsWaitSeconds, + SqsRunInterval: sqsConfig.SqsRunInterval, }, nil } -func LoadClientConfig() (*AppConfig, error) { +func LoadSQSConfig() (*AppConfig, error) { sqsEndpoint := getEnv("SQS_ENDPOINT", "") if sqsEndpoint == "" { return nil, fmt.Errorf("environment variable SQS_ENDPOINT is not set") @@ -225,33 +191,70 @@ func LoadClientConfig() (*AppConfig, error) { return nil, fmt.Errorf("environment variable SQS_QUEUE_NAME is not set") } - return &AppConfig{ - SqsEndpoint: sqsEndpoint, - SqsAwsRegion: sqsAwsRegion, - SqsQueueName: sqsQueueName, - }, nil -} + sqsBatchSize := getEnv("SQS_BATCH_SIZE", "10") + if sqsBatchSize == "" { + return nil, fmt.Errorf("environment variable SQS_BATCH_SIZE is not set") + } + sqsBatchSizeInt, err := strconv.ParseInt(sqsBatchSize, 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing SQS_BATCH_SIZE: %v", err) + } -func LoadSyncClientConfig() (*AppConfig, error) { - sqsEndpoint := getEnv("SQS_ENDPOINT", "") - if sqsEndpoint == "" { - return nil, fmt.Errorf("environment variable SQS_ENDPOINT is not set") + sqsWaitSeconds := getEnv("SQS_WAIT_SECONDS", "5") + if sqsWaitSeconds == "" { + return nil, fmt.Errorf("environment variable SQS_WAIT_SECONDS is not set") + } + sqsWaitSecondsInt, err := strconv.ParseInt(sqsWaitSeconds, 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing SQS_WAIT_SECONDS: %v", err) } - sqsAwsRegion := getEnv("SQS_AWS_REGION", "") - if sqsAwsRegion == "" { - return nil, fmt.Errorf("environment variable SQS_AWS_REGION is not set") + sqsRunInterval := getEnv("SQS_RUN_INTERVAL", "20") + if sqsRunInterval == "" { + return nil, fmt.Errorf("environment variable SQS_RUN_INTERVAL is not set") + } + sqsRunIntervalInt, err := strconv.Atoi(sqsRunInterval) + if err != nil { + return nil, fmt.Errorf("error parsing SQS_RUN_INTERVAL: %v", err) } - sqsQueueName := getEnv("SQS_QUEUE_NAME", "") - if sqsQueueName == "" { - return nil, fmt.Errorf("environment variable SQS_QUEUE_NAME is not set") + sqsVisibilityTimeout := getEnv("SQS_VISIBILITY_TIMEOUT", "0") + if sqsVisibilityTimeout == "" { + return nil, fmt.Errorf("environment variable SQS_VISIBILITY_TIMEOUT is not set") + } + sqsVisibilityTimeoutInt, err := strconv.ParseInt(sqsVisibilityTimeout, 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing SQS_VISIBILITY_TIMEOUT: %v", err) + } + + sqsMaxHandlers := getEnv("SQS_MAX_HANDLERS", "10") + if sqsMaxHandlers == "" { + return nil, fmt.Errorf("environment variable SQS_MAX_HANDLERS is not set") + } + sqsMaxHandlersInt, err := strconv.Atoi(sqsMaxHandlers) + if err != nil { + return nil, fmt.Errorf("error parsing SQS_MAX_HANDLERS: %v", err) + } + + sqsBusyTimeout := getEnv("SQS_BUSY_TIMEOUT", "30") + if sqsBusyTimeout == "" { + return nil, fmt.Errorf("environment variable SQS_BUSY_TIMEOUT is not set") + } + sqsBusyTimeoutInt, err := strconv.Atoi(sqsBusyTimeout) + if err != nil { + return nil, fmt.Errorf("error parsing SQS_BUSY_TIMEOUT: %v", err) } return &AppConfig{ - SqsEndpoint: sqsEndpoint, - SqsAwsRegion: sqsAwsRegion, - SqsQueueName: sqsQueueName, + SqsEndpoint: sqsEndpoint, + SqsAwsRegion: sqsAwsRegion, + SqsQueueName: sqsQueueName, + SqsBatchSize: sqsBatchSizeInt, + SqsWaitSeconds: sqsWaitSecondsInt, + SqsRunInterval: sqsRunIntervalInt, + SqsVisibilityTimeout: sqsVisibilityTimeoutInt, + SqsMaxHandlers: sqsMaxHandlersInt, + SqsBusyTimeout: sqsBusyTimeoutInt, }, nil }