diff --git a/controller.go b/controller.go index 704bb85..17faa32 100644 --- a/controller.go +++ b/controller.go @@ -24,6 +24,7 @@ import ( "github.com/SneaksAndData/nexus-core/pkg/generated/clientset/versioned/scheme" "github.com/SneaksAndData/nexus-core/pkg/shards" "github.com/SneaksAndData/nexus-core/pkg/telemetry" + "github.com/SneaksAndData/nexus-core/pkg/util" "golang.org/x/time/rate" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -133,6 +134,118 @@ func (c *Controller) enqueueMachineLearningAlgorithm(obj interface{}) { } } +func (c *Controller) handleDereferencedSecrets(mla *v1.MachineLearningAlgorithm, dereferencedSecretNames []string) { + logger := klog.FromContext(context.Background()) + if len(dereferencedSecretNames) > 0 { + // first remove owner references in all clusters + // handle controller cluster first + // in the controller cluster we only **remove** references, but do not delete the referenced resources + for _, secretName := range dereferencedSecretNames { + secret, err := c.controllerkubeclientset.CoreV1().Secrets(mla.Namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + // dereferenced secret does not exist - report warning to logger only and continue to next one + if err != nil { + logger.V(3).Info("Dereferenced resource no longer exists", "resource", secretName) + continue + } + _, err = util.RemoveOwner[corev1.Secret](context.TODO(), secret, mla.UID, c.controllerkubeclientset, FieldManager) + if err != nil { + // if owner removal fails, handle error and continue to the next one + utilruntime.HandleErrorWithContext(context.Background(), nil, "Unable to update owners on the dereferenced secret", "secret", secretName) + continue + } + // now do the same in all shards + for _, shard := range c.nexusShards { + // remove owner reference first + shardMla, err := shard.MlaLister.MachineLearningAlgorithms(mla.Namespace).Get(mla.Name) + // if shardMla is missing, skip processing + if k8serrors.IsNotFound(err) { + continue + } + // on a different error, handle it and return + if err != nil { + utilruntime.HandleErrorWithContext(context.Background(), nil, "Error retrieving a shard algorithm resource", "algorithm", mla.Name) + return + } + shardSecret, err := shard.SecretLister.Secrets(mla.Namespace).Get(secretName) + // if a secret is gone already, simply continue + if k8serrors.IsNotFound(err) { + continue + } + // otherwise, handle and return + if err != nil { + utilruntime.HandleErrorWithContext(context.Background(), nil, "Error retrieving a secret referenced by a shard algorithm resource", "secret", secretName) + return + } + + err = shard.DereferenceSecret(shardSecret, shardMla, FieldManager) + // if unable to dereference, handle and return + if err != nil { + utilruntime.HandleErrorWithContext(context.Background(), nil, "Error dereferencing a secret previously referenced by a shard algorithm resource", "secret", secretName, "algorithm", mla.Name) + return + } + } + } + } + + return +} + +func (c *Controller) handleDereferencedConfigMaps(mla *v1.MachineLearningAlgorithm, dereferencedConfigMapNames []string) { + logger := klog.FromContext(context.Background()) + if len(dereferencedConfigMapNames) > 0 { + // first remove owner references in all clusters + // handle controller cluster first + // in the controller cluster we only **remove** references, but do not delete the referenced resources + for _, configMapName := range dereferencedConfigMapNames { + configMap, err := c.controllerkubeclientset.CoreV1().ConfigMaps(mla.Namespace).Get(context.TODO(), configMapName, metav1.GetOptions{}) + // dereferenced configMap does not exist - report warning to logger only and continue to next one + if err != nil { + logger.V(3).Info("Dereferenced resource no longer exists", "resource", configMapName) + continue + } + _, err = util.RemoveOwner[corev1.ConfigMap](context.TODO(), configMap, mla.UID, c.controllerkubeclientset, FieldManager) + if err != nil { + // if owner removal fails, handle error and continue to the next one + utilruntime.HandleErrorWithContext(context.Background(), nil, "Unable to update owners on the dereferenced configMap", "configMap", configMapName) + continue + } + // now do the same in all shards + for _, shard := range c.nexusShards { + // remove owner reference first + shardMla, err := shard.MlaLister.MachineLearningAlgorithms(mla.Namespace).Get(mla.Name) + // if shardMla is missing, skip processing + if k8serrors.IsNotFound(err) { + continue + } + // on a different error, handle it and return + if err != nil { + utilruntime.HandleErrorWithContext(context.Background(), nil, "Error retrieving a shard algorithm resource", "algorithm", mla.Name) + return + } + shardConfigMap, err := shard.ConfigMapLister.ConfigMaps(mla.Namespace).Get(configMapName) + // if a configMap is gone already, simply continue + if k8serrors.IsNotFound(err) { + continue + } + // otherwise, handle and return + if err != nil { + utilruntime.HandleErrorWithContext(context.Background(), nil, "Error retrieving a configMap referenced by a shard algorithm resource", "configMap", configMapName) + return + } + + err = shard.DereferenceConfigMap(shardConfigMap, shardMla, FieldManager) + // if unable to dereference, handle and return + if err != nil { + utilruntime.HandleErrorWithContext(context.Background(), nil, "Error dereferencing a configMap previously referenced by a shard algorithm resource", "configMap", configMapName, "algorithm", mla.Name) + return + } + } + } + } + + return +} + // handleObject will take any resource implementing metav1.Object and attempt // to find the MachineLearningAlgorithm resource that 'owns' it. It does this by looking at the // objects metadata.ownerReferences field for an appropriate OwnerReference. @@ -252,6 +365,17 @@ func NewController( _, handlerErr := controllerMlaInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.enqueueMachineLearningAlgorithm, UpdateFunc: func(old, new interface{}) { + oldMla := old.(*v1.MachineLearningAlgorithm) + newMla := new.(*v1.MachineLearningAlgorithm) + if newMla.ResourceVersion == oldMla.ResourceVersion { + return + } + + // handle resource dereferencing + controller.handleDereferencedSecrets(newMla, oldMla.GetSecretDiff(newMla)) + controller.handleDereferencedConfigMaps(newMla, oldMla.GetConfigmapDiff(newMla)) + + // engqueue algorithm for update controller.enqueueMachineLearningAlgorithm(new) }, DeleteFunc: controller.handleObject, diff --git a/go.mod b/go.mod index 86d9b97..a92332d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.23.1 require ( github.com/DataDog/datadog-go/v5 v5.5.0 - github.com/SneaksAndData/nexus-core v0.2.0 + github.com/SneaksAndData/nexus-core v0.2.1 golang.org/x/time v0.8.0 k8s.io/api v0.31.3 k8s.io/apimachinery v0.31.3 diff --git a/go.sum b/go.sum index f684cee..e2e379d 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/SneaksAndData/nexus-core v0.2.0 h1:qysiHyGV3Xj8ARkd6GByLO+Uq4WguZVG24lyfENQY2Q= -github.com/SneaksAndData/nexus-core v0.2.0/go.mod h1:S2W7r6/PmugzpF86eiUjVMwM/DveR2gOhxO1wwhImTQ= +github.com/SneaksAndData/nexus-core v0.2.1 h1:1r18qVl/kddXSKdrWefFKi/y0jdAa/ZeKhN2DYJFXPI= +github.com/SneaksAndData/nexus-core v0.2.1/go.mod h1:S2W7r6/PmugzpF86eiUjVMwM/DveR2gOhxO1wwhImTQ= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=