diff --git a/.gitignore b/.gitignore index 2b0c6e44..2db54606 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ go.work.sum .env bin/ +config/**/charts diff --git a/api/v1alpha/networkbinding_types.go b/api/v1alpha/networkbinding_types.go index b43e40f5..82c2bd62 100644 --- a/api/v1alpha/networkbinding_types.go +++ b/api/v1alpha/networkbinding_types.go @@ -63,12 +63,17 @@ const ( // +kubebuilder:subresource:status // NetworkBinding is the Schema for the networkbindings API +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason` type NetworkBinding struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` // +kubebuilder:validation:Required - Spec NetworkBindingSpec `json:"spec,omitempty"` + Spec NetworkBindingSpec `json:"spec,omitempty"` + + // +kubebuilder:default={conditions:{{type:"Ready",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}} Status NetworkBindingStatus `json:"status,omitempty"` } diff --git a/api/v1alpha/networkcontext_types.go b/api/v1alpha/networkcontext_types.go index f8c022a5..54d4e2c1 100644 --- a/api/v1alpha/networkcontext_types.go +++ b/api/v1alpha/networkcontext_types.go @@ -26,19 +26,41 @@ type NetworkContextStatus struct { } const ( - // NetworkContextReady indicates that the network context is ready for use. + // NetworkContextReady indicates whether or not the network context is ready for use. NetworkContextReady = "Ready" + + // NetworkContextProgrammed indicates whether or not the network context has been programmed. + NetworkContextProgrammed = "Programmed" +) + +const ( + // NetworkContextProgrammedReasonNotProgrammed indicates that the network context is not ready because it has not been programmed. + NetworkContextProgrammedReasonNotProgrammed = "NotProgrammed" + + // NetworkContextProgrammedReasonProgramming indicates that the network context is being programmed. + NetworkContextProgrammedReasonProgrammingInProgress = "ProgrammingInProgress" + + // NetworkContextProgrammedReasonProgrammed indicates that the network context has been programmed. + NetworkContextProgrammedReasonProgrammed = "Programmed" + + // NetworkContextReadyReasonReady indicates that the network context is ready for use. + NetworkContextReadyReasonReady = "Ready" ) // +kubebuilder:object:root=true // +kubebuilder:subresource:status // NetworkContext is the Schema for the networkcontexts API +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason` type NetworkContext struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec NetworkContextSpec `json:"spec,omitempty"` + Spec NetworkContextSpec `json:"spec,omitempty"` + + // +kubebuilder:default={conditions:{{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Ready",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}} Status NetworkContextStatus `json:"status,omitempty"` } diff --git a/api/v1alpha/subnet_types.go b/api/v1alpha/subnet_types.go index 5f7fb69f..6b58c140 100644 --- a/api/v1alpha/subnet_types.go +++ b/api/v1alpha/subnet_types.go @@ -51,15 +51,47 @@ type SubnetStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty"` } +const ( + // SubnetAllocated indicates that the subnet has been allocated a prefix + SubnetAllocated = "Allocated" + + // SubnetProgrammed indicates that the subnet has been programmed + SubnetProgrammed = "Programmed" + + // SubnetReady indicates that the subnet is ready to use + SubnetReady = "Ready" +) + +const ( + // SubnetProgrammedReasonNotProgrammed indicates that the subnet has not been programmed + SubnetProgrammedReasonNotProgrammed = "NotProgrammed" + + // SubnetProgrammedReasonProgrammingInProgress indicates that the subnet is being programmed. + SubnetProgrammedReasonProgrammingInProgress = "ProgrammingInProgress" + + // SubnetProgrammedReasonProgrammed indicates that the subnet has been programmed + SubnetProgrammedReasonProgrammed = "Programmed" + + // SubnetReadyReasonReady indicates that the subnet is ready to use + SubnetReadyReasonReady = "Ready" +) + // +kubebuilder:object:root=true // +kubebuilder:subresource:status // Subnet is the Schema for the subnets API +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason` +// +kubebuilder:printcolumn:name="Start Address",type=string,JSONPath=`.status.startAddress` +// +kubebuilder:printcolumn:name="Prefix Length",type=string,JSONPath=`.status.prefixLength` type Subnet struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec SubnetSpec `json:"spec,omitempty"` + Spec SubnetSpec `json:"spec,omitempty"` + + // +kubebuilder:default={conditions:{{type:"Allocated",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Ready",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}} Status SubnetStatus `json:"status,omitempty"` } diff --git a/api/v1alpha/subnetclaim_types.go b/api/v1alpha/subnetclaim_types.go index 86f25a06..a5867067 100644 --- a/api/v1alpha/subnetclaim_types.go +++ b/api/v1alpha/subnetclaim_types.go @@ -58,11 +58,16 @@ type SubnetClaimStatus struct { // +kubebuilder:subresource:status // SubnetClaim is the Schema for the subnetclaims API +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason` type SubnetClaim struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec SubnetClaimSpec `json:"spec,omitempty"` + Spec SubnetClaimSpec `json:"spec,omitempty"` + + // +kubebuilder:default={conditions:{{type:"Allocated",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Ready",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}} Status SubnetClaimStatus `json:"status,omitempty"` } diff --git a/cmd/main.go b/cmd/main.go index e06547e7..35268145 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -219,6 +219,11 @@ func main() { os.Exit(1) } + if err = controller.AddIndexers(ctx, mgr); err != nil { + setupLog.Error(err, "unable to add indexers") + os.Exit(1) + } + validationOpts := validation.GatewayValidationOptions{ ControllerName: serverConfig.Gateway.ControllerName, PermittedTLSOptions: serverConfig.Gateway.PermittedTLSOptions, diff --git a/config/crd/bases/networking.datumapis.com_networkbindings.yaml b/config/crd/bases/networking.datumapis.com_networkbindings.yaml index 6b7fe5ce..217bece5 100644 --- a/config/crd/bases/networking.datumapis.com_networkbindings.yaml +++ b/config/crd/bases/networking.datumapis.com_networkbindings.yaml @@ -14,7 +14,17 @@ spec: singular: networkbinding scope: Namespaced versions: - - name: v1alpha + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + name: v1alpha schema: openAPIV3Schema: description: NetworkBinding is the Schema for the networkbindings API @@ -72,6 +82,13 @@ spec: - network type: object status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Ready description: NetworkBindingStatus defines the observed state of NetworkBinding properties: conditions: diff --git a/config/crd/bases/networking.datumapis.com_networkcontexts.yaml b/config/crd/bases/networking.datumapis.com_networkcontexts.yaml index e66ed3c0..af2681a7 100644 --- a/config/crd/bases/networking.datumapis.com_networkcontexts.yaml +++ b/config/crd/bases/networking.datumapis.com_networkcontexts.yaml @@ -14,7 +14,17 @@ spec: singular: networkcontext scope: Namespaced versions: - - name: v1alpha + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + name: v1alpha schema: openAPIV3Schema: description: NetworkContext is the Schema for the networkcontexts API @@ -66,6 +76,18 @@ spec: - network type: object status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Ready description: NetworkContextStatus defines the observed state of NetworkContext properties: conditions: diff --git a/config/crd/bases/networking.datumapis.com_subnetclaims.yaml b/config/crd/bases/networking.datumapis.com_subnetclaims.yaml index 4b8e1c2f..eb10db0f 100644 --- a/config/crd/bases/networking.datumapis.com_subnetclaims.yaml +++ b/config/crd/bases/networking.datumapis.com_subnetclaims.yaml @@ -14,7 +14,17 @@ spec: singular: subnetclaim scope: Namespaced versions: - - name: v1alpha + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + name: v1alpha schema: openAPIV3Schema: description: SubnetClaim is the Schema for the subnetclaims API @@ -84,6 +94,23 @@ spec: - subnetClass type: object status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Allocated + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Ready description: SubnetClaimStatus defines the observed state of SubnetClaim properties: conditions: diff --git a/config/crd/bases/networking.datumapis.com_subnets.yaml b/config/crd/bases/networking.datumapis.com_subnets.yaml index 1a10b048..84434ce7 100644 --- a/config/crd/bases/networking.datumapis.com_subnets.yaml +++ b/config/crd/bases/networking.datumapis.com_subnets.yaml @@ -14,7 +14,23 @@ spec: singular: subnet scope: Namespaced versions: - - name: v1alpha + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + - jsonPath: .status.startAddress + name: Start Address + type: string + - jsonPath: .status.prefixLength + name: Prefix Length + type: string + name: v1alpha schema: openAPIV3Schema: description: Subnet is the Schema for the subnets API @@ -86,6 +102,23 @@ spec: - subnetClass type: object status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Allocated + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Ready description: SubnetStatus defines the observed state of a Subnet properties: conditions: diff --git a/config/dev/config.yaml b/config/dev/config.yaml index c13e37c7..010855e3 100644 --- a/config/dev/config.yaml +++ b/config/dev/config.yaml @@ -4,6 +4,7 @@ metricsServer: bindAddress: "0" webhookServer: + port: 9444 tls: secretRef: name: network-services-operator-webhook-server-cert diff --git a/config/dev/webhook_patch.yaml b/config/dev/webhook_patch.yaml index 90a66455..fed57dc7 100644 --- a/config/dev/webhook_patch.yaml +++ b/config/dev/webhook_patch.yaml @@ -59,7 +59,7 @@ apiVersion: builtin kind: PrefixSuffixTransformer metadata: name: hostPrefix -prefix: "https://host.docker.internal:9443" +prefix: "https://host.docker.internal:9444" fieldSpecs: - kind: ValidatingWebhookConfiguration path: webhooks/clientConfig/url diff --git a/internal/controller/indexers.go b/internal/controller/indexers.go new file mode 100644 index 00000000..1d40a5d1 --- /dev/null +++ b/internal/controller/indexers.go @@ -0,0 +1,42 @@ +package controller + +import ( + "context" + "errors" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const ( + networkContextControllerNetworkUIDIndex = "networkContextControllerNetworkUIDIndex" +) + +func AddIndexers(ctx context.Context, mgr mcmanager.Manager) error { + return errors.Join( + addNetworkContextControllerIndexers(ctx, mgr), + ) +} + +func addNetworkContextControllerIndexers(ctx context.Context, mgr mcmanager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &networkingv1alpha.NetworkContext{}, networkContextControllerNetworkUIDIndex, networkContextControllerNetworkUIDIndexFunc); err != nil { + return fmt.Errorf("failed to add network context controller indexer %q: %w", networkContextControllerNetworkUIDIndex, err) + } + + return nil +} + +func networkContextControllerNetworkUIDIndexFunc(o client.Object) []string { + + if networkRef := metav1.GetControllerOf(o); networkRef != nil { + return []string{ + string(networkRef.UID), + } + } + + return nil +} diff --git a/internal/controller/network_controller.go b/internal/controller/network_controller.go index 55e1b693..33e4db3a 100644 --- a/internal/controller/network_controller.go +++ b/internal/controller/network_controller.go @@ -4,48 +4,140 @@ package controller import ( "context" + "errors" + "fmt" + apierrors "k8s.io/apimachinery/pkg/api/errors" + kerrors "k8s.io/apimachinery/pkg/util/errors" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/finalizer" "sigs.k8s.io/controller-runtime/pkg/log" mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mccontext "sigs.k8s.io/multicluster-runtime/pkg/context" mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" ) +const networkControllerFinalizer = "networking.datumapis.com/network-controller" + // NetworkReconciler reconciles a Network object type NetworkReconciler struct { - mgr mcmanager.Manager + mgr mcmanager.Manager + finalizers finalizer.Finalizers } // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networks,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networks/status,verbs=get;update;patch // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networks/finalizers,verbs=update -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the Network object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/reconcile func (r *NetworkReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx, "cluster", req.ClusterName) + logger := log.FromContext(ctx) + + cl, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + + ctx = mccontext.WithCluster(ctx, req.ClusterName) + + var network networkingv1alpha.Network + if err := cl.GetClient().Get(ctx, req.NamespacedName, &network); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } logger.Info("reconciling network") defer logger.Info("reconcile complete") + finalizationResult, err := r.finalizers.Finalize(ctx, &network) + if err != nil { + if v, ok := err.(kerrors.Aggregate); ok && v.Is(errNetworkContextsExist) { + // Don't produce an error in this case and let the watch on network contexts + // result in another reconcile schedule. + logger.Info("network still has network contexts, waiting until removal") + return ctrl.Result{}, nil + } else { + return ctrl.Result{}, fmt.Errorf("failed to finalize: %w", err) + } + } + if finalizationResult.Updated { + if err = cl.GetClient().Update(ctx, &network); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to update based on finalization result: %w", err) + } + return ctrl.Result{}, nil + } + + if !network.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + return ctrl.Result{}, nil } +var errNetworkContextsExist = errors.New("network contexts exist") + +func (r *NetworkReconciler) Finalize(ctx context.Context, obj client.Object) (finalizer.Result, error) { + logger := log.FromContext(ctx) + logger.Info("finalizing network") + + clusterName, ok := mccontext.ClusterFrom(ctx) + if !ok { + return finalizer.Result{}, fmt.Errorf("cluster name not found in context") + } + + cl, err := r.mgr.GetCluster(ctx, clusterName) + if err != nil { + return finalizer.Result{}, err + } + + listOpts := client.MatchingFields{ + networkContextControllerNetworkUIDIndex: string(obj.GetUID()), + } + var networkContexts networkingv1alpha.NetworkContextList + if err := cl.GetClient().List(ctx, &networkContexts, listOpts); err != nil { + return finalizer.Result{}, err + } + + if len(networkContexts.Items) == 0 { + log.FromContext(ctx).Info("network contexts have been removed") + return finalizer.Result{}, nil + } + + // All deployments need to be deleted before the workload may be deleted + for _, networkContext := range networkContexts.Items { + if networkContext.DeletionTimestamp.IsZero() { + logger.Info("deleting network context", "network context", networkContext.Name) + // Deletion will result in another reconcile of the workload, where we + // will remove the finalizers. + if err := cl.GetClient().Delete(ctx, &networkContext); client.IgnoreNotFound(err) != nil { + return finalizer.Result{}, fmt.Errorf("failed deleting network context: %w", err) + } + } + } + + // Really don't like using errors for communication here. I think we'd need + // to move away from the finalizer helper to ensure we can wait on child + // resources to be gone before allowing the finalizer to be removed. + return finalizer.Result{}, errNetworkContextsExist +} + // SetupWithManager sets up the controller with the Manager. func (r *NetworkReconciler) SetupWithManager(mgr mcmanager.Manager) error { r.mgr = mgr + + r.finalizers = finalizer.NewFinalizers() + if err := r.finalizers.Register(networkControllerFinalizer, r); err != nil { + return fmt.Errorf("failed to register finalizer: %w", err) + } + return mcbuilder.ControllerManagedBy(mgr). For(&networkingv1alpha.Network{}, mcbuilder.WithEngageWithLocalCluster(false)). + Owns(&networkingv1alpha.NetworkContext{}, mcbuilder.WithEngageWithLocalCluster(false)). Named("network"). Complete(r) } diff --git a/internal/controller/networkbinding_controller.go b/internal/controller/networkbinding_controller.go index 09fbe7a0..92ea50b2 100644 --- a/internal/controller/networkbinding_controller.go +++ b/internal/controller/networkbinding_controller.go @@ -4,7 +4,9 @@ package controller import ( "context" + "errors" "fmt" + "time" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -65,14 +67,8 @@ func (r *NetworkBindingReconciler) Reconcile(ctx context.Context, req mcreconcil } defer func() { - if err != nil { - // Don't update the status if errors are encountered - return - } - statusChanged := apimeta.SetStatusCondition(&binding.Status.Conditions, readyCondition) - - if statusChanged { - err = cl.GetClient().Status().Update(ctx, &binding) + if apimeta.SetStatusCondition(&binding.Status.Conditions, readyCondition) { + err = errors.Join(err, cl.GetClient().Status().Update(ctx, &binding)) } }() @@ -131,10 +127,16 @@ func (r *NetworkBindingReconciler) Reconcile(ctx context.Context, req mcreconcil readyCondition.Reason = "NetworkContextNotReady" readyCondition.Message = "Network context is not ready." + condition := apimeta.FindStatusCondition(networkContext.Status.Conditions, networkingv1alpha.NetworkContextProgrammed) + if condition != nil && condition.Status == metav1.ConditionFalse && condition.Reason == "NetworkFailedToCreate" { + readyCondition.Reason = "NetworkFailedToCreate" + readyCondition.Message = condition.Message + } + // Choosing to requeue here instead of establishing a watch on contexts, as // once the context is created an ready, future bindings will immediately // become ready. - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: time.Second * 5}, nil } binding.Status.NetworkContextRef = &networkingv1alpha.NetworkContextRef{ diff --git a/internal/controller/networkcontext_controller.go b/internal/controller/networkcontext_controller.go index 9068b3b2..1ea294ae 100644 --- a/internal/controller/networkcontext_controller.go +++ b/internal/controller/networkcontext_controller.go @@ -4,7 +4,11 @@ package controller import ( "context" + "fmt" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" @@ -33,9 +37,41 @@ type NetworkContextReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/reconcile func (r *NetworkContextReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx, "cluster", req.ClusterName) + logger := log.FromContext(ctx, "cluster", req.ClusterName) - // TODO(user): your logic here + cl, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + + var networkContext networkingv1alpha.NetworkContext + if err := cl.GetClient().Get(ctx, req.NamespacedName, &networkContext); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + if !networkContext.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + logger.Info("reconciling network context") + defer logger.Info("reconcile complete") + + if apimeta.IsStatusConditionTrue(networkContext.Status.Conditions, networkingv1alpha.NetworkContextProgrammed) { + if apimeta.SetStatusCondition(&networkContext.Status.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkContextReady, + Status: metav1.ConditionTrue, + Reason: networkingv1alpha.NetworkContextReadyReasonReady, + ObservedGeneration: networkContext.Generation, + Message: "Network context is ready", + }) { + if err := cl.GetClient().Status().Update(ctx, &networkContext); err != nil { + return ctrl.Result{}, fmt.Errorf("failed updating network context status") + } + } + } return ctrl.Result{}, nil } diff --git a/internal/controller/subnet_controller.go b/internal/controller/subnet_controller.go index 6d728c88..23e01253 100644 --- a/internal/controller/subnet_controller.go +++ b/internal/controller/subnet_controller.go @@ -64,68 +64,102 @@ func (r *SubnetReconciler) Reconcile(ctx context.Context, req mcreconcile.Reques // TODO(jreese) finalizer work - var networkContext networkingv1alpha.NetworkContext - networkContextObjectKey := client.ObjectKey{ - Namespace: subnet.Namespace, - Name: subnet.Spec.NetworkContext.Name, - } - if err := cl.GetClient().Get(ctx, networkContextObjectKey, &networkContext); err != nil { - return ctrl.Result{}, fmt.Errorf("failed fetching network context: %w", err) - } + needsStatusUpdate := false + if subnet.Status.StartAddress == nil { + needsStatusUpdate = true + var networkContext networkingv1alpha.NetworkContext + networkContextObjectKey := client.ObjectKey{ + Namespace: subnet.Namespace, + Name: subnet.Spec.NetworkContext.Name, + } + if err := cl.GetClient().Get(ctx, networkContextObjectKey, &networkContext); err != nil { + return ctrl.Result{}, fmt.Errorf("failed fetching network context: %w", err) + } - var location networkingv1alpha.Location - locationObjectKey := client.ObjectKey{ - Namespace: networkContext.Spec.Location.Namespace, - Name: networkContext.Spec.Location.Name, - } - if err := cl.GetClient().Get(ctx, locationObjectKey, &location); err != nil { - return ctrl.Result{}, fmt.Errorf("failed fetching network context location: %w", err) - } + if !apimeta.IsStatusConditionTrue(networkContext.Status.Conditions, networkingv1alpha.NetworkContextReady) { + return ctrl.Result{}, fmt.Errorf("network context is not ready") + } - // TODO(jreese) get topology key from well known package - cityCode, ok := location.Spec.Topology["topology.datum.net/city-code"] - if !ok { - return ctrl.Result{}, fmt.Errorf("unable to find topology key: topology.datum.net/city-code") - } + var location networkingv1alpha.Location + locationObjectKey := client.ObjectKey{ + Namespace: networkContext.Spec.Location.Namespace, + Name: networkContext.Spec.Location.Name, + } + if err := cl.GetClient().Get(ctx, locationObjectKey, &location); err != nil { + return ctrl.Result{}, fmt.Errorf("failed fetching network context location: %w", err) + } - // TODO(jreese) move to proper higher level subnet allocation logic, this is - // for the rough POC! Pay attention to the subnet class, etc. - // - // GCP allocates a /20 per region. Distribution seems to be as new regions - // come online, a /20 is allocated, but there appears to be at least a /15 - // between each region's /20. For example: - // - // europe-west9 10.200.0.0/20 - // us-east5 10.202.0.0/20 - // europe-southwest1 10.204.0.0/20 - // us-south1 10.206.0.0/20 - // me-west1 10.208.0.0/20 - // - // There's a few scenarios where this isn't the case. - - var startAddress string - switch cityCode { - case "DFW": - startAddress = "10.128.0.0" - case "DLS": - startAddress = "10.130.0.0" - case "LHR": - startAddress = "10.132.0.0" - } + // TODO(jreese) get topology key from well known package + cityCode, ok := location.Spec.Topology["topology.datum.net/city-code"] + if !ok { + return ctrl.Result{}, fmt.Errorf("unable to find topology key: topology.datum.net/city-code") + } - subnet.Status.StartAddress = proto.String(startAddress) - subnet.Status.PrefixLength = proto.Int32(20) + // TODO(jreese) move to proper higher level subnet allocation logic, this is + // for the rough POC! Pay attention to the subnet class, etc. + // + // GCP allocates a /20 per region. Distribution seems to be as new regions + // come online, a /20 is allocated, but there appears to be at least a /15 + // between each region's /20. For example: + // + // europe-west9 10.200.0.0/20 + // us-east5 10.202.0.0/20 + // europe-southwest1 10.204.0.0/20 + // us-south1 10.206.0.0/20 + // me-west1 10.208.0.0/20 + // + // There's a few scenarios where this isn't the case. + + var startAddress string + switch cityCode { + case "DFW": + startAddress = "10.128.0.0" + case "DLS": + startAddress = "10.130.0.0" + case "LHR": + startAddress = "10.132.0.0" + } + + subnet.Status.StartAddress = proto.String(startAddress) + subnet.Status.PrefixLength = proto.Int32(20) + + } - apimeta.SetStatusCondition(&subnet.Status.Conditions, metav1.Condition{ - Type: "Ready", + if apimeta.SetStatusCondition(&subnet.Status.Conditions, metav1.Condition{ + Type: networkingv1alpha.SubnetAllocated, Status: metav1.ConditionTrue, Reason: "PrefixAllocated", ObservedGeneration: subnet.Generation, Message: "Subnet has been allocated a prefix", - }) + }) { + needsStatusUpdate = true + } + + subnetProgrammed := apimeta.IsStatusConditionTrue(subnet.Status.Conditions, networkingv1alpha.SubnetProgrammed) - if err := cl.GetClient().Status().Update(ctx, &subnet); err != nil { - return ctrl.Result{}, fmt.Errorf("failed updating subnet status") + readyMessage := "Subnet is not yet programmed" + readyStatus := metav1.ConditionFalse + readyReason := networkingv1alpha.SubnetProgrammedReasonNotProgrammed + if subnetProgrammed { + readyStatus = metav1.ConditionTrue + readyReason = networkingv1alpha.SubnetReadyReasonReady + readyMessage = "Subnet is ready to use" + } + + if apimeta.SetStatusCondition(&subnet.Status.Conditions, metav1.Condition{ + Type: networkingv1alpha.SubnetReady, + Status: readyStatus, + Reason: readyReason, + ObservedGeneration: subnet.Generation, + Message: readyMessage, + }) { + needsStatusUpdate = true + } + + if needsStatusUpdate { + if err := cl.GetClient().Status().Update(ctx, &subnet); err != nil { + return ctrl.Result{}, fmt.Errorf("failed updating subnet status") + } } return ctrl.Result{}, nil @@ -138,10 +172,9 @@ func (r *SubnetReconciler) SetupWithManager(mgr mcmanager.Manager) error { For(&networkingv1alpha.Subnet{}, mcbuilder.WithPredicates( predicate.NewPredicateFuncs(func(object client.Object) bool { - // Don't bother processing subnets that have been allocated and are not - // deleting + // Don't bother processing subnets that are ready o := object.(*networkingv1alpha.Subnet) - return o.Status.StartAddress == nil || !o.DeletionTimestamp.IsZero() + return !apimeta.IsStatusConditionTrue(o.Status.Conditions, networkingv1alpha.SubnetReady) }), ), mcbuilder.WithEngageWithLocalCluster(false), diff --git a/internal/providers/datum/provider.go b/internal/providers/datum/provider.go index fb804738..f23a1b1e 100644 --- a/internal/providers/datum/provider.go +++ b/internal/providers/datum/provider.go @@ -87,6 +87,12 @@ func New(localMgr manager.Manager, opts Options) (*Provider, error) { return p, nil } +type index struct { + object client.Object + field string + extractValue client.IndexerFunc +} + // Provider is a cluster Provider that works with Datum type Provider struct { opts Options @@ -98,6 +104,7 @@ type Provider struct { mcMgr mcmanager.Manager projects map[string]cluster.Cluster cancelFns map[string]context.CancelFunc + indexers []index } // Get returns the cluster with the given name, if it is known. @@ -206,11 +213,11 @@ func (p *Provider) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result if err != nil { return ctrl.Result{}, fmt.Errorf("failed to create cluster: %w", err) } - // for _, idx := range p.indexers { - // if err := cl.GetCache().IndexField(ctx, idx.object, idx.field, idx.extractValue); err != nil { - // return ctrl.Result{}, fmt.Errorf("failed to index field %q: %w", idx.field, err) - // } - // } + for _, idx := range p.indexers { + if err := cl.GetCache().IndexField(ctx, idx.object, idx.field, idx.extractValue); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to index field %q: %w", idx.field, err) + } + } clusterCtx, cancel := context.WithCancel(ctx) go func() { @@ -243,7 +250,22 @@ func (p *Provider) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result } func (p *Provider) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { - // TODO(jreese) + p.lock.Lock() + defer p.lock.Unlock() + + // save for future projects. + p.indexers = append(p.indexers, index{ + object: obj, + field: field, + extractValue: extractValue, + }) + + // apply to existing projects. + for name, cl := range p.projects { + if err := cl.GetCache().IndexField(ctx, obj, field, extractValue); err != nil { + return fmt.Errorf("failed to index field %q on project %q: %w", field, name, err) + } + } return nil }