From 69c65e15c00df982c71886968c1298ff1b33f823 Mon Sep 17 00:00:00 2001 From: Joshua Reese Date: Tue, 20 May 2025 19:13:37 -0500 Subject: [PATCH] feat: Direct Instance Lifecycle Control Instance control: Introduced an instance control library, with a stateful implementation inspired by Kubernetes StatefulSets. The implementation does not consider adoption of instances, which are always owned by a WorkloadDeployment. This library returns a set of actions for the caller to consider and execute. It is possible for actions to be returned that will not be executed, but can be useful in troubleshooting or bringing visibility into the progress of a rollout. This design also opens the door for allowing bursting or parallel application of instance changes. Instance control is managed at the context of a WorkloadDeployment, with actions being built up by the instance control strategy chosen, and executed by the reconciler. Once an instance has no scheduling gates, the provider responsible for the location which it is attached to will be responsible for for provisioning the instance and updating the `Programmed` and `Running` conditions with success or failure information. Once an Instance is seen as both Programmed and Running, it will be considered Ready. Scheduling gates: An instance can have one or more scheduling gates added to its spec, which are used to indicate to other systems that the instance may not be at a state to be processed. The initial scheduling gate of `Network` results in instances not being processed until the network they are attached to is ready. Scheduling gates for quota claims, volumes, or other dependencies should be expected to come in future revisions. CRD improvements: - Refined default conditions and printer columns for Workloads, WorkloadDeployments, and Instances. - Added many more condition reasons to communicate the state of resources. - Added ReadyReplicas to Workloads and WorkloadDeployments. Network readiness: The operator will now ensure that networks being attached to are ready for use, and at least one subnet has been issued to any network context with an attached instance. Subnet issuance occurs by creating a SubnetClaim, and waiting for it to be issued a prefix by controllers in network-services-operator. General improvements: - Improved garbage collection logic. - Propagation of errors from related resources to the instance, such as network creation failures. --- api/v1alpha/instance_types.go | 99 ++++- api/v1alpha/labels.go | 2 + api/v1alpha/workload_types.go | 45 +- api/v1alpha/workloaddeployment_types.go | 18 +- api/v1alpha/zz_generated.deepcopy.go | 65 +++ cmd/main.go | 5 + .../compute.datumapis.com_instances.yaml | 85 +++- ...ute.datumapis.com_workloaddeployments.yaml | 91 +++- .../compute.datumapis.com_workloads.yaml | 100 ++++- go.mod | 75 ++-- go.sum | 147 ++++--- internal/controller/indexers.go | 87 ++++ internal/controller/instance_controller.go | 159 +++++-- .../controller/instance_controller_test.go | 359 +++++++++------ .../instancecontrol/controller_utils.go | 30 ++ .../instancecontrol/instancecontrol.go | 106 +++++ .../instancecontrol/scheduling_gates.go | 11 + .../stateful/stateful_control.go | 159 +++++++ .../stateful/stateful_control_test.go | 233 ++++++++++ .../stateful/stateful_control_util.go | 46 ++ .../stateful/stateful_control_util_test.go | 74 ++++ internal/controller/workload_controller.go | 106 +++-- .../workloaddeployment_controller.go | 413 +++++++++++++++++- internal/providers/datum/provider_test.go | 2 - internal/webhook/v1alpha/workload_webhook.go | 11 +- 25 files changed, 2144 insertions(+), 384 deletions(-) create mode 100644 internal/controller/indexers.go create mode 100644 internal/controller/instancecontrol/controller_utils.go create mode 100644 internal/controller/instancecontrol/instancecontrol.go create mode 100644 internal/controller/instancecontrol/scheduling_gates.go create mode 100644 internal/controller/instancecontrol/stateful/stateful_control.go create mode 100644 internal/controller/instancecontrol/stateful/stateful_control_test.go create mode 100644 internal/controller/instancecontrol/stateful/stateful_control_util.go create mode 100644 internal/controller/instancecontrol/stateful/stateful_control_util_test.go diff --git a/api/v1alpha/instance_types.go b/api/v1alpha/instance_types.go index 00ecda18..3d9f38ea 100644 --- a/api/v1alpha/instance_types.go +++ b/api/v1alpha/instance_types.go @@ -26,8 +26,37 @@ type InstanceSpec struct { // +kubebuilder:validation:Optional // +listType=map // +listMapKey=name - Volumes []InstanceVolume `json:"volumes,omitempty"` + + // The location which the instance has been scheduled to + // + // +kubebuilder:validation:Optional + Location *networkingv1alpha.LocationReference `json:"location,omitempty"` + + // Controller contains settings driven by the controller managing the instance. + // + // +kubebuilder:validation:Optional + Controller *InstanceController `json:"controller,omitempty"` +} + +type InstanceController struct { + // TemplateHash is the hash of the instance template applied for this instance. + // + // +kubebuilder:validation:Required + TemplateHash string `json:"templateHash"` + + // SchedulingGates is a list of gates that must be satisfied before the + // instance can be scheduled. + // + // +kubebuilder:validation:Optional + // +listType=map + // +listMapKey=name + SchedulingGates []SchedulingGate `json:"schedulingGates,omitempty"` +} + +type SchedulingGate struct { + // The name of the gate. + Name string `json:"name"` } type InstanceRuntimeSpec struct { @@ -337,8 +366,60 @@ type InstanceStatus struct { // Network interface information NetworkInterfaces []InstanceNetworkInterfaceStatus `json:"networkInterfaces,omitempty"` + + // Controller contains status information about the controller managing the instance. + // + // +kubebuilder:validation:Optional + Controller *InstanceControllerStatus `json:"controller,omitempty"` } +type InstanceControllerStatus struct { + // ObservedTemplateHash is the hash of the instance template applied for this instance. + // + // +kubebuilder:validation:Required + ObservedTemplateHash string `json:"observedTemplateHash"` +} + +const ( + // InstanceReady indicates that the instance is ready + InstanceReady = "Ready" + + // InstanceRunning indicates that the instance is running + InstanceRunning = "Running" + + // InstanceProgrammed indicates that the instance has been programmed + InstanceProgrammed = "Programmed" +) + +const ( + // InstanceReadyReasonSchedulingGatesPresent indicates that the instance is not ready because scheduling gates are present. + InstanceReadyReasonSchedulingGatesPresent = "SchedulingGatesPresent" + + // InstanceReadyReasonRunning indicates that the instance is running + InstanceReadyReasonRunning = "Running" + + // InstanceRunningReasonStopped indicates that the instance is stopped + InstanceRunningReasonStopped = "Stopped" + + // InstanceRunningReasonStarting indicates that the instance is starting + InstanceRunningReasonStarting = "Starting" + + // InstanceRunningReasonStopping indicates that the instance is stopping + InstanceRunningReasonStopping = "Stopping" + + // InstanceRunningReasonRunning indicates that the instance is running + InstanceRunningReasonRunning = "Running" + + // InstanceProgrammedReasonPendingProgramming indicates that the instance has not been programmed + InstanceProgrammedReasonPendingProgramming = "PendingProgramming" + + // InstanceProgrammedReasonProgrammingInProgress indicates that the instance is being programmed. + InstanceProgrammedReasonProgrammingInProgress = "ProgrammingInProgress" + + // InstanceProgrammedReasonProgrammed indicates that the instance has been programmed + InstanceProgrammedReasonProgrammed = "Programmed" +) + type InstanceTemplateSpec struct { // Metadata of the instances created from this template // @@ -355,15 +436,21 @@ type InstanceTemplateSpec struct { // Instance is the Schema for the instances API // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" -// +kubebuilder:printcolumn:name="Available",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].status` -// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].reason` -// +kubebuilder:printcolumn:name="Network IP",type=string,JSONPath=`.status.networkInterfaces[0].assignments.networkIP`,priority=1 -// +kubebuilder:printcolumn:name="External IP",type=string,JSONPath=`.status.networkInterfaces[0].assignments.externalIP`,priority=1 +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason` +// +kubebuilder:printcolumn:name="Network IP",type=string,JSONPath=`.status.networkInterfaces[0].assignments.networkIP` +// +kubebuilder:printcolumn:name="External IP",type=string,JSONPath=`.status.networkInterfaces[0].assignments.externalIP` +// +kubebuilder:printcolumn:name="Message",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].message`,priority=1 type Instance struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec InstanceSpec `json:"spec,omitempty"` + // Spec defines the desired state of an Instance. + Spec InstanceSpec `json:"spec,omitempty"` + + // Status defines the current state of an Instance. + // + // +kubebuilder:default={conditions:{{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Running",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Ready",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}} Status InstanceStatus `json:"status,omitempty"` } diff --git a/api/v1alpha/labels.go b/api/v1alpha/labels.go index 9d308927..e1dac308 100644 --- a/api/v1alpha/labels.go +++ b/api/v1alpha/labels.go @@ -5,4 +5,6 @@ const ( WorkloadUIDLabel = LabelNamespace + "/workload-uid" WorkloadDeploymentUIDLabel = LabelNamespace + "/workload-deployment-uid" + + InstanceIndexLabel = LabelNamespace + "/instance-index" ) diff --git a/api/v1alpha/workload_types.go b/api/v1alpha/workload_types.go index 2156037c..da34a1c9 100644 --- a/api/v1alpha/workload_types.go +++ b/api/v1alpha/workload_types.go @@ -52,17 +52,20 @@ type WorkloadStatus struct { // Known condition types are: "Available", "Progressing" Conditions []metav1.Condition `json:"conditions,omitempty"` - // The number of instances created by a placement + // The number of deployments that currently exist + Deployments int32 `json:"deployments"` + + // The number of instances that currently exist Replicas int32 `json:"replicas"` - // The number of instances created by a placement and have the latest - // workload generation settings applied. + // The number of instances which have the latest workload settings applied. CurrentReplicas int32 `json:"currentReplicas"` - // The desired number of instances to be managed by a placement. + // The desired number of instances DesiredReplicas int32 `json:"desiredReplicas"` - // TODO(jreese) ReadyReplicas? + // The number of instances which are ready. + ReadyReplicas int32 `json:"readyReplicas"` // The current status of placemetns in a workload. Placements []WorkloadPlacementStatus `json:"placements,omitempty"` @@ -91,6 +94,11 @@ type WorkloadGatewayStatus struct { // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // +kubebuilder:printcolumn:name="Available",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].status` // +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].reason` +// +kubebuilder:printcolumn:name="Deployments",type=string,JSONPath=`.status.deployments` +// +kubebuilder:printcolumn:name="Replicas",type=string,JSONPath=`.status.replicas` +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.readyReplicas` +// +kubebuilder:printcolumn:name="Desired",type=string,JSONPath=`.status.desiredReplicas` +// +kubebuilder:printcolumn:name="Up-to-date",type=string,JSONPath=`.status.currentReplicas` type Workload struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -134,17 +142,17 @@ type WorkloadPlacementStatus struct { // Known condition types are: "Available", "Progressing" Conditions []metav1.Condition `json:"conditions,omitempty"` - // The number of instances created by a placement + // The number of instances that currently exist Replicas int32 `json:"replicas"` - // The number of instances created by a placement and have the latest - // workload generation settings applied. + // The number of instances which have the latest workload settings applied. CurrentReplicas int32 `json:"currentReplicas"` - // The desired number of instances to be managed by a placement. + // The desired number of instances DesiredReplicas int32 `json:"desiredReplicas"` - // TODO(jreese) ReadyReplicas? + // The number of instances which are ready. + ReadyReplicas int32 `json:"readyReplicas"` } type HorizontalScaleSettings struct { @@ -165,8 +173,25 @@ type HorizontalScaleSettings struct { // TODO(jreese) wire in behavior // See https://github.com/kubernetes/kubernetes/blob/dd87bc064631354885193fc1a97d0e7b603e77b4/staging/src/k8s.io/api/autoscaling/v2/types.go#L84 + // Defines the policy for managing instances. + + // TODO(jreese) Add instance update policy? RollingUpdate vs OrderedReady + + // Controls how instances are managed during scale up and down, as well as + // during maintenance events. + // + // +kubebuilder:validation:Required + // +kubebuilder:default=OrderedReady + InstanceManagementPolicy InstanceManagementPolicyType `json:"instanceManagementPolicy,omitempty"` } +type InstanceManagementPolicyType string + +const ( + OrderedReadyInstanceManagementPolicyType InstanceManagementPolicyType = "OrderedReady" + // ParallelInstanceManagementPolicyType InstanceManagementPolicyType = "Parallel" +) + type MetricSpec struct { // Resource metrics known to Datum. // diff --git a/api/v1alpha/workloaddeployment_types.go b/api/v1alpha/workloaddeployment_types.go index 7cf8a0d4..76b6179e 100644 --- a/api/v1alpha/workloaddeployment_types.go +++ b/api/v1alpha/workloaddeployment_types.go @@ -46,17 +46,17 @@ type WorkloadDeploymentStatus struct { // Known condition types are: "Available", "Progressing" Conditions []metav1.Condition `json:"conditions,omitempty"` - // The number of instances created by a deployment + // The number of instances created Replicas int32 `json:"replicas"` - // The number of instances created by a deployment and have the latest - // deployment generation settings applied. + // The number of instances which have the latest workload settings applied. CurrentReplicas int32 `json:"currentReplicas"` - // The desired number of instances to be managed by a deployment. + // The desired number of instances DesiredReplicas int32 `json:"desiredReplicas"` - // TODO(jreese) ReadyReplicas? + // The number of instances which are ready. + ReadyReplicas int32 `json:"readyReplicas"` } const ( @@ -70,10 +70,14 @@ const ( // WorkloadDeployment is the Schema for the workloaddeployments API // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" -// +kubebuilder:printcolumn:name="Location Namespce",type=string,JSONPath=`.status.location.namespace` -// +kubebuilder:printcolumn:name="Location Name",type=string,JSONPath=`.status.location.name` // +kubebuilder:printcolumn:name="Available",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].status` // +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].reason` +// +kubebuilder:printcolumn:name="Replicas",type=string,JSONPath=`.status.replicas` +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.readyReplicas` +// +kubebuilder:printcolumn:name="Desired",type=string,JSONPath=`.status.desiredReplicas` +// +kubebuilder:printcolumn:name="Up-to-date",type=string,JSONPath=`.status.currentReplicas` +// +kubebuilder:printcolumn:name="Location Namespace",type=string,JSONPath=`.status.location.namespace`,priority=1 +// +kubebuilder:printcolumn:name="Location Name",type=string,JSONPath=`.status.location.name`,priority=1 type WorkloadDeployment struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index ffbf3a27..8ecc1bae 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -241,6 +241,41 @@ func (in *Instance) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InstanceController) DeepCopyInto(out *InstanceController) { + *out = *in + if in.SchedulingGates != nil { + in, out := &in.SchedulingGates, &out.SchedulingGates + *out = make([]SchedulingGate, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceController. +func (in *InstanceController) DeepCopy() *InstanceController { + if in == nil { + return nil + } + out := new(InstanceController) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InstanceControllerStatus) DeepCopyInto(out *InstanceControllerStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceControllerStatus. +func (in *InstanceControllerStatus) DeepCopy() *InstanceControllerStatus { + if in == nil { + return nil + } + out := new(InstanceControllerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InstanceList) DeepCopyInto(out *InstanceList) { *out = *in @@ -423,6 +458,16 @@ func (in *InstanceSpec) DeepCopyInto(out *InstanceSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Location != nil { + in, out := &in.Location, &out.Location + *out = new(apiv1alpha.LocationReference) + **out = **in + } + if in.Controller != nil { + in, out := &in.Controller, &out.Controller + *out = new(InstanceController) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceSpec. @@ -452,6 +497,11 @@ func (in *InstanceStatus) DeepCopyInto(out *InstanceStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Controller != nil { + in, out := &in.Controller, &out.Controller + *out = new(InstanceControllerStatus) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceStatus. @@ -666,6 +716,21 @@ func (in *SandboxRuntime) DeepCopy() *SandboxRuntime { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulingGate) DeepCopyInto(out *SchedulingGate) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulingGate. +func (in *SchedulingGate) DeepCopy() *SchedulingGate { + if in == nil { + return nil + } + out := new(SchedulingGate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualMachineRuntime) DeepCopyInto(out *VirtualMachineRuntime) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index acb5d6b6..11bc9de5 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -180,6 +180,11 @@ func main() { // +kubebuilder:scaffold:builder + if err = controller.AddIndexers(ctx, mgr); err != nil { + setupLog.Error(err, "unable to add indexers") + os.Exit(1) + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/config/crd/bases/compute.datumapis.com_instances.yaml b/config/crd/bases/compute.datumapis.com_instances.yaml index 0115f0d6..07a61f37 100644 --- a/config/crd/bases/compute.datumapis.com_instances.yaml +++ b/config/crd/bases/compute.datumapis.com_instances.yaml @@ -18,18 +18,20 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date - - jsonPath: .status.conditions[?(@.type=="Available")].status - name: Available + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready type: string - - jsonPath: .status.conditions[?(@.type=="Available")].reason + - jsonPath: .status.conditions[?(@.type=="Ready")].reason name: Reason type: string - jsonPath: .status.networkInterfaces[0].assignments.networkIP name: Network IP - priority: 1 type: string - jsonPath: .status.networkInterfaces[0].assignments.externalIP name: External IP + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Message priority: 1 type: string name: v1alpha @@ -55,8 +57,48 @@ spec: metadata: type: object spec: - description: InstanceSpec defines the desired state of Instance + description: Spec defines the desired state of an Instance. properties: + controller: + description: Controller contains settings driven by the controller + managing the instance. + properties: + schedulingGates: + description: |- + SchedulingGates is a list of gates that must be satisfied before the + instance can be scheduled. + items: + properties: + name: + description: The name of the gate. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + templateHash: + description: TemplateHash is the hash of the instance template + applied for this instance. + type: string + required: + - templateHash + type: object + location: + description: The location which the instance has been scheduled to + properties: + name: + description: Name of a datum location + type: string + namespace: + description: Namespace for the datum location + type: string + required: + - name + - namespace + type: object networkInterfaces: description: Network interface configuration. items: @@ -519,6 +561,9 @@ spec: - resources type: object volumes: + description: |- + Volumes that must be available to attach to an instance's containers or + Virtual Machine. items: properties: configMap: @@ -766,7 +811,24 @@ spec: - runtime type: object status: - description: InstanceStatus defines the observed state of Instance + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Running + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Ready + description: Status defines the current state of an Instance. properties: conditions: description: |- @@ -827,6 +889,17 @@ spec: - type type: object type: array + controller: + description: Controller contains status information about the controller + managing the instance. + properties: + observedTemplateHash: + description: ObservedTemplateHash is the hash of the instance + template applied for this instance. + type: string + required: + - observedTemplateHash + type: object networkInterfaces: description: Network interface information items: diff --git a/config/crd/bases/compute.datumapis.com_workloaddeployments.yaml b/config/crd/bases/compute.datumapis.com_workloaddeployments.yaml index f42d8489..a13a40e5 100644 --- a/config/crd/bases/compute.datumapis.com_workloaddeployments.yaml +++ b/config/crd/bases/compute.datumapis.com_workloaddeployments.yaml @@ -18,18 +18,32 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date - - jsonPath: .status.location.namespace - name: Location Namespce - type: string - - jsonPath: .status.location.name - name: Location Name - type: string - jsonPath: .status.conditions[?(@.type=="Available")].status name: Available type: string - jsonPath: .status.conditions[?(@.type=="Available")].reason name: Reason type: string + - jsonPath: .status.replicas + name: Replicas + type: string + - jsonPath: .status.readyReplicas + name: Ready + type: string + - jsonPath: .status.desiredReplicas + name: Desired + type: string + - jsonPath: .status.currentReplicas + name: Up-to-date + type: string + - jsonPath: .status.location.namespace + name: Location Namespace + priority: 1 + type: string + - jsonPath: .status.location.name + name: Location Name + priority: 1 + type: string name: v1alpha schema: openAPIV3Schema: @@ -66,6 +80,12 @@ spec: scaleSettings: description: Scale settings such as minimum and maximum replica counts. properties: + instanceManagementPolicy: + default: OrderedReady + description: |- + Controls how instances are managed during scale up and down, as well as + during maintenance events. + type: string maxReplicas: description: The maximum number of replicas. format: int32 @@ -120,6 +140,7 @@ spec: format: int32 type: integer required: + - instanceManagementPolicy - minReplicas type: object template: @@ -148,6 +169,47 @@ spec: spec: description: Describes the desired configuration of an instance properties: + controller: + description: Controller contains settings driven by the controller + managing the instance. + properties: + schedulingGates: + description: |- + SchedulingGates is a list of gates that must be satisfied before the + instance can be scheduled. + items: + properties: + name: + description: The name of the gate. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + templateHash: + description: TemplateHash is the hash of the instance + template applied for this instance. + type: string + required: + - templateHash + type: object + location: + description: The location which the instance has been scheduled + to + properties: + name: + description: Name of a datum location + type: string + namespace: + description: Namespace for the datum location + type: string + required: + - name + - namespace + type: object networkInterfaces: description: Network interface configuration. items: @@ -621,6 +683,9 @@ spec: - resources type: object volumes: + description: |- + Volumes that must be available to attach to an instance's containers or + Virtual Machine. items: properties: configMap: @@ -961,13 +1026,12 @@ spec: type: object type: array currentReplicas: - description: |- - The number of instances created by a deployment and have the latest - deployment generation settings applied. + description: The number of instances which have the latest workload + settings applied. format: int32 type: integer desiredReplicas: - description: The desired number of instances to be managed by a deployment. + description: The desired number of instances format: int32 type: integer location: @@ -984,13 +1048,18 @@ spec: - name - namespace type: object + readyReplicas: + description: The number of instances which are ready. + format: int32 + type: integer replicas: - description: The number of instances created by a deployment + description: The number of instances created format: int32 type: integer required: - currentReplicas - desiredReplicas + - readyReplicas - replicas type: object type: object diff --git a/config/crd/bases/compute.datumapis.com_workloads.yaml b/config/crd/bases/compute.datumapis.com_workloads.yaml index eadbf100..6c7082cc 100644 --- a/config/crd/bases/compute.datumapis.com_workloads.yaml +++ b/config/crd/bases/compute.datumapis.com_workloads.yaml @@ -24,6 +24,21 @@ spec: - jsonPath: .status.conditions[?(@.type=="Available")].reason name: Reason type: string + - jsonPath: .status.deployments + name: Deployments + type: string + - jsonPath: .status.replicas + name: Replicas + type: string + - jsonPath: .status.readyReplicas + name: Ready + type: string + - jsonPath: .status.desiredReplicas + name: Desired + type: string + - jsonPath: .status.currentReplicas + name: Up-to-date + type: string name: v1alpha schema: openAPIV3Schema: @@ -68,6 +83,12 @@ spec: description: Scale settings such as minimum and maximum replica counts. properties: + instanceManagementPolicy: + default: OrderedReady + description: |- + Controls how instances are managed during scale up and down, as well as + during maintenance events. + type: string maxReplicas: description: The maximum number of replicas. format: int32 @@ -122,6 +143,7 @@ spec: format: int32 type: integer required: + - instanceManagementPolicy - minReplicas type: object required: @@ -157,6 +179,47 @@ spec: spec: description: Describes the desired configuration of an instance properties: + controller: + description: Controller contains settings driven by the controller + managing the instance. + properties: + schedulingGates: + description: |- + SchedulingGates is a list of gates that must be satisfied before the + instance can be scheduled. + items: + properties: + name: + description: The name of the gate. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + templateHash: + description: TemplateHash is the hash of the instance + template applied for this instance. + type: string + required: + - templateHash + type: object + location: + description: The location which the instance has been scheduled + to + properties: + name: + description: Name of a datum location + type: string + namespace: + description: Namespace for the datum location + type: string + required: + - name + - namespace + type: object networkInterfaces: description: Network interface configuration. items: @@ -630,6 +693,9 @@ spec: - resources type: object volumes: + description: |- + Volumes that must be available to attach to an instance's containers or + Virtual Machine. items: properties: configMap: @@ -954,13 +1020,16 @@ spec: type: object type: array currentReplicas: - description: |- - The number of instances created by a placement and have the latest - workload generation settings applied. + description: The number of instances which have the latest workload + settings applied. + format: int32 + type: integer + deployments: + description: The number of deployments that currently exist format: int32 type: integer desiredReplicas: - description: The desired number of instances to be managed by a placement. + description: The desired number of instances format: int32 type: integer gateway: @@ -1302,37 +1371,46 @@ spec: type: object type: array currentReplicas: - description: |- - The number of instances created by a placement and have the latest - workload generation settings applied. + description: The number of instances which have the latest workload + settings applied. format: int32 type: integer desiredReplicas: - description: The desired number of instances to be managed by - a placement. + description: The desired number of instances format: int32 type: integer name: description: The name of the placement type: string + readyReplicas: + description: The number of instances which are ready. + format: int32 + type: integer replicas: - description: The number of instances created by a placement + description: The number of instances that currently exist format: int32 type: integer required: - currentReplicas - desiredReplicas - name + - readyReplicas - replicas type: object type: array + readyReplicas: + description: The number of instances which are ready. + format: int32 + type: integer replicas: - description: The number of instances created by a placement + description: The number of instances that currently exist format: int32 type: integer required: - currentReplicas + - deployments - desiredReplicas + - readyReplicas - replicas type: object required: diff --git a/go.mod b/go.mod index 7547276d..09ed49da 100644 --- a/go.mod +++ b/go.mod @@ -1,30 +1,31 @@ module go.datum.net/workload-operator -go 1.23.0 +go 1.24.0 + +toolchain go1.24.1 require ( - github.com/go-logr/logr v1.4.2 + github.com/go-logr/logr v1.4.3 github.com/google/go-cmp v0.7.0 github.com/onsi/ginkgo/v2 v2.23.4 github.com/onsi/gomega v1.37.0 github.com/stretchr/testify v1.10.0 go.datum.net/network-services-operator v0.1.0 golang.org/x/crypto v0.37.0 - golang.org/x/sync v0.13.0 + golang.org/x/sync v0.15.0 google.golang.org/protobuf v1.36.6 - k8s.io/api v0.32.1 - k8s.io/apimachinery v0.32.1 - k8s.io/client-go v0.32.1 - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 - sigs.k8s.io/controller-runtime v0.20.4 + k8s.io/api v0.33.1 + k8s.io/apimachinery v0.33.1 + k8s.io/client-go v0.33.1 + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + sigs.k8s.io/controller-runtime v0.21.0 sigs.k8s.io/gateway-api v1.2.1 - sigs.k8s.io/multicluster-runtime v0.20.4-alpha.6 + sigs.k8s.io/multicluster-runtime v0.21.0-alpha.8 ) require ( - cel.dev/expr v0.18.0 // indirect + cel.dev/expr v0.19.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect - github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect @@ -42,14 +43,12 @@ require ( github.com/go-openapi/swag v0.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect - github.com/google/cel-go v0.22.0 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/gofuzz v1.2.0 // indirect + github.com/google/cel-go v0.23.2 // indirect + github.com/google/gnostic-models v0.6.9 // indirect github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -59,47 +58,49 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect - go.opentelemetry.io/otel v1.29.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.29.0 // indirect - go.opentelemetry.io/otel/sdk v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.29.0 // indirect - go.opentelemetry.io/proto/otlp v1.3.1 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/otel/sdk v1.33.0 // indirect + go.opentelemetry.io/otel/trace v1.33.0 // indirect + go.opentelemetry.io/proto/otlp v1.4.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.38.0 // indirect - golang.org/x/oauth2 v0.24.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sys v0.32.0 // indirect golang.org/x/term v0.31.0 // indirect golang.org/x/text v0.24.0 // indirect - golang.org/x/time v0.7.0 // indirect + golang.org/x/time v0.9.0 // indirect golang.org/x/tools v0.31.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 // indirect - google.golang.org/grpc v1.67.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/grpc v1.68.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.32.1 // indirect - k8s.io/apiserver v0.32.1 // indirect - k8s.io/component-base v0.32.1 // indirect + k8s.io/apiextensions-apiserver v0.33.1 // indirect + k8s.io/apiserver v0.33.1 // indirect + k8s.io/component-base v0.33.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 931eb0d6..360a7b32 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,7 @@ -cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo= -cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= +cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -30,8 +28,8 @@ github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyT github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -50,10 +48,10 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/cel-go v0.22.0 h1:b3FJZxpiv1vTMo2/5RDUqAHPxkT8mmMfJIrq1llbf7g= -github.com/google/cel-go v0.22.0/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4= +github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -64,8 +62,8 @@ github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -74,10 +72,14 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -98,16 +100,16 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= @@ -118,6 +120,8 @@ github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8w github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -130,22 +134,24 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.datum.net/network-services-operator v0.1.0 h1:PAXOZ5DdJFgRoeVBPIXhqkCm6DxbP4tVOPcr3Y7h/So= go.datum.net/network-services-operator v0.1.0/go.mod h1:uloVfxqE+8DgSiMB651X8UC9yECpXbwp/NBstofCceE= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= -go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= -go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= -go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= -go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= -go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= -go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= +go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -169,13 +175,13 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= -golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -187,8 +193,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -201,12 +207,12 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= -google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= -google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= -google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -219,35 +225,40 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= -k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k= -k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw= -k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= -k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs= -k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/apiserver v0.32.1 h1:oo0OozRos66WFq87Zc5tclUX2r0mymoVHRq8JmR7Aak= -k8s.io/apiserver v0.32.1/go.mod h1:UcB9tWjBY7aryeI5zAgzVJB/6k7E97bkr1RgqDz0jPw= -k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU= -k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg= -k8s.io/component-base v0.32.1 h1:/5IfJ0dHIKBWysGV0yKTFfacZ5yNV1sulPh3ilJjRZk= -k8s.io/component-base v0.32.1/go.mod h1:j1iMMHi/sqAHeG5z+O9BFNCF698a1u0186zkjMZQ28w= +k8s.io/api v0.33.1 h1:tA6Cf3bHnLIrUK4IqEgb2v++/GYUtqiu9sRVk3iBXyw= +k8s.io/api v0.33.1/go.mod h1:87esjTn9DRSRTD4fWMXamiXxJhpOIREjWOSjsW1kEHw= +k8s.io/apiextensions-apiserver v0.33.1 h1:N7ccbSlRN6I2QBcXevB73PixX2dQNIW0ZRuguEE91zI= +k8s.io/apiextensions-apiserver v0.33.1/go.mod h1:uNQ52z1A1Gu75QSa+pFK5bcXc4hq7lpOXbweZgi4dqA= +k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4= +k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apiserver v0.33.1 h1:yLgLUPDVC6tHbNcw5uE9mo1T6ELhJj7B0geifra3Qdo= +k8s.io/apiserver v0.33.1/go.mod h1:VMbE4ArWYLO01omz+k8hFjAdYfc3GVAYPrhP2tTKccs= +k8s.io/client-go v0.33.1 h1:ZZV/Ks2g92cyxWkRRnfUDsnhNn28eFpt26aGc8KbXF4= +k8s.io/client-go v0.33.1/go.mod h1:JAsUrl1ArO7uRVFWfcj6kOomSlCv+JpvIsp6usAGefA= +k8s.io/component-base v0.33.1 h1:EoJ0xA+wr77T+G8p6T3l4efT2oNwbqBVKR71E0tBIaI= +k8s.io/component-base v0.33.1/go.mod h1:guT/w/6piyPfTgq7gfvgetyXMIh10zuXA6cRRm3rDuY= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 h1:CPT0ExVicCzcpeN4baWEV2ko2Z/AsiZgEdwgcfwLgMo= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/controller-runtime v0.20.4 h1:X3c+Odnxz+iPTRobG4tp092+CvBU9UK0t/bRf+n0DGU= -sigs.k8s.io/controller-runtime v0.20.4/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= +sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= sigs.k8s.io/gateway-api v1.2.1 h1:fZZ/+RyRb+Y5tGkwxFKuYuSRQHu9dZtbjenblleOLHM= sigs.k8s.io/gateway-api v1.2.1/go.mod h1:EpNfEXNjiYfUJypf0eZ0P5iXA9ekSGWaS1WgPaM42X0= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= -sigs.k8s.io/multicluster-runtime v0.20.4-alpha.6 h1:xaBJoiYZY+DlaHBIXfS+ZUH+L5GOD6A+klL46vekggE= -sigs.k8s.io/multicluster-runtime v0.20.4-alpha.6/go.mod h1:2N2/c3p08bYC9eDaRs0dllTxgAm5xiLDSkmGZpWKyw4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/multicluster-runtime v0.21.0-alpha.8 h1:Pq69tTKfN8ADw8m8A3wUtP8wJ9SPQbbOsgapm3BZEPw= +sigs.k8s.io/multicluster-runtime v0.21.0-alpha.8/go.mod h1:CpBzLMLQKdm+UCchd2FiGPiDdCxM5dgCCPKuaQ6Fsv0= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/controller/indexers.go b/internal/controller/indexers.go new file mode 100644 index 00000000..5c3351da --- /dev/null +++ b/internal/controller/indexers.go @@ -0,0 +1,87 @@ +package controller + +import ( + "context" + "errors" + "fmt" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + + computev1alpha "go.datum.net/workload-operator/api/v1alpha" +) + +const ( + deploymentWorkloadUIDIndex = "deploymentWorkloadUIDIndex" + workloadNetworksIndex = "workloadNetworksIndex" + deploymentLocationIndex = "deploymentLocationIndex" +) + +func AddIndexers(ctx context.Context, mgr mcmanager.Manager) error { + return errors.Join( + addWorkloadDeploymentIndexers(ctx, mgr), + addWorkloadIndexers(ctx, mgr), + ) +} + +func addWorkloadDeploymentIndexers(ctx context.Context, mgr mcmanager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &computev1alpha.WorkloadDeployment{}, deploymentWorkloadUIDIndex, deploymentWorkloadUIDIndexFunc); err != nil { + return fmt.Errorf("failed to add workload deployment indexer %q: %w", deploymentWorkloadUIDIndex, err) + } + + // Index workload deployments by location + if err := mgr.GetFieldIndexer().IndexField(ctx, &computev1alpha.WorkloadDeployment{}, deploymentLocationIndex, deploymentLocationIndexFunc); err != nil { + return fmt.Errorf("failed to add workload deployment indexer %q: %w", deploymentLocationIndex, err) + } + + return nil +} + +func deploymentWorkloadUIDIndexFunc(o client.Object) []string { + return []string{ + string(o.(*computev1alpha.WorkloadDeployment).Spec.WorkloadRef.UID), + } +} + +func deploymentLocationIndexFunc(o client.Object) []string { + deployment := o.(*computev1alpha.WorkloadDeployment) + if deployment.Status.Location == nil { + return nil + } + + return []string{ + types.NamespacedName{ + Namespace: deployment.Status.Location.Namespace, + Name: deployment.Status.Location.Name, + }.String(), + } +} + +func addWorkloadIndexers(ctx context.Context, mgr mcmanager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &computev1alpha.Workload{}, workloadNetworksIndex, workloadNetworksIndexFunc); err != nil { + return fmt.Errorf("failed to add workload indexer %q: %w", workloadNetworksIndex, err) + } + + return nil +} + +func workloadNetworksIndexFunc(o client.Object) []string { + workload := o.(*computev1alpha.Workload) + + networks := make([]string, 0, len(workload.Spec.Template.Spec.NetworkInterfaces)) + for _, network := range workload.Spec.Template.Spec.NetworkInterfaces { + namespacedName := types.NamespacedName{ + Namespace: network.Network.Namespace, + Name: network.Network.Name, + } + + if namespacedName.Namespace == "" { + namespacedName.Namespace = workload.GetNamespace() + } + + networks = append(networks, namespacedName.String()) + } + + return networks +} diff --git a/internal/controller/instance_controller.go b/internal/controller/instance_controller.go index 5f56cda3..6be1fccf 100644 --- a/internal/controller/instance_controller.go +++ b/internal/controller/instance_controller.go @@ -5,9 +5,10 @@ package controller import ( "context" "fmt" + "strings" - "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -17,6 +18,7 @@ import ( mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" computev1alpha "go.datum.net/workload-operator/api/v1alpha" ) @@ -29,7 +31,7 @@ type InstanceReconciler struct { // +kubebuilder:rbac:groups=compute.datumapis.com,resources=instances/status,verbs=get;update;patch // +kubebuilder:rbac:groups=compute.datumapis.com,resources=instances/finalizers,verbs=update -func (r *InstanceReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { +func (r *InstanceReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (_ ctrl.Result, err error) { logger := log.FromContext(ctx) cl, err := r.mgr.GetCluster(ctx, req.ClusterName) @@ -46,53 +48,150 @@ func (r *InstanceReconciler) Reconcile(ctx context.Context, req mcreconcile.Requ return ctrl.Result{}, err } - if !instance.DeletionTimestamp.IsZero() { - return ctrl.Result{}, nil - } - logger.Info("reconciling instance") defer logger.Info("reconcile complete") - // Ensure the instance has labels necessary for being able to identify - // instances associated with a specific workload or workload deployment via - // label selectors. - // - // This logic will not be necessary once we complete the work defined in - // https://github.com/datum-cloud/enhancements/issues/28 + if changed, err := r.reconcileInstanceReadyCondition(ctx, cl.GetClient(), &instance, r.checkForNetworkCreationFailure); err != nil { + return ctrl.Result{}, err + } else if changed { + return ctrl.Result{}, cl.GetClient().Status().Update(ctx, &instance) + } + + return ctrl.Result{}, nil +} + +// networkFailureChecker is a function that checks if a network creation failure +// has occurred. It returns a boolean indicating if a failure has occurred, a +// message describing the failure, and an error if the check fails. +type networkFailureChecker func(ctx context.Context, upstreamClient client.Client, instance *computev1alpha.Instance) (failed bool, message string, err error) + +func (r *InstanceReconciler) reconcileInstanceReadyCondition( + ctx context.Context, + clusterClient client.Client, + instance *computev1alpha.Instance, + networkFailureChecker networkFailureChecker, +) (changed bool, err error) { + logger := log.FromContext(ctx) + + readyCondition := apimeta.FindStatusCondition(instance.Status.Conditions, computev1alpha.InstanceReady) + if readyCondition == nil { + readyCondition = &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: computev1alpha.InstanceProgrammedReasonPendingProgramming, + ObservedGeneration: instance.Generation, + Message: "Instance has not been programmed", + } + } else { + readyCondition = readyCondition.DeepCopy() + } - workloadDeploymentRef := metav1.GetControllerOf(&instance) + if instance.Spec.Controller != nil && len(instance.Spec.Controller.SchedulingGates) > 0 { + // Update Ready condition to False, Reason to "SchedulingGatesPresent" + // and Message to "Scheduling gates present" + + // Collect a list of scheduling gate names + var schedulingGateNames []string + for _, gate := range instance.Spec.Controller.SchedulingGates { + schedulingGateNames = append(schedulingGateNames, gate.Name) + } + + networkCreationFailure, networkCreationFailureMessage, err := networkFailureChecker(ctx, clusterClient, instance) + if err != nil { + return false, fmt.Errorf("failed checking for network creation failure: %w", err) + } + + if networkCreationFailure { + readyCondition.Reason = "NetworkFailedToCreate" + readyCondition.Message = networkCreationFailureMessage + } else { + readyCondition.Reason = computev1alpha.InstanceReadyReasonSchedulingGatesPresent + readyCondition.Message = fmt.Sprintf("Scheduling gates present: %s", strings.Join(schedulingGateNames, ", ")) + } + + return apimeta.SetStatusCondition(&instance.Status.Conditions, *readyCondition), nil + } + + pendingReason := "Pending" + programmedCondition := apimeta.FindStatusCondition(instance.Status.Conditions, computev1alpha.InstanceProgrammed) + if programmedCondition == nil || programmedCondition.Status != metav1.ConditionTrue { + logger.Info("instance is not programmed", "instance", instance.Name) + + readyCondition.Reason = computev1alpha.InstanceProgrammedReasonPendingProgramming + if programmedCondition != nil && programmedCondition.Reason != pendingReason { + readyCondition.Reason = programmedCondition.Reason + } + + readyCondition.Message = "Instance has not been programmed" + if programmedCondition != nil && programmedCondition.Status != metav1.ConditionUnknown { + readyCondition.Message = programmedCondition.Message + } + + return apimeta.SetStatusCondition(&instance.Status.Conditions, *readyCondition), nil + } + + logger.Info("instance is programmed", "instance", instance.Name) + + runningCondition := apimeta.FindStatusCondition(instance.Status.Conditions, computev1alpha.InstanceRunning) + if runningCondition == nil || runningCondition.Status != metav1.ConditionTrue { + logger.Info("instance is not running", "instance", instance.Name) + + readyCondition.Reason = pendingReason + if runningCondition != nil && runningCondition.Reason != pendingReason { + readyCondition.Reason = runningCondition.Reason + } + + readyCondition.Message = "Instance is not running" + if runningCondition != nil && runningCondition.Status != metav1.ConditionUnknown { + readyCondition.Message = runningCondition.Message + } + + return apimeta.SetStatusCondition(&instance.Status.Conditions, *readyCondition), nil + } + + readyCondition.Status = metav1.ConditionTrue + readyCondition.Reason = computev1alpha.InstanceReadyReasonRunning + readyCondition.Message = "Instance is ready" + + return apimeta.SetStatusCondition(&instance.Status.Conditions, *readyCondition), nil +} + +// Rough way to propagate creation errors up to the instance as soon as possible. +// Lots of room for improvement here. +func (r *InstanceReconciler) checkForNetworkCreationFailure(ctx context.Context, upstreamClient client.Client, instance *computev1alpha.Instance) (failed bool, message string, err error) { + workloadDeploymentRef := metav1.GetControllerOf(instance) if workloadDeploymentRef == nil { - return ctrl.Result{}, fmt.Errorf("failed to get controller owner of Instance") + return false, "", fmt.Errorf("instance is not owned by a workload deployment") } + // Load the WorkloadDeployment for the instance var workloadDeployment computev1alpha.WorkloadDeployment workloadDeploymentObjectKey := client.ObjectKey{ Namespace: instance.Namespace, Name: workloadDeploymentRef.Name, } - if err := cl.GetClient().Get(ctx, workloadDeploymentObjectKey, &workloadDeployment); err != nil { - return ctrl.Result{}, err + if err := upstreamClient.Get(ctx, workloadDeploymentObjectKey, &workloadDeployment); err != nil { + return false, "", fmt.Errorf("failed fetching workload deployment: %w", err) } - workloadRef := metav1.GetControllerOf(&workloadDeployment) - if workloadRef == nil { - return ctrl.Result{}, fmt.Errorf("failed to get controller owner of WorkloadDeployment") - } + for i := range instance.Spec.NetworkInterfaces { + var networkBinding networkingv1alpha.NetworkBinding + networkBindingObjectKey := client.ObjectKey{ + Namespace: workloadDeployment.Namespace, + Name: fmt.Sprintf("%s-net-%d", workloadDeployment.Name, i), + } - updated := instance.DeepCopy() - if updated.Labels == nil { - updated.Labels = map[string]string{} - } - updated.Labels[computev1alpha.WorkloadUIDLabel] = string(workloadRef.UID) - updated.Labels[computev1alpha.WorkloadDeploymentUIDLabel] = string(workloadDeploymentRef.UID) + if err := upstreamClient.Get(ctx, networkBindingObjectKey, &networkBinding); client.IgnoreNotFound(err) != nil { + return false, "", fmt.Errorf("failed checking for existing network binding: %w", err) + } - if !equality.Semantic.DeepEqual(updated, instance) { - if err := cl.GetClient().Update(ctx, updated); err != nil { - return ctrl.Result{}, fmt.Errorf("failed updating instance: %w", err) + condition := apimeta.FindStatusCondition(networkBinding.Status.Conditions, networkingv1alpha.NetworkBindingReady) + if condition != nil && condition.Status == metav1.ConditionFalse && condition.Reason == "NetworkFailedToCreate" { + return true, condition.Message, nil } } - return ctrl.Result{}, nil + return false, "", nil } // SetupWithManager sets up the controller with the Manager. diff --git a/internal/controller/instance_controller_test.go b/internal/controller/instance_controller_test.go index b4725a75..7eca925c 100644 --- a/internal/controller/instance_controller_test.go +++ b/internal/controller/instance_controller_test.go @@ -5,187 +5,276 @@ import ( "testing" "github.com/stretchr/testify/assert" - "google.golang.org/protobuf/proto" + "github.com/stretchr/testify/require" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" - mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" computev1alpha "go.datum.net/workload-operator/api/v1alpha" ) -type testManager struct { - ctrl.Manager - client client.Client - scheme *runtime.Scheme -} - -func (m *testManager) GetClient() client.Client { - return m.client -} - -func (m *testManager) GetScheme() *runtime.Scheme { - return m.scheme -} - -func TestInstanceReconciler(t *testing.T) { - scheme := runtime.NewScheme() - utilruntime.Must(computev1alpha.AddToScheme(scheme)) +func TestReconcileInstanceReadyCondition(t *testing.T) { tests := []struct { - name string - objs []client.Object - req ctrl.Request - expectedErr string - expectedLabels map[string]string + name string + instance *computev1alpha.Instance + networkFailureFunc networkFailureChecker + expectedChanged bool + expectedCondition *metav1.Condition }{ { - name: "missing controller owner in instance", - objs: []client.Object{ - &computev1alpha.Instance{ - ObjectMeta: metav1.ObjectMeta{ - Name: "instance-no-owner", - Namespace: "default", + name: "instance without ready condition should create default", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + }, + expectedChanged: true, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: computev1alpha.InstanceProgrammedReasonPendingProgramming, + Message: "Instance has not been programmed", + ObservedGeneration: 1, + }, + }, + { + name: "instance with scheduling gates should set scheduling gates present", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + Spec: computev1alpha.InstanceSpec{ + Controller: &computev1alpha.InstanceController{ + SchedulingGates: []computev1alpha.SchedulingGate{ + {Name: "Network"}, + }, + }, + }, + Status: computev1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + { + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: computev1alpha.InstanceProgrammedReasonPendingProgramming, + Message: "Instance has not been programmed", + ObservedGeneration: 1, + LastTransitionTime: metav1.Now(), + }, }, }, }, - req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "instance-no-owner"}}, - expectedErr: "failed to get controller owner of Instance", + expectedChanged: true, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: computev1alpha.InstanceReadyReasonSchedulingGatesPresent, + Message: "Scheduling gates present: Network", + ObservedGeneration: 1, + }, }, { - name: "workload deployment not found", - objs: []client.Object{ - &computev1alpha.Instance{ - ObjectMeta: metav1.ObjectMeta{ - Name: "instance-missing-wd", - Namespace: "default", - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "compute.datumapis.com/v1alpha", - Kind: "WorkloadDeployment", - Name: "wd1", - UID: "33c325cb-3f2e-4b2a-be0c-6d7e03aa475a", - Controller: proto.Bool(true), - }, + name: "instance with scheduling gates and network failure should set network failed", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + Spec: computev1alpha.InstanceSpec{ + Controller: &computev1alpha.InstanceController{ + SchedulingGates: []computev1alpha.SchedulingGate{ + {Name: "Network"}, }, }, }, }, - req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "instance-missing-wd"}}, - expectedErr: "not found", + networkFailureFunc: func(ctx context.Context, upstreamClient client.Client, instance *computev1alpha.Instance) (bool, string, error) { + return true, "Network creation failed: timeout", nil + }, + expectedChanged: true, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: "NetworkFailedToCreate", + Message: "Network creation failed: timeout", + ObservedGeneration: 1, + }, }, { - name: "missing controller owner in workload deployment", - objs: []client.Object{ - &computev1alpha.Instance{ - ObjectMeta: metav1.ObjectMeta{ - Name: "instance-wd-no-owner", - Namespace: "default", - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "compute.datumapis.com/v1alpha", - Kind: "WorkloadDeployment", - Name: "wd1", - UID: "33c325cb-3f2e-4b2a-be0c-6d7e03aa475a", - Controller: proto.Bool(true), - }, + name: "instance not programmed should set pending programming", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + Status: computev1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + { + Type: computev1alpha.InstanceProgrammed, + Status: metav1.ConditionFalse, + Reason: "TestReason", + Message: "Test message", }, }, }, - &computev1alpha.WorkloadDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "wd1", - Namespace: "default", - // Missing controller owner + }, + expectedChanged: true, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: "TestReason", + Message: "Test message", + ObservedGeneration: 1, + }, + }, + { + name: "instance programmed but not running should wait for running", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + Status: computev1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + { + Type: computev1alpha.InstanceProgrammed, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceProgrammedReasonProgrammed, + Message: "Instance has been programmed", + }, + { + Type: computev1alpha.InstanceRunning, + Status: metav1.ConditionFalse, + Reason: "TestReason", + Message: "Test message", + }, }, }, }, - req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "instance-wd-no-owner"}}, - expectedErr: "failed to get controller owner of WorkloadDeployment", + expectedChanged: true, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: "TestReason", + Message: "Test message", + ObservedGeneration: 1, + }, }, { - name: "successful reconcile and update", - objs: []client.Object{ - &computev1alpha.Instance{ - ObjectMeta: metav1.ObjectMeta{ - Name: "instance-success", - Namespace: "default", - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "compute.datumapis.com/v1alpha", - Kind: "WorkloadDeployment", - Name: "wd1", - UID: "33c325cb-3f2e-4b2a-be0c-6d7e03aa475a", - Controller: proto.Bool(true), - }, + name: "instance fully ready should set ready condition", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + Status: computev1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + { + Type: computev1alpha.InstanceProgrammed, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceProgrammedReasonProgrammed, + Message: "Instance has been programmed", + }, + { + Type: computev1alpha.InstanceRunning, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceRunningReasonRunning, + Message: "Instance is running", }, }, }, - &computev1alpha.WorkloadDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "wd1", - Namespace: "default", - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "compute.datumapis.com/v1alpha", - Kind: "Workload", - Name: "w1", - UID: "2561b624-3f7d-49db-bc74-b9dc71c4c08d", - Controller: proto.Bool(true), - }, + }, + expectedChanged: true, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceReadyReasonRunning, + Message: "Instance is ready", + ObservedGeneration: 1, + }, + }, + { + name: "no change when condition already matches", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + Status: computev1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + { + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceReadyReasonRunning, + Message: "Instance is ready", + ObservedGeneration: 1, + LastTransitionTime: metav1.Now(), + }, + { + Type: computev1alpha.InstanceProgrammed, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceProgrammedReasonProgrammed, + Message: "Instance has been programmed", + }, + { + Type: computev1alpha.InstanceRunning, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceRunningReasonRunning, + Message: "Instance is running", }, }, }, }, - req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "instance-success"}}, - expectedErr: "", - expectedLabels: map[string]string{ - computev1alpha.WorkloadUIDLabel: "2561b624-3f7d-49db-bc74-b9dc71c4c08d", - computev1alpha.WorkloadDeploymentUIDLabel: "33c325cb-3f2e-4b2a-be0c-6d7e03aa475a", + expectedChanged: false, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceReadyReasonRunning, + Message: "Instance is ready", + ObservedGeneration: 1, }, }, } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cl := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(tc.objs...).Build() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { - testMgr := &testManager{ - client: cl, - scheme: scheme, - } + reconciler := &InstanceReconciler{} - mgr, err := mcmanager.WithMultiCluster(testMgr, nil) - if err != nil { - t.Fatalf("failed to create manager: %v", err) + networkFailureFunc := tt.networkFailureFunc + if networkFailureFunc == nil { + networkFailureFunc = func(ctx context.Context, upstreamClient client.Client, instance *computev1alpha.Instance) (bool, string, error) { + return false, "", nil + } } - reconciler := &InstanceReconciler{mgr} - _, err = reconciler.Reconcile(context.Background(), mcreconcile.Request{ - Request: tc.req, - ClusterName: "", - }) + changed, err := reconciler.reconcileInstanceReadyCondition( + ctx, + nil, + tt.instance, + networkFailureFunc, + ) - // Check error - if tc.expectedErr == "" { - assert.NoError(t, err) - } else { - assert.ErrorContains(t, err, tc.expectedErr) - } + require.NoError(t, err) + assert.Equal(t, tt.expectedChanged, changed) - // If labels are expected, fetch the instance and validate the labels. - if tc.expectedLabels != nil { - instance := &computev1alpha.Instance{} - if err := cl.Get(context.Background(), tc.req.NamespacedName, instance); err != nil { - t.Fatalf("failed to get instance: %v", err) - } - assert.Equal(t, tc.expectedLabels, instance.Labels) - } + readyCondition := apimeta.FindStatusCondition(tt.instance.Status.Conditions, computev1alpha.InstanceReady) + require.NotNil(t, readyCondition) + + assert.Equal(t, tt.expectedCondition.Type, readyCondition.Type) + assert.Equal(t, tt.expectedCondition.Status, readyCondition.Status) + assert.Equal(t, tt.expectedCondition.Reason, readyCondition.Reason) + assert.Equal(t, tt.expectedCondition.Message, readyCondition.Message) + assert.Equal(t, tt.expectedCondition.ObservedGeneration, readyCondition.ObservedGeneration) }) } } diff --git a/internal/controller/instancecontrol/controller_utils.go b/internal/controller/instancecontrol/controller_utils.go new file mode 100644 index 00000000..8dd890ee --- /dev/null +++ b/internal/controller/instancecontrol/controller_utils.go @@ -0,0 +1,30 @@ +package instancecontrol + +import ( + "fmt" + "hash" + "hash/fnv" + + "k8s.io/apimachinery/pkg/util/dump" + "k8s.io/apimachinery/pkg/util/rand" +) + +// ComputeHash returns a hash value calculated from pod template and +// a collisionCount to avoid hash collision. The hash will be safe encoded to +// avoid bad words. +// +// Inspired by: https://github.com/kubernetes/kubernetes/blob/8aae5398b3885dc271d407c4d661e19653daaf88/pkg/controller/controller_utils.go#L1295 +func ComputeHash(obj interface{}) string { + hasher := fnv.New32a() + DeepHashObject(hasher, obj) + + return rand.SafeEncodeString(fmt.Sprint(hasher.Sum32())) +} + +// DeepHashObject writes specified object to hash using the spew library +// which follows pointers and prints actual values of the nested objects +// ensuring the hash does not change when a pointer changes. +func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) { + hasher.Reset() + _, _ = fmt.Fprintf(hasher, "%v", dump.ForHash(objectToWrite)) +} diff --git a/internal/controller/instancecontrol/instancecontrol.go b/internal/controller/instancecontrol/instancecontrol.go new file mode 100644 index 00000000..a217e4a2 --- /dev/null +++ b/internal/controller/instancecontrol/instancecontrol.go @@ -0,0 +1,106 @@ +package instancecontrol + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "go.datum.net/workload-operator/api/v1alpha" +) + +type Strategy interface { + // GetActions returns a set of actions that should be taken in order to drive + // the workload deployment to the desired state. Some actions may be informational, + // such as providing context into pending actions when instance management + // policies may require waiting for pod readiness. + GetActions( + ctx context.Context, + scheme *runtime.Scheme, + deployment *v1alpha.WorkloadDeployment, + currentInstances []v1alpha.Instance, + ) ([]Action, error) +} + +type ActionType string + +const ( + ActionTypeCreate ActionType = "Create" + ActionTypeUpdate ActionType = "Update" + ActionTypeDelete ActionType = "Delete" + ActionTypeWait ActionType = "Wait" +) + +type Action struct { + Object client.Object + actionType ActionType + skipExecution bool + fn func(ctx context.Context, c client.Client) error +} + +func (a Action) Execute(ctx context.Context, c client.Client) error { + if a.skipExecution { + return nil + } + + return a.fn(ctx, c) +} + +func (a *Action) SkipExecution() { + a.skipExecution = true +} + +func (a Action) IsSkipped() bool { + return a.skipExecution +} + +func (a Action) ActionType() ActionType { + return a.actionType +} + +func NewCreateAction(object client.Object) Action { + return Action{ + Object: object, + actionType: ActionTypeCreate, + fn: func(ctx context.Context, c client.Client) error { + if err := c.Create(ctx, object); err != nil { + return fmt.Errorf("failed to create %T: %w", object, err) + } + + return nil + }, + } +} + +func NewUpdateAction(object client.Object) Action { + return Action{ + Object: object, + actionType: ActionTypeUpdate, + fn: func(ctx context.Context, c client.Client) error { + if err := c.Update(ctx, object); err != nil { + return fmt.Errorf("failed to update %T: %w", object, err) + } + + return nil + }, + } +} + +func NewDeleteAction(object client.Object) Action { + return Action{ + Object: object, + actionType: ActionTypeDelete, + fn: func(ctx context.Context, c client.Client) error { + return c.Delete(ctx, object) + }, + } +} + +func NewWaitAction(object client.Object) Action { + return Action{ + Object: object, + actionType: ActionTypeWait, + fn: func(ctx context.Context, c client.Client) error { return nil }, + } +} diff --git a/internal/controller/instancecontrol/scheduling_gates.go b/internal/controller/instancecontrol/scheduling_gates.go new file mode 100644 index 00000000..5ff5ce9a --- /dev/null +++ b/internal/controller/instancecontrol/scheduling_gates.go @@ -0,0 +1,11 @@ +package instancecontrol + +type SchedulingGate string + +const ( + NetworkSchedulingGate SchedulingGate = "Network" +) + +func (s SchedulingGate) String() string { + return string(s) +} diff --git a/internal/controller/instancecontrol/stateful/stateful_control.go b/internal/controller/instancecontrol/stateful/stateful_control.go new file mode 100644 index 00000000..05bb9ee0 --- /dev/null +++ b/internal/controller/instancecontrol/stateful/stateful_control.go @@ -0,0 +1,159 @@ +package stateful + +import ( + "context" + "fmt" + "slices" + "strconv" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "go.datum.net/workload-operator/api/v1alpha" + "go.datum.net/workload-operator/internal/controller/instancecontrol" +) + +// Behavior inspired by https://github.com/kubernetes/kubernetes/tree/master/pkg/controller/statefulset +// Does not currently implement exact behavior. +type statefulControl struct { +} + +func New() instancecontrol.Strategy { + return &statefulControl{} +} + +func (c *statefulControl) GetActions( + ctx context.Context, + scheme *runtime.Scheme, + deployment *v1alpha.WorkloadDeployment, + currentInstances []v1alpha.Instance, +) ([]instancecontrol.Action, error) { + instanceTemplateHash := instancecontrol.ComputeHash(deployment.Spec.Template) + + // lowest -> highest + var createActions []instancecontrol.Action + var waitActions []instancecontrol.Action + + // highest -> lowest + var updateActions []instancecontrol.Action + + // highest -> lowest + var deleteActions []instancecontrol.Action + + // Instances that are desired to exist. We do not currently support the + // concept of a partition, so will fill the entire slice. + desiredInstances := make([]*v1alpha.Instance, deployment.Spec.ScaleSettings.MinReplicas) + + for _, instance := range currentInstances { + instanceIndex := getInstanceOrdinal(instance.Name) + if instanceIndex >= len(desiredInstances) { + deleteActions = append(deleteActions, instancecontrol.NewDeleteAction(&instance)) + } else { + desiredInstances[instanceIndex] = &instance + } + } + + // It's possible that the incoming currentInstances will have gaps in + // instances, so fill them in. + for i := range deployment.Spec.ScaleSettings.MinReplicas { + if desiredInstances[i] == nil { + desiredInstances[i] = &v1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Labels: deployment.Spec.Template.Labels, + Annotations: deployment.Spec.Template.Annotations, + Name: fmt.Sprintf("%s-%d", deployment.Name, i), + Namespace: deployment.Namespace, + }, + Spec: deployment.Spec.Template.Spec, + } + desiredInstances[i].Spec.Location = deployment.Status.Location + + // TODO(jreese) consider adding scheduling gates via mutating webhooks + desiredInstances[i].Spec.Controller = &v1alpha.InstanceController{ + TemplateHash: instanceTemplateHash, + SchedulingGates: []v1alpha.SchedulingGate{ + { + Name: instancecontrol.NetworkSchedulingGate.String(), + }, + }, + } + + addInstanceControllerLabels(desiredInstances[i], getInstanceOrdinal(desiredInstances[i].Name), deployment) + + if err := controllerutil.SetControllerReference(deployment, desiredInstances[i], scheme); err != nil { + return nil, fmt.Errorf("failed to set controller reference: %w", err) + } + } + } + + for _, instance := range desiredInstances { + if instance.CreationTimestamp.IsZero() { + action := instancecontrol.NewCreateAction(instance) + + createActions = append(createActions, action) + } else if !instance.DeletionTimestamp.IsZero() { + // Wait for graceful deletion before continuing processing additional + // instances. + waitActions = append(waitActions, instancecontrol.NewWaitAction(instance)) + + } else if instance.DeletionTimestamp.IsZero() { + // Wait for the instance to be ready before continuing processing + if !apimeta.IsStatusConditionTrue(instance.Status.Conditions, v1alpha.InstanceReady) { + waitActions = append(waitActions, instancecontrol.NewWaitAction(instance)) + } else if needsUpdate(instance, instanceTemplateHash) { + updatedInstance := instance.DeepCopy() + updatedInstance.Annotations = deployment.Spec.Template.Annotations + updatedInstance.Labels = deployment.Spec.Template.Labels + + addInstanceControllerLabels(updatedInstance, getInstanceOrdinal(updatedInstance.Name), deployment) + + updatedInstance.Spec = deployment.Spec.Template.Spec + updateActions = append(updateActions, instancecontrol.NewUpdateAction(updatedInstance)) + } + } + } + + slices.SortFunc(updateActions, descendingOrdinal) + slices.SortFunc(deleteActions, descendingOrdinal) + + actions := make([]instancecontrol.Action, 0, len(createActions)+len(waitActions)+len(updateActions)+len(deleteActions)) + + switch deployment.Spec.ScaleSettings.InstanceManagementPolicy { + case v1alpha.OrderedReadyInstanceManagementPolicyType: + + // Add create and wait actions, and sort by ordinal. This allows us to wait + // for instances to be processed in the correct order. + // + // For instance, we may have instance 0 that needs to wait to be ready, but + // instance 1 wants to be created. + actions = append(actions, createActions...) + actions = append(actions, waitActions...) + + slices.SortFunc(actions, ascendingOrdinal) + + actions = append(actions, updateActions...) + actions = append(actions, deleteActions...) + + // Skip all actions except the first one. + for i := range actions { + if i > 0 { + actions[i].SkipExecution() + } + } + + } + + return actions, nil +} + +func addInstanceControllerLabels(instance *v1alpha.Instance, index int, deployment *v1alpha.WorkloadDeployment) { + if instance.Labels == nil { + instance.Labels = map[string]string{} + } + + instance.Labels[v1alpha.InstanceIndexLabel] = strconv.Itoa(index) + instance.Labels[v1alpha.WorkloadUIDLabel] = string(deployment.Spec.WorkloadRef.UID) + instance.Labels[v1alpha.WorkloadDeploymentUIDLabel] = string(deployment.GetUID()) +} diff --git a/internal/controller/instancecontrol/stateful/stateful_control_test.go b/internal/controller/instancecontrol/stateful/stateful_control_test.go new file mode 100644 index 00000000..ad99e749 --- /dev/null +++ b/internal/controller/instancecontrol/stateful/stateful_control_test.go @@ -0,0 +1,233 @@ +package stateful + +import ( + "context" + "fmt" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/utils/ptr" + + "go.datum.net/workload-operator/api/v1alpha" + "go.datum.net/workload-operator/internal/controller/instancecontrol" +) + +var ( + scheme = runtime.NewScheme() +) + +func init() { + utilruntime.Must(v1alpha.AddToScheme(scheme)) +} + +func TestFreshDeployment(t *testing.T) { + ctx := context.Background() + control := New() + + deployment := getWorkloadDeployment("test-fresh-deploy", 2) + + // No instances + var currentInstances []v1alpha.Instance + actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + + assert.NoError(t, err) + assert.Len(t, actions, 2) + + assert.Equal(t, "test-fresh-deploy-0", actions[0].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeCreate, actions[0].ActionType()) + assert.False(t, actions[0].IsSkipped()) + + assert.Equal(t, "test-fresh-deploy-1", actions[1].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeCreate, actions[1].ActionType()) + assert.True(t, actions[1].IsSkipped()) +} + +func TestUpdateWithAllReadyInstances(t *testing.T) { + ctx := context.Background() + control := New() + + deployment := getWorkloadDeployment("test-deploy", 2) + + var currentInstances []v1alpha.Instance + currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 0)) + currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 1)) + + deployment.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image = "test-image-update" + + actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + + assert.NoError(t, err) + assert.Len(t, actions, 2) + + assert.Equal(t, "test-deploy-1", actions[0].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeUpdate, actions[0].ActionType()) + assert.False(t, actions[0].IsSkipped()) + + assert.Equal(t, "test-deploy-0", actions[1].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeUpdate, actions[1].ActionType()) + assert.True(t, actions[1].IsSkipped()) +} + +func TestScaleUpWithNotReadyInstance(t *testing.T) { + ctx := context.Background() + control := New() + + deployment := getWorkloadDeployment("test-deploy", 3) + + var currentInstances []v1alpha.Instance + currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 0)) + + notReadyInstance := getInstanceForDeployment(deployment, 1) + apimeta.SetStatusCondition(¬ReadyInstance.Status.Conditions, metav1.Condition{ + Type: v1alpha.InstanceReady, + Status: metav1.ConditionFalse, + }) + currentInstances = append(currentInstances, *notReadyInstance) + + actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + + assert.NoError(t, err) + assert.Len(t, actions, 2) + + assert.Equal(t, "test-deploy-1", actions[0].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeWait, actions[0].ActionType()) + assert.False(t, actions[0].IsSkipped()) + + assert.Equal(t, "test-deploy-2", actions[1].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeCreate, actions[1].ActionType()) + assert.True(t, actions[1].IsSkipped()) +} + +func TestScaleUpWithDeletingReadyInstance(t *testing.T) { + ctx := context.Background() + control := New() + + deployment := getWorkloadDeployment("test-deploy", 3) + + var currentInstances []v1alpha.Instance + currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 0)) + + deletingInstance := getInstanceForDeployment(deployment, 1) + deletingInstance.DeletionTimestamp = ptr.To(metav1.Now()) + currentInstances = append(currentInstances, *deletingInstance) + + actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + + assert.NoError(t, err) + assert.Len(t, actions, 2) + + assert.Equal(t, "test-deploy-1", actions[0].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeWait, actions[0].ActionType()) + assert.False(t, actions[0].IsSkipped()) + + assert.Equal(t, "test-deploy-2", actions[1].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeCreate, actions[1].ActionType()) + assert.True(t, actions[1].IsSkipped()) +} + +func TestScaleDownWithAllReadyInstances(t *testing.T) { + ctx := context.Background() + control := New() + + deployment := getWorkloadDeployment("test-deploy", 1) + + var currentInstances []v1alpha.Instance + currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 0)) + currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 1)) + + actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + + assert.NoError(t, err) + assert.Len(t, actions, 1) + + assert.Equal(t, "test-deploy-1", actions[0].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeDelete, actions[0].ActionType()) + assert.False(t, actions[0].IsSkipped()) +} + +// Add more test functions below for different scenarios. + +func getWorkloadDeployment(name string, minReplicas int32) *v1alpha.WorkloadDeployment { + instance := getInstanceTemplate(name, 0) + deployment := &v1alpha.WorkloadDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + Spec: v1alpha.WorkloadDeploymentSpec{ + ScaleSettings: v1alpha.HorizontalScaleSettings{ + MinReplicas: minReplicas, + InstanceManagementPolicy: v1alpha.OrderedReadyInstanceManagementPolicyType, + }, + Template: v1alpha.InstanceTemplateSpec{ + ObjectMeta: instance.ObjectMeta, + Spec: instance.Spec, + }, + }, + } + + return deployment +} + +func getInstanceForDeployment(deployment *v1alpha.WorkloadDeployment, ordinal int) *v1alpha.Instance { + instance := getInstance(deployment.Name, ordinal) + instance.Spec.Controller = &v1alpha.InstanceController{ + TemplateHash: instancecontrol.ComputeHash(deployment.Spec.Template), + } + + return instance +} + +func getInstance(name string, ordinal int) *v1alpha.Instance { + instance := getInstanceTemplate(name, ordinal) + instance.CreationTimestamp = metav1.Now() + instance.Labels = map[string]string{ + v1alpha.InstanceIndexLabel: strconv.Itoa(ordinal), + } + + instance.Status = v1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + { + Type: v1alpha.InstanceReady, + Status: metav1.ConditionTrue, + Reason: "Ready", + Message: "Instance is ready", + LastTransitionTime: metav1.Now(), + }, + }, + } + + return instance +} + +func getInstanceTemplate(name string, ordinal int) *v1alpha.Instance { + instanceName := fmt.Sprintf("%s-%d", name, ordinal) + instance := &v1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: instanceName, + Namespace: "default", + }, + Spec: v1alpha.InstanceSpec{ + Runtime: v1alpha.InstanceRuntimeSpec{ + Resources: v1alpha.InstanceRuntimeResources{ + InstanceType: "datumcloud/d1-standard-2", + }, + Sandbox: &v1alpha.SandboxRuntime{ + Containers: []v1alpha.SandboxContainer{ + { + Name: "test", + Image: "test", + }, + }, + }, + }, + }, + } + + return instance +} diff --git a/internal/controller/instancecontrol/stateful/stateful_control_util.go b/internal/controller/instancecontrol/stateful/stateful_control_util.go new file mode 100644 index 00000000..32c27e3d --- /dev/null +++ b/internal/controller/instancecontrol/stateful/stateful_control_util.go @@ -0,0 +1,46 @@ +package stateful + +import ( + "strconv" + "strings" + + "go.datum.net/workload-operator/api/v1alpha" + "go.datum.net/workload-operator/internal/controller/instancecontrol" +) + +func needsUpdate(instance *v1alpha.Instance, instanceTemplateHash string) bool { + return instance.Spec.Controller == nil || + instance.Spec.Controller.TemplateHash != instanceTemplateHash +} + +// getInstanceOrdinal returns the ordinal of the instance, or -1 if the instance +// does not have an ordinal. +func getInstanceOrdinal(name string) int { + lastDash := strings.LastIndex(name, "-") + if lastDash == -1 { + return -1 + } + + ordinal := -1 + if i, err := strconv.Atoi(name[lastDash+1:]); err == nil { + ordinal = i + } + + return ordinal +} + +func ascendingOrdinal(a, b instancecontrol.Action) int { + if getInstanceOrdinal(a.Object.GetName()) < getInstanceOrdinal(b.Object.GetName()) { + return -1 + } else { + return 1 + } +} + +func descendingOrdinal(a, b instancecontrol.Action) int { + if getInstanceOrdinal(a.Object.GetName()) > getInstanceOrdinal(b.Object.GetName()) { + return -1 + } else { + return 1 + } +} diff --git a/internal/controller/instancecontrol/stateful/stateful_control_util_test.go b/internal/controller/instancecontrol/stateful/stateful_control_util_test.go new file mode 100644 index 00000000..5456f8df --- /dev/null +++ b/internal/controller/instancecontrol/stateful/stateful_control_util_test.go @@ -0,0 +1,74 @@ +package stateful + +import ( + "fmt" + "math/rand/v2" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "go.datum.net/workload-operator/api/v1alpha" + "go.datum.net/workload-operator/internal/controller/instancecontrol" +) + +func TestGetInstanceOrdinal(t *testing.T) { + tests := []struct { + name string + objectName string + want int + }{ + { + name: "instance with ordinal 0", + objectName: "my-instance-0", + want: 0, + }, + { + name: "instance with ordinal 1", + objectName: "my-instance-1", + want: 1, + }, + { + name: "instance with unexpected suffix", + objectName: "my-instance-foo", + want: -1, + }, + { + name: "instance with no dash in name", + objectName: "myinstance", + want: -1, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := getInstanceOrdinal(test.objectName) + if got != test.want { + t.Errorf("getInstanceOrdinal(%q) = %d, want %d", test.objectName, got, test.want) + } + }) + } +} + +func TestDescendingOrdinal(t *testing.T) { + actions := make([]instancecontrol.Action, 0, 4) + + perm := rand.Perm(4) + for i := range perm { + actions = append(actions, instancecontrol.NewWaitAction( + &v1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("my-instance-%d", perm[i]), + }, + }, + )) + } + + slices.SortFunc(actions, descendingOrdinal) + + assert.Equal(t, actions[0].Object.GetName(), "my-instance-3") + assert.Equal(t, actions[1].Object.GetName(), "my-instance-2") + assert.Equal(t, actions[2].Object.GetName(), "my-instance-1") + assert.Equal(t, actions[3].Object.GetName(), "my-instance-0") +} diff --git a/internal/controller/workload_controller.go b/internal/controller/workload_controller.go index e07da676..9e8f628c 100644 --- a/internal/controller/workload_controller.go +++ b/internal/controller/workload_controller.go @@ -12,13 +12,17 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/finalizer" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" mccontext "sigs.k8s.io/multicluster-runtime/pkg/context" mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" @@ -29,7 +33,6 @@ import ( ) const workloadControllerFinalizer = "compute.datumapis.com/workload-controller" -const deploymentWorkloadUID = "spec.workloadRef.uid" // WorkloadReconciler reconciles a Workload object type WorkloadReconciler struct { @@ -174,7 +177,6 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req mcreconcile.Requ _, err := controllerutil.CreateOrUpdate(ctx, cl.GetClient(), deployment, func() error { if deployment.CreationTimestamp.IsZero() { logger.Info("creating deployment", "deployment_name", deployment.Name) - deployment.Finalizers = desiredDeployment.Finalizers if err := controllerutil.SetControllerReference(&workload, deployment, cl.GetScheme()); err != nil { return fmt.Errorf("failed to set controller on workload deployment: %w", err) } @@ -184,6 +186,8 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req mcreconcile.Requ deployment.Annotations = desiredDeployment.Annotations deployment.Labels = desiredDeployment.Labels + + // TODO(jreese) consider how this plays well with autoscaling deployment.Spec = desiredDeployment.Spec return nil }) @@ -213,6 +217,8 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( totalReplicas := int32(0) totalCurrentReplicas := int32(0) totalDesiredReplicas := int32(0) + totalReadyReplicas := int32(0) + totalDeployments := int32(0) availablePlacementFound := false @@ -231,7 +237,7 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( } } - availableCondition := metav1.Condition{ + placementAvailableCondition := metav1.Condition{ Type: "Available", Status: metav1.ConditionFalse, Reason: "NoAvailableDeployments", @@ -242,10 +248,13 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( replicas := int32(0) currentReplicas := int32(0) desiredReplicas := int32(0) + readyReplicas := int32(0) + totalDeployments += int32(len(placementDeployments)) for _, deployment := range placementDeployments { replicas += deployment.Status.Replicas - currentReplicas += deployment.Status.Replicas - desiredReplicas += deployment.Status.Replicas + currentReplicas += deployment.Status.CurrentReplicas + desiredReplicas += deployment.Status.DesiredReplicas + readyReplicas += deployment.Status.ReadyReplicas if apimeta.IsStatusConditionTrue(deployment.Status.Conditions, "Available") { foundAvailableDeployment = true @@ -254,19 +263,21 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( totalReplicas += replicas totalCurrentReplicas += currentReplicas totalDesiredReplicas += desiredReplicas + totalReadyReplicas += readyReplicas placementStatus.Replicas = replicas placementStatus.CurrentReplicas = currentReplicas placementStatus.DesiredReplicas = desiredReplicas + placementStatus.ReadyReplicas = readyReplicas if foundAvailableDeployment { - availableCondition.Status = metav1.ConditionTrue - availableCondition.Reason = "AvailableDeploymentFound" - availableCondition.Message = "At least one available deployment was found" + placementAvailableCondition.Status = metav1.ConditionTrue + placementAvailableCondition.Reason = "AvailableDeploymentFound" + placementAvailableCondition.Message = "At least one available deployment was found" availablePlacementFound = true } - apimeta.SetStatusCondition(&placementStatus.Conditions, availableCondition) + apimeta.SetStatusCondition(&placementStatus.Conditions, placementAvailableCondition) newWorkloadStatus.Placements = append(newWorkloadStatus.Placements, placementStatus) } @@ -286,9 +297,11 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( apimeta.SetStatusCondition(&newWorkloadStatus.Conditions, availableCondition) + newWorkloadStatus.Deployments = totalDeployments newWorkloadStatus.Replicas = totalReplicas newWorkloadStatus.CurrentReplicas = totalCurrentReplicas newWorkloadStatus.DesiredReplicas = totalDesiredReplicas + newWorkloadStatus.ReadyReplicas = totalReadyReplicas if equality.Semantic.DeepEqual(workload.Status, newWorkloadStatus) { return nil @@ -296,7 +309,7 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( workload.Status = *newWorkloadStatus if err := upstreamClient.Status().Update(ctx, workload); err != nil { - return fmt.Errorf("failed updating workload status") + return fmt.Errorf("failed updating workload status: %w", err) } return nil @@ -317,7 +330,7 @@ func (r *WorkloadReconciler) Finalize(ctx context.Context, obj client.Object) (f } listOpts := client.MatchingFields{ - deploymentWorkloadUID: string(obj.GetUID()), + deploymentWorkloadUIDIndex: string(obj.GetUID()), } var deployments computev1alpha.WorkloadDeploymentList if err := cl.GetClient().List(ctx, &deployments, listOpts); err != nil { @@ -334,21 +347,9 @@ func (r *WorkloadReconciler) Finalize(ctx context.Context, obj client.Object) (f if deployment.DeletionTimestamp.IsZero() { // Deletion will result in another reconcile of the workload, where we // will remove the finalizers. - if err := cl.GetClient().Delete(ctx, &deployment); err != nil { - if apierrors.IsNotFound(err) { - // List cache was not up to date - continue - } + if err := cl.GetClient().Delete(ctx, &deployment); client.IgnoreNotFound(err) != nil { return finalizer.Result{}, fmt.Errorf("failed deleting workload deployment: %w", err) } - } else { - // Remove the finalizer if it's still present - if controllerutil.RemoveFinalizer(&deployment, workloadControllerFinalizer) { - if err := cl.GetClient().Update(ctx, &deployment); err != nil { - return finalizer.Result{}, fmt.Errorf("failed removing finalizer from workload deployment: %w", err) - } - } - } } @@ -368,7 +369,7 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( ) (desired []computev1alpha.WorkloadDeployment, orphaned []computev1alpha.WorkloadDeployment, err error) { listOpts := client.MatchingFields{ - deploymentWorkloadUID: string(workload.UID), + deploymentWorkloadUIDIndex: string(workload.UID), } var deployments computev1alpha.WorkloadDeploymentList if err := upstreamClient.List(ctx, &deployments, listOpts); err != nil { @@ -423,9 +424,6 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( Labels: map[string]string{ computev1alpha.WorkloadUIDLabel: string(workload.UID), }, - Finalizers: []string{ - workloadControllerFinalizer, - }, }, Spec: computev1alpha.WorkloadDeploymentSpec{ WorkloadRef: computev1alpha.WorkloadReference{ @@ -462,22 +460,46 @@ func (r *WorkloadReconciler) SetupWithManager(mgr mcmanager.Manager) error { return fmt.Errorf("failed to register finalizer: %w", err) } - // TODO(jreese) move to indexer package - - err := mgr.GetFieldIndexer().IndexField(context.Background(), &computev1alpha.WorkloadDeployment{}, deploymentWorkloadUID, func(o client.Object) []string { - return []string{ - string(o.(*computev1alpha.WorkloadDeployment).Spec.WorkloadRef.UID), - } - }) - if err != nil { - return fmt.Errorf("failed to add workload deployment field indexer: %w", err) - } - - // TODO(jreese) add watch against networks that triggers a reconcile for - // workloads that are attached and are in an error state for networks not - // existing. return mcbuilder.ControllerManagedBy(mgr). For(&computev1alpha.Workload{}, mcbuilder.WithEngageWithLocalCluster(false)). Owns(&computev1alpha.WorkloadDeployment{}, mcbuilder.WithEngageWithLocalCluster(false)). + Watches(&networkingv1alpha.Network{}, func(clusterName string, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, network client.Object) []mcreconcile.Request { + logger := log.FromContext(ctx) + + cluster, err := mgr.GetCluster(ctx, clusterName) + if err != nil { + logger.Error(err, "failed to get cluster") + return nil + } + clusterClient := cluster.GetClient() + + networkName := client.ObjectKeyFromObject(network).String() + listOpts := client.MatchingFields{ + workloadNetworksIndex: networkName, + } + + var workloads computev1alpha.WorkloadList + if err := clusterClient.List(ctx, &workloads, listOpts); err != nil { + logger.Error(err, "failed to list workloads") + return nil + } + + var requests []mcreconcile.Request + for _, workload := range workloads.Items { + requests = append(requests, mcreconcile.Request{ + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: workload.Namespace, + Name: workload.Name, + }, + }, + ClusterName: clusterName, + }) + } + + return requests + }) + }). Complete(r) } diff --git a/internal/controller/workloaddeployment_controller.go b/internal/controller/workloaddeployment_controller.go index e3ff994a..30ae3a2f 100644 --- a/internal/controller/workloaddeployment_controller.go +++ b/internal/controller/workloaddeployment_controller.go @@ -4,14 +4,23 @@ package controller import ( "context" + "errors" "fmt" + "slices" apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + kerrors "k8s.io/apimachinery/pkg/util/errors" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/finalizer" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" mccontext "sigs.k8s.io/multicluster-runtime/pkg/context" mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" @@ -19,11 +28,15 @@ import ( networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" computev1alpha "go.datum.net/workload-operator/api/v1alpha" + + "go.datum.net/workload-operator/internal/controller/instancecontrol" + instancecontrolstateful "go.datum.net/workload-operator/internal/controller/instancecontrol/stateful" ) // WorkloadDeploymentReconciler reconciles a WorkloadDeployment object type WorkloadDeploymentReconciler struct { - mgr mcmanager.Manager + mgr mcmanager.Manager + finalizers finalizer.Finalizers } // +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloaddeployments,verbs=get;list;watch;create;update;patch;delete @@ -48,6 +61,24 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco return ctrl.Result{}, err } + finalizationResult, err := r.finalizers.Finalize(ctx, &deployment) + if err != nil { + if v, ok := err.(kerrors.Aggregate); ok && v.Is(errDeploymentHasInstances) { + // Don't produce an error in this case and let the watch on deployments + // result in another reconcile schedule. + logger.Info("deployment still has instances, waiting until removal") + return ctrl.Result{}, nil + } else { + return ctrl.Result{}, fmt.Errorf("failed to finalize: %w", err) + } + } + if finalizationResult.Updated { + if err = cl.GetClient().Update(ctx, &deployment); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to update based on finalization result: %w", err) + } + return ctrl.Result{}, nil + } + if !deployment.DeletionTimestamp.IsZero() { return ctrl.Result{}, nil } @@ -55,16 +86,140 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco logger.Info("reconciling deployment") defer logger.Info("reconcile complete") - // Ensure that a `NetworkBinding` is created for each network interface's - // network. - if deployment.Status.Location == nil { return ctrl.Result{}, nil } - // TODO(jreese) shortcut work on a status condition for network bindings - // being ready + // Collect all instances for this deployment + listOpts := client.MatchingLabels{ + computev1alpha.WorkloadDeploymentUIDLabel: string(deployment.GetUID()), + } + + var instances computev1alpha.InstanceList + if err := cl.GetClient().List(ctx, &instances, listOpts); err != nil { + return ctrl.Result{}, fmt.Errorf("failed listing instances: %w", err) + } + + instanceControl := instancecontrolstateful.New() + + actions, err := instanceControl.GetActions(ctx, cl.GetScheme(), &deployment, instances.Items) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed getting instance control actions: %w", err) + } + + logger.Info("collected instance control actions", "count", len(actions)) + for _, action := range actions { + // We don't need to actually check this, but it'll reduce log noise. + if action.IsSkipped() { + continue + } + + logger.Info("instance control action", "instance", action.Object.GetName(), "action", action.ActionType()) + + if err := action.Execute(ctx, cl.GetClient()); err != nil { + return ctrl.Result{}, fmt.Errorf("failed executing instance control action: %w", err) + } + } + + networkReady, err := r.reconcileNetworks(ctx, cl.GetClient(), &deployment) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed reconciling networks: %w", err) + } + + // Networks are all ready with subnets ready to use, remove any scheduling + // gates on instances. If any instances were created by actions above, those + // will result in this reconciler being processed again, so will properly have + // their gates removed. + + replicas := len(instances.Items) + currentReplicas := 0 + desiredReplicas := deployment.Spec.ScaleSettings.MinReplicas + if dt := deployment.DeletionTimestamp; !dt.IsZero() { + desiredReplicas = 0 + } + + readyReplicas := 0 + for _, instance := range instances.Items { + if networkReady && len(instance.Spec.Controller.SchedulingGates) > 0 { + newGates := slices.DeleteFunc(instance.Spec.Controller.SchedulingGates, func(gate computev1alpha.SchedulingGate) bool { + return gate.Name == instancecontrol.NetworkSchedulingGate.String() + }) + + if len(newGates) != len(instance.Spec.Controller.SchedulingGates) { + if _, err := controllerutil.CreateOrPatch(ctx, cl.GetClient(), &instance, func() error { + instance.Spec.Controller.SchedulingGates = newGates + return nil + }); err != nil { + return ctrl.Result{}, fmt.Errorf("failed updating instance: %w", err) + } + } + } + + if apimeta.IsStatusConditionTrue(instance.Status.Conditions, computev1alpha.InstanceProgrammed) { + if instance.Status.Controller.ObservedTemplateHash == instancecontrol.ComputeHash(deployment.Spec.Template) { + currentReplicas++ + } + } + + if apimeta.IsStatusConditionTrue(instance.Status.Conditions, computev1alpha.InstanceReady) { + readyReplicas++ + } + } + + patchResult, err := controllerutil.CreateOrPatch(ctx, cl.GetClient(), &deployment, func() error { + deployment.Status.Replicas = int32(replicas) + deployment.Status.CurrentReplicas = int32(currentReplicas) + deployment.Status.DesiredReplicas = desiredReplicas + deployment.Status.ReadyReplicas = int32(readyReplicas) + + if readyReplicas > 0 { + apimeta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ + Type: computev1alpha.WorkloadDeploymentAvailable, + Status: metav1.ConditionTrue, + Reason: "StableInstanceFound", + Message: fmt.Sprintf("%d/%d instances are ready", readyReplicas, replicas), + }) + } else if !networkReady { + apimeta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ + Type: computev1alpha.WorkloadDeploymentAvailable, + Status: metav1.ConditionFalse, + Reason: "ProvisioningNetwork", + Message: "Network is being provisioned", + }) + } else if replicas > 0 { + apimeta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ + Type: computev1alpha.WorkloadDeploymentAvailable, + Status: metav1.ConditionFalse, + Reason: "ProvisioningInstances", + Message: "Instances are being provisioned", + }) + } + + return nil + }) + + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed updating deployment status: %w", err) + } + + logger.Info("deployment status processed", "operation_result", patchResult) + + return ctrl.Result{}, nil +} + +func (r *WorkloadDeploymentReconciler) reconcileNetworks( + ctx context.Context, + c client.Client, + deployment *computev1alpha.WorkloadDeployment, +) (bool, error) { + logger := log.FromContext(ctx) + + // First, ensure we have a NetworkBinding for each interface, and that the + // binding is ready before we move on to create SubnetClaims. + + var networkContextRefs []networkingv1alpha.NetworkContextRef + allNetworkBindingsReady := true for i, networkInterface := range deployment.Spec.Template.Spec.NetworkInterfaces { var networkBinding networkingv1alpha.NetworkBinding networkBindingObjectKey := client.ObjectKey{ @@ -72,8 +227,8 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco Name: fmt.Sprintf("%s-net-%d", deployment.Name, i), } - if err := cl.GetClient().Get(ctx, networkBindingObjectKey, &networkBinding); client.IgnoreNotFound(err) != nil { - return ctrl.Result{}, fmt.Errorf("failed checking for existing network binding: %w", err) + if err := c.Get(ctx, networkBindingObjectKey, &networkBinding); client.IgnoreNotFound(err) != nil { + return false, fmt.Errorf("failed checking for existing network binding: %w", err) } if networkBinding.CreationTimestamp.IsZero() { @@ -88,25 +243,255 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco }, } - if err := controllerutil.SetControllerReference(&deployment, &networkBinding, cl.GetScheme()); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to set controller on network binding: %w", err) + if err := controllerutil.SetControllerReference(deployment, &networkBinding, c.Scheme()); err != nil { + return false, fmt.Errorf("failed to set controller on network binding: %w", err) } - if err := cl.GetClient().Create(ctx, &networkBinding); err != nil { - return ctrl.Result{}, fmt.Errorf("failed creating network binding: %w", err) + if err := c.Create(ctx, &networkBinding); err != nil { + return false, fmt.Errorf("failed creating network binding: %w", err) } } + if !apimeta.IsStatusConditionTrue(networkBinding.Status.Conditions, networkingv1alpha.NetworkBindingReady) { + allNetworkBindingsReady = false + } else if networkBinding.Status.NetworkContextRef != nil { + networkContextRefs = append(networkContextRefs, *networkBinding.Status.NetworkContextRef) + } } - return ctrl.Result{}, nil + if !allNetworkBindingsReady { + logger.Info("waiting for network bindings to be ready") + return false, nil + } + + // TODO(jreese): Currently this makes a SubnetClaim that will be used by + // many instances. Move to a claim per instance interface, and allocate from + // a larger subnet. In addition, it does not handle allocation of more than + // one subnet per network context. We'll have a future IPAM controller in + // network-services-operator that will handle this. + // + // Also, only handling ipv4 + + for _, networkContextRef := range networkContextRefs { + var networkContext networkingv1alpha.NetworkContext + networkContextObjectKey := client.ObjectKey{ + Namespace: networkContextRef.Namespace, + Name: networkContextRef.Name, + } + + if err := c.Get(ctx, networkContextObjectKey, &networkContext); client.IgnoreNotFound(err) != nil { + return false, fmt.Errorf("failed checking for existing network context: %w", err) + } + + if !apimeta.IsStatusConditionTrue(networkContext.Status.Conditions, networkingv1alpha.NetworkContextReady) { + logger.Info("waiting for network context to be ready", "network_context", networkContext.Name) + return false, nil + } + + var subnetClaims networkingv1alpha.SubnetClaimList + listOpts := []client.ListOption{ + client.InNamespace(networkContext.Namespace), + } + + if err := c.List(ctx, &subnetClaims, listOpts...); err != nil { + return false, fmt.Errorf("failed listing subnet claims: %w", err) + } + + var subnetClaim networkingv1alpha.SubnetClaim + for _, claim := range subnetClaims.Items { + // If it's not the same subnet class, don't consider the subnet claim. + if claim.Spec.SubnetClass != "private" { + continue + } + + // If it's not ipv4, don't consider the subnet claim. + if claim.Spec.IPFamily != networkingv1alpha.IPv4Protocol { + continue + } + + // If it's not the same network context, don't consider the subnet claim. + if claim.Spec.NetworkContext.Name != networkContext.Name { + continue + } + + // If it's not the same location, don't consider the subnet claim. + if claim.Spec.Location.Namespace != deployment.Status.Location.Namespace || + claim.Spec.Location.Name != deployment.Status.Location.Name { + continue + } + + subnetClaim = claim + break + } + + if subnetClaim.CreationTimestamp.IsZero() { + subnetClaim = networkingv1alpha.SubnetClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: networkContext.Namespace, + // In the future, subnets will be created with an ordinal that increases. + // This ensures that we don't create duplicate subnet claims when the + // cache is not up to date. + Name: fmt.Sprintf("%s-0", networkContext.Name), + }, + Spec: networkingv1alpha.SubnetClaimSpec{ + SubnetClass: "private", + IPFamily: networkingv1alpha.IPv4Protocol, + NetworkContext: networkingv1alpha.LocalNetworkContextRef{ + Name: networkContext.Name, + }, + Location: *deployment.Status.Location, + }, + } + + if err := controllerutil.SetOwnerReference(&networkContext, &subnetClaim, c.Scheme()); err != nil { + return false, fmt.Errorf("failed to set controller on subnet claim: %w", err) + } + + if err := c.Create(ctx, &subnetClaim); err != nil { + return false, fmt.Errorf("failed creating subnet claim: %w", err) + } + + logger.Info("created subnet claim", "subnetClaim", subnetClaim.Name) + + return false, nil + } + + logger.Info("found subnet claim", "subnetClaim", subnetClaim.Name) + + if !apimeta.IsStatusConditionTrue(subnetClaim.Status.Conditions, "Ready") { + logger.Info("waiting for subnet claim to be ready", "subnetClaim", subnetClaim.Name) + return false, nil + } + + var subnet networkingv1alpha.Subnet + subnetObjectKey := client.ObjectKey{ + Namespace: subnetClaim.Namespace, + Name: subnetClaim.Status.SubnetRef.Name, + } + if err := c.Get(ctx, subnetObjectKey, &subnet); err != nil { + return false, fmt.Errorf("failed fetching subnet: %w", err) + } + + if !apimeta.IsStatusConditionTrue(subnet.Status.Conditions, "Ready") { + logger.Info("waiting for subnet to be ready", "subnet", subnet.Name) + return false, nil + } + + logger.Info("subnet is ready", "subnet", subnet.Name) + + } + + return true, nil +} + +var errDeploymentHasInstances = errors.New("deployment has instances") + +func (r *WorkloadDeploymentReconciler) Finalize(ctx context.Context, obj client.Object) (finalizer.Result, error) { + clusterName, ok := mccontext.ClusterFrom(ctx) + if !ok { + return finalizer.Result{}, fmt.Errorf("cluster name not found in context") + } + + cl, err := r.mgr.GetCluster(ctx, clusterName) + if err != nil { + return finalizer.Result{}, err + } + + var instanceList computev1alpha.InstanceList + listOpts := []client.ListOption{ + client.InNamespace(obj.GetNamespace()), + client.MatchingLabels{ + computev1alpha.WorkloadDeploymentUIDLabel: string(obj.GetUID()), + }, + } + + if err := cl.GetClient().List(ctx, &instanceList, listOpts...); err != nil { + return finalizer.Result{}, fmt.Errorf("failed listing instances: %w", err) + } + + if len(instanceList.Items) == 0 { + log.FromContext(ctx).Info("instances have been removed") + return finalizer.Result{}, nil + } + + // All instances need to be deleted before the deployment may be deleted + for _, instance := range instanceList.Items { + if instance.DeletionTimestamp.IsZero() { + if err := cl.GetClient().Delete(ctx, &instance); client.IgnoreNotFound(err) != nil { + return finalizer.Result{}, fmt.Errorf("failed deleting instance: %w", err) + } + } + } + + // Really don't like using errors for communication here. I think we'd need + // to move away from the finalizer helper to ensure we can wait on child + // resources to be gone before allowing the finalizer to be removed. + return finalizer.Result{}, errDeploymentHasInstances } // SetupWithManager sets up the controller with the Manager. func (r *WorkloadDeploymentReconciler) SetupWithManager(mgr mcmanager.Manager) error { r.mgr = mgr - // TODO(jreese) finalizers + r.finalizers = finalizer.NewFinalizers() + if err := r.finalizers.Register(workloadControllerFinalizer, r); err != nil { + return fmt.Errorf("failed to register finalizer: %w", err) + } return mcbuilder.ControllerManagedBy(mgr). For(&computev1alpha.WorkloadDeployment{}, mcbuilder.WithEngageWithLocalCluster(false)). + Owns(&computev1alpha.Instance{}). + Owns(&networkingv1alpha.NetworkBinding{}). + Watches(&networkingv1alpha.SubnetClaim{}, func(clusterName string, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []mcreconcile.Request { + subnetClaim := o.(*networkingv1alpha.SubnetClaim) + return enqueueWorkloadDeploymentByLocation(ctx, mgr, clusterName, subnetClaim.Spec.Location) + }) + }). + Watches(&networkingv1alpha.Subnet{}, func(clusterName string, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []mcreconcile.Request { + subnet := o.(*networkingv1alpha.Subnet) + return enqueueWorkloadDeploymentByLocation(ctx, mgr, clusterName, subnet.Spec.Location) + }) + }). Complete(r) } + +func enqueueWorkloadDeploymentByLocation(ctx context.Context, mgr mcmanager.Manager, clusterName string, locationRef networkingv1alpha.LocationReference) []mcreconcile.Request { + logger := log.FromContext(ctx) + + cluster, err := mgr.GetCluster(ctx, clusterName) + if err != nil { + logger.Error(err, "failed to get cluster") + return nil + } + clusterClient := cluster.GetClient() + + locationName := (types.NamespacedName{ + Namespace: locationRef.Namespace, + Name: locationRef.Name, + }).String() + listOpts := client.MatchingFields{ + deploymentLocationIndex: locationName, + } + + var workloadDeployments computev1alpha.WorkloadDeploymentList + + if err := clusterClient.List(ctx, &workloadDeployments, listOpts); err != nil { + logger.Error(err, "failed to list workloads") + return nil + } + + requests := make([]mcreconcile.Request, 0, len(workloadDeployments.Items)) + for _, workload := range workloadDeployments.Items { + requests = append(requests, mcreconcile.Request{ + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: workload.Namespace, + Name: workload.Name, + }, + }, + ClusterName: clusterName, + }) + } + + return requests +} diff --git a/internal/providers/datum/provider_test.go b/internal/providers/datum/provider_test.go index d748fb93..7064737e 100644 --- a/internal/providers/datum/provider_test.go +++ b/internal/providers/datum/provider_test.go @@ -43,7 +43,6 @@ func TestNotReadyProject(t *testing.T) { result, err := provider.Reconcile(context.Background(), req) assert.NoError(t, err, "unexpected error returned from reconciler") - assert.Equal(t, false, result.Requeue) assert.Zero(t, result.RequeueAfter) assert.Len(t, provider.projects, 0) } @@ -57,7 +56,6 @@ func TestReadyProject(t *testing.T) { result, err := provider.Reconcile(context.Background(), req) assert.NoError(t, err, "unexpected error returned from reconciler") - assert.Equal(t, false, result.Requeue) assert.Zero(t, result.RequeueAfter) assert.Len(t, provider.projects, 1) diff --git a/internal/webhook/v1alpha/workload_webhook.go b/internal/webhook/v1alpha/workload_webhook.go index 8692ba56..ebf27537 100644 --- a/internal/webhook/v1alpha/workload_webhook.go +++ b/internal/webhook/v1alpha/workload_webhook.go @@ -5,11 +5,11 @@ import ( "fmt" - "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" @@ -24,8 +24,7 @@ import ( func SetupWorkloadWebhookWithManager(mgr mcmanager.Manager) error { webhook := &workloadWebhook{ - mgr: mgr, - logger: mgr.GetLogger(), + mgr: mgr, } return ctrl.NewWebhookManagedBy(mgr.GetLocalManager()). @@ -38,8 +37,7 @@ func SetupWorkloadWebhookWithManager(mgr mcmanager.Manager) error { // +kubebuilder:webhook:path=/mutate-compute-datumapis-com-v1alpha-workload,mutating=true,failurePolicy=fail,sideEffects=None,groups=compute.datumapis.com,resources=workloads,verbs=create;update,versions=v1alpha,name=mworkload.kb.io,admissionReviewVersions=v1 type workloadWebhook struct { - mgr mcmanager.Manager - logger logr.Logger + mgr mcmanager.Manager } var _ admission.CustomDefaulter = &workloadWebhook{} @@ -91,6 +89,9 @@ func (r *workloadWebhook) ValidateCreate(ctx context.Context, obj runtime.Object } clusterClient := cluster.GetClient() + logger := logf.FromContext(ctx).WithValues("cluster", clusterName) + logger.Info("Validating Workload Create", "name", workload.GetName(), "cluster", clusterName) + req, err := admission.RequestFromContext(ctx) if err != nil { return nil, err