diff --git a/api/v1alpha/instance_types.go b/api/v1alpha/instance_types.go index 00ecda18..3d9f38ea 100644 --- a/api/v1alpha/instance_types.go +++ b/api/v1alpha/instance_types.go @@ -26,8 +26,37 @@ type InstanceSpec struct { // +kubebuilder:validation:Optional // +listType=map // +listMapKey=name - Volumes []InstanceVolume `json:"volumes,omitempty"` + + // The location which the instance has been scheduled to + // + // +kubebuilder:validation:Optional + Location *networkingv1alpha.LocationReference `json:"location,omitempty"` + + // Controller contains settings driven by the controller managing the instance. + // + // +kubebuilder:validation:Optional + Controller *InstanceController `json:"controller,omitempty"` +} + +type InstanceController struct { + // TemplateHash is the hash of the instance template applied for this instance. + // + // +kubebuilder:validation:Required + TemplateHash string `json:"templateHash"` + + // SchedulingGates is a list of gates that must be satisfied before the + // instance can be scheduled. + // + // +kubebuilder:validation:Optional + // +listType=map + // +listMapKey=name + SchedulingGates []SchedulingGate `json:"schedulingGates,omitempty"` +} + +type SchedulingGate struct { + // The name of the gate. + Name string `json:"name"` } type InstanceRuntimeSpec struct { @@ -337,8 +366,60 @@ type InstanceStatus struct { // Network interface information NetworkInterfaces []InstanceNetworkInterfaceStatus `json:"networkInterfaces,omitempty"` + + // Controller contains status information about the controller managing the instance. + // + // +kubebuilder:validation:Optional + Controller *InstanceControllerStatus `json:"controller,omitempty"` } +type InstanceControllerStatus struct { + // ObservedTemplateHash is the hash of the instance template applied for this instance. + // + // +kubebuilder:validation:Required + ObservedTemplateHash string `json:"observedTemplateHash"` +} + +const ( + // InstanceReady indicates that the instance is ready + InstanceReady = "Ready" + + // InstanceRunning indicates that the instance is running + InstanceRunning = "Running" + + // InstanceProgrammed indicates that the instance has been programmed + InstanceProgrammed = "Programmed" +) + +const ( + // InstanceReadyReasonSchedulingGatesPresent indicates that the instance is not ready because scheduling gates are present. + InstanceReadyReasonSchedulingGatesPresent = "SchedulingGatesPresent" + + // InstanceReadyReasonRunning indicates that the instance is running + InstanceReadyReasonRunning = "Running" + + // InstanceRunningReasonStopped indicates that the instance is stopped + InstanceRunningReasonStopped = "Stopped" + + // InstanceRunningReasonStarting indicates that the instance is starting + InstanceRunningReasonStarting = "Starting" + + // InstanceRunningReasonStopping indicates that the instance is stopping + InstanceRunningReasonStopping = "Stopping" + + // InstanceRunningReasonRunning indicates that the instance is running + InstanceRunningReasonRunning = "Running" + + // InstanceProgrammedReasonPendingProgramming indicates that the instance has not been programmed + InstanceProgrammedReasonPendingProgramming = "PendingProgramming" + + // InstanceProgrammedReasonProgrammingInProgress indicates that the instance is being programmed. + InstanceProgrammedReasonProgrammingInProgress = "ProgrammingInProgress" + + // InstanceProgrammedReasonProgrammed indicates that the instance has been programmed + InstanceProgrammedReasonProgrammed = "Programmed" +) + type InstanceTemplateSpec struct { // Metadata of the instances created from this template // @@ -355,15 +436,21 @@ type InstanceTemplateSpec struct { // Instance is the Schema for the instances API // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" -// +kubebuilder:printcolumn:name="Available",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].status` -// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].reason` -// +kubebuilder:printcolumn:name="Network IP",type=string,JSONPath=`.status.networkInterfaces[0].assignments.networkIP`,priority=1 -// +kubebuilder:printcolumn:name="External IP",type=string,JSONPath=`.status.networkInterfaces[0].assignments.externalIP`,priority=1 +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason` +// +kubebuilder:printcolumn:name="Network IP",type=string,JSONPath=`.status.networkInterfaces[0].assignments.networkIP` +// +kubebuilder:printcolumn:name="External IP",type=string,JSONPath=`.status.networkInterfaces[0].assignments.externalIP` +// +kubebuilder:printcolumn:name="Message",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].message`,priority=1 type Instance struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec InstanceSpec `json:"spec,omitempty"` + // Spec defines the desired state of an Instance. + Spec InstanceSpec `json:"spec,omitempty"` + + // Status defines the current state of an Instance. + // + // +kubebuilder:default={conditions:{{type:"Programmed",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Running",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"},{type:"Ready",status:"Unknown",reason:"Pending", message:"Waiting for controller", lastTransitionTime: "1970-01-01T00:00:00Z"}}} Status InstanceStatus `json:"status,omitempty"` } diff --git a/api/v1alpha/labels.go b/api/v1alpha/labels.go index 9d308927..e1dac308 100644 --- a/api/v1alpha/labels.go +++ b/api/v1alpha/labels.go @@ -5,4 +5,6 @@ const ( WorkloadUIDLabel = LabelNamespace + "/workload-uid" WorkloadDeploymentUIDLabel = LabelNamespace + "/workload-deployment-uid" + + InstanceIndexLabel = LabelNamespace + "/instance-index" ) diff --git a/api/v1alpha/workload_types.go b/api/v1alpha/workload_types.go index 2156037c..da34a1c9 100644 --- a/api/v1alpha/workload_types.go +++ b/api/v1alpha/workload_types.go @@ -52,17 +52,20 @@ type WorkloadStatus struct { // Known condition types are: "Available", "Progressing" Conditions []metav1.Condition `json:"conditions,omitempty"` - // The number of instances created by a placement + // The number of deployments that currently exist + Deployments int32 `json:"deployments"` + + // The number of instances that currently exist Replicas int32 `json:"replicas"` - // The number of instances created by a placement and have the latest - // workload generation settings applied. + // The number of instances which have the latest workload settings applied. CurrentReplicas int32 `json:"currentReplicas"` - // The desired number of instances to be managed by a placement. + // The desired number of instances DesiredReplicas int32 `json:"desiredReplicas"` - // TODO(jreese) ReadyReplicas? + // The number of instances which are ready. + ReadyReplicas int32 `json:"readyReplicas"` // The current status of placemetns in a workload. Placements []WorkloadPlacementStatus `json:"placements,omitempty"` @@ -91,6 +94,11 @@ type WorkloadGatewayStatus struct { // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // +kubebuilder:printcolumn:name="Available",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].status` // +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].reason` +// +kubebuilder:printcolumn:name="Deployments",type=string,JSONPath=`.status.deployments` +// +kubebuilder:printcolumn:name="Replicas",type=string,JSONPath=`.status.replicas` +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.readyReplicas` +// +kubebuilder:printcolumn:name="Desired",type=string,JSONPath=`.status.desiredReplicas` +// +kubebuilder:printcolumn:name="Up-to-date",type=string,JSONPath=`.status.currentReplicas` type Workload struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -134,17 +142,17 @@ type WorkloadPlacementStatus struct { // Known condition types are: "Available", "Progressing" Conditions []metav1.Condition `json:"conditions,omitempty"` - // The number of instances created by a placement + // The number of instances that currently exist Replicas int32 `json:"replicas"` - // The number of instances created by a placement and have the latest - // workload generation settings applied. + // The number of instances which have the latest workload settings applied. CurrentReplicas int32 `json:"currentReplicas"` - // The desired number of instances to be managed by a placement. + // The desired number of instances DesiredReplicas int32 `json:"desiredReplicas"` - // TODO(jreese) ReadyReplicas? + // The number of instances which are ready. + ReadyReplicas int32 `json:"readyReplicas"` } type HorizontalScaleSettings struct { @@ -165,8 +173,25 @@ type HorizontalScaleSettings struct { // TODO(jreese) wire in behavior // See https://github.com/kubernetes/kubernetes/blob/dd87bc064631354885193fc1a97d0e7b603e77b4/staging/src/k8s.io/api/autoscaling/v2/types.go#L84 + // Defines the policy for managing instances. + + // TODO(jreese) Add instance update policy? RollingUpdate vs OrderedReady + + // Controls how instances are managed during scale up and down, as well as + // during maintenance events. + // + // +kubebuilder:validation:Required + // +kubebuilder:default=OrderedReady + InstanceManagementPolicy InstanceManagementPolicyType `json:"instanceManagementPolicy,omitempty"` } +type InstanceManagementPolicyType string + +const ( + OrderedReadyInstanceManagementPolicyType InstanceManagementPolicyType = "OrderedReady" + // ParallelInstanceManagementPolicyType InstanceManagementPolicyType = "Parallel" +) + type MetricSpec struct { // Resource metrics known to Datum. // diff --git a/api/v1alpha/workloaddeployment_types.go b/api/v1alpha/workloaddeployment_types.go index 7cf8a0d4..76b6179e 100644 --- a/api/v1alpha/workloaddeployment_types.go +++ b/api/v1alpha/workloaddeployment_types.go @@ -46,17 +46,17 @@ type WorkloadDeploymentStatus struct { // Known condition types are: "Available", "Progressing" Conditions []metav1.Condition `json:"conditions,omitempty"` - // The number of instances created by a deployment + // The number of instances created Replicas int32 `json:"replicas"` - // The number of instances created by a deployment and have the latest - // deployment generation settings applied. + // The number of instances which have the latest workload settings applied. CurrentReplicas int32 `json:"currentReplicas"` - // The desired number of instances to be managed by a deployment. + // The desired number of instances DesiredReplicas int32 `json:"desiredReplicas"` - // TODO(jreese) ReadyReplicas? + // The number of instances which are ready. + ReadyReplicas int32 `json:"readyReplicas"` } const ( @@ -70,10 +70,14 @@ const ( // WorkloadDeployment is the Schema for the workloaddeployments API // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" -// +kubebuilder:printcolumn:name="Location Namespce",type=string,JSONPath=`.status.location.namespace` -// +kubebuilder:printcolumn:name="Location Name",type=string,JSONPath=`.status.location.name` // +kubebuilder:printcolumn:name="Available",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].status` // +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].reason` +// +kubebuilder:printcolumn:name="Replicas",type=string,JSONPath=`.status.replicas` +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.readyReplicas` +// +kubebuilder:printcolumn:name="Desired",type=string,JSONPath=`.status.desiredReplicas` +// +kubebuilder:printcolumn:name="Up-to-date",type=string,JSONPath=`.status.currentReplicas` +// +kubebuilder:printcolumn:name="Location Namespace",type=string,JSONPath=`.status.location.namespace`,priority=1 +// +kubebuilder:printcolumn:name="Location Name",type=string,JSONPath=`.status.location.name`,priority=1 type WorkloadDeployment struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index ffbf3a27..8ecc1bae 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -241,6 +241,41 @@ func (in *Instance) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InstanceController) DeepCopyInto(out *InstanceController) { + *out = *in + if in.SchedulingGates != nil { + in, out := &in.SchedulingGates, &out.SchedulingGates + *out = make([]SchedulingGate, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceController. +func (in *InstanceController) DeepCopy() *InstanceController { + if in == nil { + return nil + } + out := new(InstanceController) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InstanceControllerStatus) DeepCopyInto(out *InstanceControllerStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceControllerStatus. +func (in *InstanceControllerStatus) DeepCopy() *InstanceControllerStatus { + if in == nil { + return nil + } + out := new(InstanceControllerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InstanceList) DeepCopyInto(out *InstanceList) { *out = *in @@ -423,6 +458,16 @@ func (in *InstanceSpec) DeepCopyInto(out *InstanceSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Location != nil { + in, out := &in.Location, &out.Location + *out = new(apiv1alpha.LocationReference) + **out = **in + } + if in.Controller != nil { + in, out := &in.Controller, &out.Controller + *out = new(InstanceController) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceSpec. @@ -452,6 +497,11 @@ func (in *InstanceStatus) DeepCopyInto(out *InstanceStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Controller != nil { + in, out := &in.Controller, &out.Controller + *out = new(InstanceControllerStatus) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceStatus. @@ -666,6 +716,21 @@ func (in *SandboxRuntime) DeepCopy() *SandboxRuntime { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SchedulingGate) DeepCopyInto(out *SchedulingGate) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SchedulingGate. +func (in *SchedulingGate) DeepCopy() *SchedulingGate { + if in == nil { + return nil + } + out := new(SchedulingGate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualMachineRuntime) DeepCopyInto(out *VirtualMachineRuntime) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index acb5d6b6..11bc9de5 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -180,6 +180,11 @@ func main() { // +kubebuilder:scaffold:builder + if err = controller.AddIndexers(ctx, mgr); err != nil { + setupLog.Error(err, "unable to add indexers") + os.Exit(1) + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/config/crd/bases/compute.datumapis.com_instances.yaml b/config/crd/bases/compute.datumapis.com_instances.yaml index 0115f0d6..07a61f37 100644 --- a/config/crd/bases/compute.datumapis.com_instances.yaml +++ b/config/crd/bases/compute.datumapis.com_instances.yaml @@ -18,18 +18,20 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date - - jsonPath: .status.conditions[?(@.type=="Available")].status - name: Available + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready type: string - - jsonPath: .status.conditions[?(@.type=="Available")].reason + - jsonPath: .status.conditions[?(@.type=="Ready")].reason name: Reason type: string - jsonPath: .status.networkInterfaces[0].assignments.networkIP name: Network IP - priority: 1 type: string - jsonPath: .status.networkInterfaces[0].assignments.externalIP name: External IP + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].message + name: Message priority: 1 type: string name: v1alpha @@ -55,8 +57,48 @@ spec: metadata: type: object spec: - description: InstanceSpec defines the desired state of Instance + description: Spec defines the desired state of an Instance. properties: + controller: + description: Controller contains settings driven by the controller + managing the instance. + properties: + schedulingGates: + description: |- + SchedulingGates is a list of gates that must be satisfied before the + instance can be scheduled. + items: + properties: + name: + description: The name of the gate. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + templateHash: + description: TemplateHash is the hash of the instance template + applied for this instance. + type: string + required: + - templateHash + type: object + location: + description: The location which the instance has been scheduled to + properties: + name: + description: Name of a datum location + type: string + namespace: + description: Namespace for the datum location + type: string + required: + - name + - namespace + type: object networkInterfaces: description: Network interface configuration. items: @@ -519,6 +561,9 @@ spec: - resources type: object volumes: + description: |- + Volumes that must be available to attach to an instance's containers or + Virtual Machine. items: properties: configMap: @@ -766,7 +811,24 @@ spec: - runtime type: object status: - description: InstanceStatus defines the observed state of Instance + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Programmed + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Running + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for controller + reason: Pending + status: Unknown + type: Ready + description: Status defines the current state of an Instance. properties: conditions: description: |- @@ -827,6 +889,17 @@ spec: - type type: object type: array + controller: + description: Controller contains status information about the controller + managing the instance. + properties: + observedTemplateHash: + description: ObservedTemplateHash is the hash of the instance + template applied for this instance. + type: string + required: + - observedTemplateHash + type: object networkInterfaces: description: Network interface information items: diff --git a/config/crd/bases/compute.datumapis.com_workloaddeployments.yaml b/config/crd/bases/compute.datumapis.com_workloaddeployments.yaml index f42d8489..a13a40e5 100644 --- a/config/crd/bases/compute.datumapis.com_workloaddeployments.yaml +++ b/config/crd/bases/compute.datumapis.com_workloaddeployments.yaml @@ -18,18 +18,32 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date - - jsonPath: .status.location.namespace - name: Location Namespce - type: string - - jsonPath: .status.location.name - name: Location Name - type: string - jsonPath: .status.conditions[?(@.type=="Available")].status name: Available type: string - jsonPath: .status.conditions[?(@.type=="Available")].reason name: Reason type: string + - jsonPath: .status.replicas + name: Replicas + type: string + - jsonPath: .status.readyReplicas + name: Ready + type: string + - jsonPath: .status.desiredReplicas + name: Desired + type: string + - jsonPath: .status.currentReplicas + name: Up-to-date + type: string + - jsonPath: .status.location.namespace + name: Location Namespace + priority: 1 + type: string + - jsonPath: .status.location.name + name: Location Name + priority: 1 + type: string name: v1alpha schema: openAPIV3Schema: @@ -66,6 +80,12 @@ spec: scaleSettings: description: Scale settings such as minimum and maximum replica counts. properties: + instanceManagementPolicy: + default: OrderedReady + description: |- + Controls how instances are managed during scale up and down, as well as + during maintenance events. + type: string maxReplicas: description: The maximum number of replicas. format: int32 @@ -120,6 +140,7 @@ spec: format: int32 type: integer required: + - instanceManagementPolicy - minReplicas type: object template: @@ -148,6 +169,47 @@ spec: spec: description: Describes the desired configuration of an instance properties: + controller: + description: Controller contains settings driven by the controller + managing the instance. + properties: + schedulingGates: + description: |- + SchedulingGates is a list of gates that must be satisfied before the + instance can be scheduled. + items: + properties: + name: + description: The name of the gate. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + templateHash: + description: TemplateHash is the hash of the instance + template applied for this instance. + type: string + required: + - templateHash + type: object + location: + description: The location which the instance has been scheduled + to + properties: + name: + description: Name of a datum location + type: string + namespace: + description: Namespace for the datum location + type: string + required: + - name + - namespace + type: object networkInterfaces: description: Network interface configuration. items: @@ -621,6 +683,9 @@ spec: - resources type: object volumes: + description: |- + Volumes that must be available to attach to an instance's containers or + Virtual Machine. items: properties: configMap: @@ -961,13 +1026,12 @@ spec: type: object type: array currentReplicas: - description: |- - The number of instances created by a deployment and have the latest - deployment generation settings applied. + description: The number of instances which have the latest workload + settings applied. format: int32 type: integer desiredReplicas: - description: The desired number of instances to be managed by a deployment. + description: The desired number of instances format: int32 type: integer location: @@ -984,13 +1048,18 @@ spec: - name - namespace type: object + readyReplicas: + description: The number of instances which are ready. + format: int32 + type: integer replicas: - description: The number of instances created by a deployment + description: The number of instances created format: int32 type: integer required: - currentReplicas - desiredReplicas + - readyReplicas - replicas type: object type: object diff --git a/config/crd/bases/compute.datumapis.com_workloads.yaml b/config/crd/bases/compute.datumapis.com_workloads.yaml index eadbf100..6c7082cc 100644 --- a/config/crd/bases/compute.datumapis.com_workloads.yaml +++ b/config/crd/bases/compute.datumapis.com_workloads.yaml @@ -24,6 +24,21 @@ spec: - jsonPath: .status.conditions[?(@.type=="Available")].reason name: Reason type: string + - jsonPath: .status.deployments + name: Deployments + type: string + - jsonPath: .status.replicas + name: Replicas + type: string + - jsonPath: .status.readyReplicas + name: Ready + type: string + - jsonPath: .status.desiredReplicas + name: Desired + type: string + - jsonPath: .status.currentReplicas + name: Up-to-date + type: string name: v1alpha schema: openAPIV3Schema: @@ -68,6 +83,12 @@ spec: description: Scale settings such as minimum and maximum replica counts. properties: + instanceManagementPolicy: + default: OrderedReady + description: |- + Controls how instances are managed during scale up and down, as well as + during maintenance events. + type: string maxReplicas: description: The maximum number of replicas. format: int32 @@ -122,6 +143,7 @@ spec: format: int32 type: integer required: + - instanceManagementPolicy - minReplicas type: object required: @@ -157,6 +179,47 @@ spec: spec: description: Describes the desired configuration of an instance properties: + controller: + description: Controller contains settings driven by the controller + managing the instance. + properties: + schedulingGates: + description: |- + SchedulingGates is a list of gates that must be satisfied before the + instance can be scheduled. + items: + properties: + name: + description: The name of the gate. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + templateHash: + description: TemplateHash is the hash of the instance + template applied for this instance. + type: string + required: + - templateHash + type: object + location: + description: The location which the instance has been scheduled + to + properties: + name: + description: Name of a datum location + type: string + namespace: + description: Namespace for the datum location + type: string + required: + - name + - namespace + type: object networkInterfaces: description: Network interface configuration. items: @@ -630,6 +693,9 @@ spec: - resources type: object volumes: + description: |- + Volumes that must be available to attach to an instance's containers or + Virtual Machine. items: properties: configMap: @@ -954,13 +1020,16 @@ spec: type: object type: array currentReplicas: - description: |- - The number of instances created by a placement and have the latest - workload generation settings applied. + description: The number of instances which have the latest workload + settings applied. + format: int32 + type: integer + deployments: + description: The number of deployments that currently exist format: int32 type: integer desiredReplicas: - description: The desired number of instances to be managed by a placement. + description: The desired number of instances format: int32 type: integer gateway: @@ -1302,37 +1371,46 @@ spec: type: object type: array currentReplicas: - description: |- - The number of instances created by a placement and have the latest - workload generation settings applied. + description: The number of instances which have the latest workload + settings applied. format: int32 type: integer desiredReplicas: - description: The desired number of instances to be managed by - a placement. + description: The desired number of instances format: int32 type: integer name: description: The name of the placement type: string + readyReplicas: + description: The number of instances which are ready. + format: int32 + type: integer replicas: - description: The number of instances created by a placement + description: The number of instances that currently exist format: int32 type: integer required: - currentReplicas - desiredReplicas - name + - readyReplicas - replicas type: object type: array + readyReplicas: + description: The number of instances which are ready. + format: int32 + type: integer replicas: - description: The number of instances created by a placement + description: The number of instances that currently exist format: int32 type: integer required: - currentReplicas + - deployments - desiredReplicas + - readyReplicas - replicas type: object required: diff --git a/go.mod b/go.mod index 7547276d..09ed49da 100644 --- a/go.mod +++ b/go.mod @@ -1,30 +1,31 @@ module go.datum.net/workload-operator -go 1.23.0 +go 1.24.0 + +toolchain go1.24.1 require ( - github.com/go-logr/logr v1.4.2 + github.com/go-logr/logr v1.4.3 github.com/google/go-cmp v0.7.0 github.com/onsi/ginkgo/v2 v2.23.4 github.com/onsi/gomega v1.37.0 github.com/stretchr/testify v1.10.0 go.datum.net/network-services-operator v0.1.0 golang.org/x/crypto v0.37.0 - golang.org/x/sync v0.13.0 + golang.org/x/sync v0.15.0 google.golang.org/protobuf v1.36.6 - k8s.io/api v0.32.1 - k8s.io/apimachinery v0.32.1 - k8s.io/client-go v0.32.1 - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 - sigs.k8s.io/controller-runtime v0.20.4 + k8s.io/api v0.33.1 + k8s.io/apimachinery v0.33.1 + k8s.io/client-go v0.33.1 + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + sigs.k8s.io/controller-runtime v0.21.0 sigs.k8s.io/gateway-api v1.2.1 - sigs.k8s.io/multicluster-runtime v0.20.4-alpha.6 + sigs.k8s.io/multicluster-runtime v0.21.0-alpha.8 ) require ( - cel.dev/expr v0.18.0 // indirect + cel.dev/expr v0.19.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect - github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect @@ -42,14 +43,12 @@ require ( github.com/go-openapi/swag v0.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect - github.com/google/cel-go v0.22.0 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/gofuzz v1.2.0 // indirect + github.com/google/cel-go v0.23.2 // indirect + github.com/google/gnostic-models v0.6.9 // indirect github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -59,47 +58,49 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect - go.opentelemetry.io/otel v1.29.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.29.0 // indirect - go.opentelemetry.io/otel/sdk v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.29.0 // indirect - go.opentelemetry.io/proto/otlp v1.3.1 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/otel/sdk v1.33.0 // indirect + go.opentelemetry.io/otel/trace v1.33.0 // indirect + go.opentelemetry.io/proto/otlp v1.4.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.38.0 // indirect - golang.org/x/oauth2 v0.24.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sys v0.32.0 // indirect golang.org/x/term v0.31.0 // indirect golang.org/x/text v0.24.0 // indirect - golang.org/x/time v0.7.0 // indirect + golang.org/x/time v0.9.0 // indirect golang.org/x/tools v0.31.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 // indirect - google.golang.org/grpc v1.67.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/grpc v1.68.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.32.1 // indirect - k8s.io/apiserver v0.32.1 // indirect - k8s.io/component-base v0.32.1 // indirect + k8s.io/apiextensions-apiserver v0.33.1 // indirect + k8s.io/apiserver v0.33.1 // indirect + k8s.io/component-base v0.33.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 931eb0d6..360a7b32 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,7 @@ -cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo= -cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= +cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -30,8 +28,8 @@ github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyT github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -50,10 +48,10 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/cel-go v0.22.0 h1:b3FJZxpiv1vTMo2/5RDUqAHPxkT8mmMfJIrq1llbf7g= -github.com/google/cel-go v0.22.0/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4= +github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -64,8 +62,8 @@ github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -74,10 +72,14 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -98,16 +100,16 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= @@ -118,6 +120,8 @@ github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8w github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -130,22 +134,24 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.datum.net/network-services-operator v0.1.0 h1:PAXOZ5DdJFgRoeVBPIXhqkCm6DxbP4tVOPcr3Y7h/So= go.datum.net/network-services-operator v0.1.0/go.mod h1:uloVfxqE+8DgSiMB651X8UC9yECpXbwp/NBstofCceE= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= -go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= -go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= -go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= -go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= -go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= -go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= +go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -169,13 +175,13 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= -golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -187,8 +193,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -201,12 +207,12 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= -google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= -google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= -google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -219,35 +225,40 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= -k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k= -k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw= -k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= -k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs= -k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/apiserver v0.32.1 h1:oo0OozRos66WFq87Zc5tclUX2r0mymoVHRq8JmR7Aak= -k8s.io/apiserver v0.32.1/go.mod h1:UcB9tWjBY7aryeI5zAgzVJB/6k7E97bkr1RgqDz0jPw= -k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU= -k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg= -k8s.io/component-base v0.32.1 h1:/5IfJ0dHIKBWysGV0yKTFfacZ5yNV1sulPh3ilJjRZk= -k8s.io/component-base v0.32.1/go.mod h1:j1iMMHi/sqAHeG5z+O9BFNCF698a1u0186zkjMZQ28w= +k8s.io/api v0.33.1 h1:tA6Cf3bHnLIrUK4IqEgb2v++/GYUtqiu9sRVk3iBXyw= +k8s.io/api v0.33.1/go.mod h1:87esjTn9DRSRTD4fWMXamiXxJhpOIREjWOSjsW1kEHw= +k8s.io/apiextensions-apiserver v0.33.1 h1:N7ccbSlRN6I2QBcXevB73PixX2dQNIW0ZRuguEE91zI= +k8s.io/apiextensions-apiserver v0.33.1/go.mod h1:uNQ52z1A1Gu75QSa+pFK5bcXc4hq7lpOXbweZgi4dqA= +k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4= +k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apiserver v0.33.1 h1:yLgLUPDVC6tHbNcw5uE9mo1T6ELhJj7B0geifra3Qdo= +k8s.io/apiserver v0.33.1/go.mod h1:VMbE4ArWYLO01omz+k8hFjAdYfc3GVAYPrhP2tTKccs= +k8s.io/client-go v0.33.1 h1:ZZV/Ks2g92cyxWkRRnfUDsnhNn28eFpt26aGc8KbXF4= +k8s.io/client-go v0.33.1/go.mod h1:JAsUrl1ArO7uRVFWfcj6kOomSlCv+JpvIsp6usAGefA= +k8s.io/component-base v0.33.1 h1:EoJ0xA+wr77T+G8p6T3l4efT2oNwbqBVKR71E0tBIaI= +k8s.io/component-base v0.33.1/go.mod h1:guT/w/6piyPfTgq7gfvgetyXMIh10zuXA6cRRm3rDuY= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 h1:CPT0ExVicCzcpeN4baWEV2ko2Z/AsiZgEdwgcfwLgMo= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/controller-runtime v0.20.4 h1:X3c+Odnxz+iPTRobG4tp092+CvBU9UK0t/bRf+n0DGU= -sigs.k8s.io/controller-runtime v0.20.4/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= +sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= sigs.k8s.io/gateway-api v1.2.1 h1:fZZ/+RyRb+Y5tGkwxFKuYuSRQHu9dZtbjenblleOLHM= sigs.k8s.io/gateway-api v1.2.1/go.mod h1:EpNfEXNjiYfUJypf0eZ0P5iXA9ekSGWaS1WgPaM42X0= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= -sigs.k8s.io/multicluster-runtime v0.20.4-alpha.6 h1:xaBJoiYZY+DlaHBIXfS+ZUH+L5GOD6A+klL46vekggE= -sigs.k8s.io/multicluster-runtime v0.20.4-alpha.6/go.mod h1:2N2/c3p08bYC9eDaRs0dllTxgAm5xiLDSkmGZpWKyw4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/multicluster-runtime v0.21.0-alpha.8 h1:Pq69tTKfN8ADw8m8A3wUtP8wJ9SPQbbOsgapm3BZEPw= +sigs.k8s.io/multicluster-runtime v0.21.0-alpha.8/go.mod h1:CpBzLMLQKdm+UCchd2FiGPiDdCxM5dgCCPKuaQ6Fsv0= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/controller/indexers.go b/internal/controller/indexers.go new file mode 100644 index 00000000..5c3351da --- /dev/null +++ b/internal/controller/indexers.go @@ -0,0 +1,87 @@ +package controller + +import ( + "context" + "errors" + "fmt" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + + computev1alpha "go.datum.net/workload-operator/api/v1alpha" +) + +const ( + deploymentWorkloadUIDIndex = "deploymentWorkloadUIDIndex" + workloadNetworksIndex = "workloadNetworksIndex" + deploymentLocationIndex = "deploymentLocationIndex" +) + +func AddIndexers(ctx context.Context, mgr mcmanager.Manager) error { + return errors.Join( + addWorkloadDeploymentIndexers(ctx, mgr), + addWorkloadIndexers(ctx, mgr), + ) +} + +func addWorkloadDeploymentIndexers(ctx context.Context, mgr mcmanager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &computev1alpha.WorkloadDeployment{}, deploymentWorkloadUIDIndex, deploymentWorkloadUIDIndexFunc); err != nil { + return fmt.Errorf("failed to add workload deployment indexer %q: %w", deploymentWorkloadUIDIndex, err) + } + + // Index workload deployments by location + if err := mgr.GetFieldIndexer().IndexField(ctx, &computev1alpha.WorkloadDeployment{}, deploymentLocationIndex, deploymentLocationIndexFunc); err != nil { + return fmt.Errorf("failed to add workload deployment indexer %q: %w", deploymentLocationIndex, err) + } + + return nil +} + +func deploymentWorkloadUIDIndexFunc(o client.Object) []string { + return []string{ + string(o.(*computev1alpha.WorkloadDeployment).Spec.WorkloadRef.UID), + } +} + +func deploymentLocationIndexFunc(o client.Object) []string { + deployment := o.(*computev1alpha.WorkloadDeployment) + if deployment.Status.Location == nil { + return nil + } + + return []string{ + types.NamespacedName{ + Namespace: deployment.Status.Location.Namespace, + Name: deployment.Status.Location.Name, + }.String(), + } +} + +func addWorkloadIndexers(ctx context.Context, mgr mcmanager.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(ctx, &computev1alpha.Workload{}, workloadNetworksIndex, workloadNetworksIndexFunc); err != nil { + return fmt.Errorf("failed to add workload indexer %q: %w", workloadNetworksIndex, err) + } + + return nil +} + +func workloadNetworksIndexFunc(o client.Object) []string { + workload := o.(*computev1alpha.Workload) + + networks := make([]string, 0, len(workload.Spec.Template.Spec.NetworkInterfaces)) + for _, network := range workload.Spec.Template.Spec.NetworkInterfaces { + namespacedName := types.NamespacedName{ + Namespace: network.Network.Namespace, + Name: network.Network.Name, + } + + if namespacedName.Namespace == "" { + namespacedName.Namespace = workload.GetNamespace() + } + + networks = append(networks, namespacedName.String()) + } + + return networks +} diff --git a/internal/controller/instance_controller.go b/internal/controller/instance_controller.go index 5f56cda3..6be1fccf 100644 --- a/internal/controller/instance_controller.go +++ b/internal/controller/instance_controller.go @@ -5,9 +5,10 @@ package controller import ( "context" "fmt" + "strings" - "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -17,6 +18,7 @@ import ( mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" computev1alpha "go.datum.net/workload-operator/api/v1alpha" ) @@ -29,7 +31,7 @@ type InstanceReconciler struct { // +kubebuilder:rbac:groups=compute.datumapis.com,resources=instances/status,verbs=get;update;patch // +kubebuilder:rbac:groups=compute.datumapis.com,resources=instances/finalizers,verbs=update -func (r *InstanceReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { +func (r *InstanceReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (_ ctrl.Result, err error) { logger := log.FromContext(ctx) cl, err := r.mgr.GetCluster(ctx, req.ClusterName) @@ -46,53 +48,150 @@ func (r *InstanceReconciler) Reconcile(ctx context.Context, req mcreconcile.Requ return ctrl.Result{}, err } - if !instance.DeletionTimestamp.IsZero() { - return ctrl.Result{}, nil - } - logger.Info("reconciling instance") defer logger.Info("reconcile complete") - // Ensure the instance has labels necessary for being able to identify - // instances associated with a specific workload or workload deployment via - // label selectors. - // - // This logic will not be necessary once we complete the work defined in - // https://github.com/datum-cloud/enhancements/issues/28 + if changed, err := r.reconcileInstanceReadyCondition(ctx, cl.GetClient(), &instance, r.checkForNetworkCreationFailure); err != nil { + return ctrl.Result{}, err + } else if changed { + return ctrl.Result{}, cl.GetClient().Status().Update(ctx, &instance) + } + + return ctrl.Result{}, nil +} + +// networkFailureChecker is a function that checks if a network creation failure +// has occurred. It returns a boolean indicating if a failure has occurred, a +// message describing the failure, and an error if the check fails. +type networkFailureChecker func(ctx context.Context, upstreamClient client.Client, instance *computev1alpha.Instance) (failed bool, message string, err error) + +func (r *InstanceReconciler) reconcileInstanceReadyCondition( + ctx context.Context, + clusterClient client.Client, + instance *computev1alpha.Instance, + networkFailureChecker networkFailureChecker, +) (changed bool, err error) { + logger := log.FromContext(ctx) + + readyCondition := apimeta.FindStatusCondition(instance.Status.Conditions, computev1alpha.InstanceReady) + if readyCondition == nil { + readyCondition = &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: computev1alpha.InstanceProgrammedReasonPendingProgramming, + ObservedGeneration: instance.Generation, + Message: "Instance has not been programmed", + } + } else { + readyCondition = readyCondition.DeepCopy() + } - workloadDeploymentRef := metav1.GetControllerOf(&instance) + if instance.Spec.Controller != nil && len(instance.Spec.Controller.SchedulingGates) > 0 { + // Update Ready condition to False, Reason to "SchedulingGatesPresent" + // and Message to "Scheduling gates present" + + // Collect a list of scheduling gate names + var schedulingGateNames []string + for _, gate := range instance.Spec.Controller.SchedulingGates { + schedulingGateNames = append(schedulingGateNames, gate.Name) + } + + networkCreationFailure, networkCreationFailureMessage, err := networkFailureChecker(ctx, clusterClient, instance) + if err != nil { + return false, fmt.Errorf("failed checking for network creation failure: %w", err) + } + + if networkCreationFailure { + readyCondition.Reason = "NetworkFailedToCreate" + readyCondition.Message = networkCreationFailureMessage + } else { + readyCondition.Reason = computev1alpha.InstanceReadyReasonSchedulingGatesPresent + readyCondition.Message = fmt.Sprintf("Scheduling gates present: %s", strings.Join(schedulingGateNames, ", ")) + } + + return apimeta.SetStatusCondition(&instance.Status.Conditions, *readyCondition), nil + } + + pendingReason := "Pending" + programmedCondition := apimeta.FindStatusCondition(instance.Status.Conditions, computev1alpha.InstanceProgrammed) + if programmedCondition == nil || programmedCondition.Status != metav1.ConditionTrue { + logger.Info("instance is not programmed", "instance", instance.Name) + + readyCondition.Reason = computev1alpha.InstanceProgrammedReasonPendingProgramming + if programmedCondition != nil && programmedCondition.Reason != pendingReason { + readyCondition.Reason = programmedCondition.Reason + } + + readyCondition.Message = "Instance has not been programmed" + if programmedCondition != nil && programmedCondition.Status != metav1.ConditionUnknown { + readyCondition.Message = programmedCondition.Message + } + + return apimeta.SetStatusCondition(&instance.Status.Conditions, *readyCondition), nil + } + + logger.Info("instance is programmed", "instance", instance.Name) + + runningCondition := apimeta.FindStatusCondition(instance.Status.Conditions, computev1alpha.InstanceRunning) + if runningCondition == nil || runningCondition.Status != metav1.ConditionTrue { + logger.Info("instance is not running", "instance", instance.Name) + + readyCondition.Reason = pendingReason + if runningCondition != nil && runningCondition.Reason != pendingReason { + readyCondition.Reason = runningCondition.Reason + } + + readyCondition.Message = "Instance is not running" + if runningCondition != nil && runningCondition.Status != metav1.ConditionUnknown { + readyCondition.Message = runningCondition.Message + } + + return apimeta.SetStatusCondition(&instance.Status.Conditions, *readyCondition), nil + } + + readyCondition.Status = metav1.ConditionTrue + readyCondition.Reason = computev1alpha.InstanceReadyReasonRunning + readyCondition.Message = "Instance is ready" + + return apimeta.SetStatusCondition(&instance.Status.Conditions, *readyCondition), nil +} + +// Rough way to propagate creation errors up to the instance as soon as possible. +// Lots of room for improvement here. +func (r *InstanceReconciler) checkForNetworkCreationFailure(ctx context.Context, upstreamClient client.Client, instance *computev1alpha.Instance) (failed bool, message string, err error) { + workloadDeploymentRef := metav1.GetControllerOf(instance) if workloadDeploymentRef == nil { - return ctrl.Result{}, fmt.Errorf("failed to get controller owner of Instance") + return false, "", fmt.Errorf("instance is not owned by a workload deployment") } + // Load the WorkloadDeployment for the instance var workloadDeployment computev1alpha.WorkloadDeployment workloadDeploymentObjectKey := client.ObjectKey{ Namespace: instance.Namespace, Name: workloadDeploymentRef.Name, } - if err := cl.GetClient().Get(ctx, workloadDeploymentObjectKey, &workloadDeployment); err != nil { - return ctrl.Result{}, err + if err := upstreamClient.Get(ctx, workloadDeploymentObjectKey, &workloadDeployment); err != nil { + return false, "", fmt.Errorf("failed fetching workload deployment: %w", err) } - workloadRef := metav1.GetControllerOf(&workloadDeployment) - if workloadRef == nil { - return ctrl.Result{}, fmt.Errorf("failed to get controller owner of WorkloadDeployment") - } + for i := range instance.Spec.NetworkInterfaces { + var networkBinding networkingv1alpha.NetworkBinding + networkBindingObjectKey := client.ObjectKey{ + Namespace: workloadDeployment.Namespace, + Name: fmt.Sprintf("%s-net-%d", workloadDeployment.Name, i), + } - updated := instance.DeepCopy() - if updated.Labels == nil { - updated.Labels = map[string]string{} - } - updated.Labels[computev1alpha.WorkloadUIDLabel] = string(workloadRef.UID) - updated.Labels[computev1alpha.WorkloadDeploymentUIDLabel] = string(workloadDeploymentRef.UID) + if err := upstreamClient.Get(ctx, networkBindingObjectKey, &networkBinding); client.IgnoreNotFound(err) != nil { + return false, "", fmt.Errorf("failed checking for existing network binding: %w", err) + } - if !equality.Semantic.DeepEqual(updated, instance) { - if err := cl.GetClient().Update(ctx, updated); err != nil { - return ctrl.Result{}, fmt.Errorf("failed updating instance: %w", err) + condition := apimeta.FindStatusCondition(networkBinding.Status.Conditions, networkingv1alpha.NetworkBindingReady) + if condition != nil && condition.Status == metav1.ConditionFalse && condition.Reason == "NetworkFailedToCreate" { + return true, condition.Message, nil } } - return ctrl.Result{}, nil + return false, "", nil } // SetupWithManager sets up the controller with the Manager. diff --git a/internal/controller/instance_controller_test.go b/internal/controller/instance_controller_test.go index b4725a75..7eca925c 100644 --- a/internal/controller/instance_controller_test.go +++ b/internal/controller/instance_controller_test.go @@ -5,187 +5,276 @@ import ( "testing" "github.com/stretchr/testify/assert" - "google.golang.org/protobuf/proto" + "github.com/stretchr/testify/require" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" - mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" computev1alpha "go.datum.net/workload-operator/api/v1alpha" ) -type testManager struct { - ctrl.Manager - client client.Client - scheme *runtime.Scheme -} - -func (m *testManager) GetClient() client.Client { - return m.client -} - -func (m *testManager) GetScheme() *runtime.Scheme { - return m.scheme -} - -func TestInstanceReconciler(t *testing.T) { - scheme := runtime.NewScheme() - utilruntime.Must(computev1alpha.AddToScheme(scheme)) +func TestReconcileInstanceReadyCondition(t *testing.T) { tests := []struct { - name string - objs []client.Object - req ctrl.Request - expectedErr string - expectedLabels map[string]string + name string + instance *computev1alpha.Instance + networkFailureFunc networkFailureChecker + expectedChanged bool + expectedCondition *metav1.Condition }{ { - name: "missing controller owner in instance", - objs: []client.Object{ - &computev1alpha.Instance{ - ObjectMeta: metav1.ObjectMeta{ - Name: "instance-no-owner", - Namespace: "default", + name: "instance without ready condition should create default", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + }, + expectedChanged: true, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: computev1alpha.InstanceProgrammedReasonPendingProgramming, + Message: "Instance has not been programmed", + ObservedGeneration: 1, + }, + }, + { + name: "instance with scheduling gates should set scheduling gates present", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + Spec: computev1alpha.InstanceSpec{ + Controller: &computev1alpha.InstanceController{ + SchedulingGates: []computev1alpha.SchedulingGate{ + {Name: "Network"}, + }, + }, + }, + Status: computev1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + { + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: computev1alpha.InstanceProgrammedReasonPendingProgramming, + Message: "Instance has not been programmed", + ObservedGeneration: 1, + LastTransitionTime: metav1.Now(), + }, }, }, }, - req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "instance-no-owner"}}, - expectedErr: "failed to get controller owner of Instance", + expectedChanged: true, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: computev1alpha.InstanceReadyReasonSchedulingGatesPresent, + Message: "Scheduling gates present: Network", + ObservedGeneration: 1, + }, }, { - name: "workload deployment not found", - objs: []client.Object{ - &computev1alpha.Instance{ - ObjectMeta: metav1.ObjectMeta{ - Name: "instance-missing-wd", - Namespace: "default", - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "compute.datumapis.com/v1alpha", - Kind: "WorkloadDeployment", - Name: "wd1", - UID: "33c325cb-3f2e-4b2a-be0c-6d7e03aa475a", - Controller: proto.Bool(true), - }, + name: "instance with scheduling gates and network failure should set network failed", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + Spec: computev1alpha.InstanceSpec{ + Controller: &computev1alpha.InstanceController{ + SchedulingGates: []computev1alpha.SchedulingGate{ + {Name: "Network"}, }, }, }, }, - req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "instance-missing-wd"}}, - expectedErr: "not found", + networkFailureFunc: func(ctx context.Context, upstreamClient client.Client, instance *computev1alpha.Instance) (bool, string, error) { + return true, "Network creation failed: timeout", nil + }, + expectedChanged: true, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: "NetworkFailedToCreate", + Message: "Network creation failed: timeout", + ObservedGeneration: 1, + }, }, { - name: "missing controller owner in workload deployment", - objs: []client.Object{ - &computev1alpha.Instance{ - ObjectMeta: metav1.ObjectMeta{ - Name: "instance-wd-no-owner", - Namespace: "default", - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "compute.datumapis.com/v1alpha", - Kind: "WorkloadDeployment", - Name: "wd1", - UID: "33c325cb-3f2e-4b2a-be0c-6d7e03aa475a", - Controller: proto.Bool(true), - }, + name: "instance not programmed should set pending programming", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + Status: computev1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + { + Type: computev1alpha.InstanceProgrammed, + Status: metav1.ConditionFalse, + Reason: "TestReason", + Message: "Test message", }, }, }, - &computev1alpha.WorkloadDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "wd1", - Namespace: "default", - // Missing controller owner + }, + expectedChanged: true, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: "TestReason", + Message: "Test message", + ObservedGeneration: 1, + }, + }, + { + name: "instance programmed but not running should wait for running", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + Status: computev1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + { + Type: computev1alpha.InstanceProgrammed, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceProgrammedReasonProgrammed, + Message: "Instance has been programmed", + }, + { + Type: computev1alpha.InstanceRunning, + Status: metav1.ConditionFalse, + Reason: "TestReason", + Message: "Test message", + }, }, }, }, - req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "instance-wd-no-owner"}}, - expectedErr: "failed to get controller owner of WorkloadDeployment", + expectedChanged: true, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionFalse, + Reason: "TestReason", + Message: "Test message", + ObservedGeneration: 1, + }, }, { - name: "successful reconcile and update", - objs: []client.Object{ - &computev1alpha.Instance{ - ObjectMeta: metav1.ObjectMeta{ - Name: "instance-success", - Namespace: "default", - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "compute.datumapis.com/v1alpha", - Kind: "WorkloadDeployment", - Name: "wd1", - UID: "33c325cb-3f2e-4b2a-be0c-6d7e03aa475a", - Controller: proto.Bool(true), - }, + name: "instance fully ready should set ready condition", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + Status: computev1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + { + Type: computev1alpha.InstanceProgrammed, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceProgrammedReasonProgrammed, + Message: "Instance has been programmed", + }, + { + Type: computev1alpha.InstanceRunning, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceRunningReasonRunning, + Message: "Instance is running", }, }, }, - &computev1alpha.WorkloadDeployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "wd1", - Namespace: "default", - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "compute.datumapis.com/v1alpha", - Kind: "Workload", - Name: "w1", - UID: "2561b624-3f7d-49db-bc74-b9dc71c4c08d", - Controller: proto.Bool(true), - }, + }, + expectedChanged: true, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceReadyReasonRunning, + Message: "Instance is ready", + ObservedGeneration: 1, + }, + }, + { + name: "no change when condition already matches", + instance: &computev1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "default", + Generation: 1, + }, + Status: computev1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + { + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceReadyReasonRunning, + Message: "Instance is ready", + ObservedGeneration: 1, + LastTransitionTime: metav1.Now(), + }, + { + Type: computev1alpha.InstanceProgrammed, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceProgrammedReasonProgrammed, + Message: "Instance has been programmed", + }, + { + Type: computev1alpha.InstanceRunning, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceRunningReasonRunning, + Message: "Instance is running", }, }, }, }, - req: ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "default", Name: "instance-success"}}, - expectedErr: "", - expectedLabels: map[string]string{ - computev1alpha.WorkloadUIDLabel: "2561b624-3f7d-49db-bc74-b9dc71c4c08d", - computev1alpha.WorkloadDeploymentUIDLabel: "33c325cb-3f2e-4b2a-be0c-6d7e03aa475a", + expectedChanged: false, + expectedCondition: &metav1.Condition{ + Type: computev1alpha.InstanceReady, + Status: metav1.ConditionTrue, + Reason: computev1alpha.InstanceReadyReasonRunning, + Message: "Instance is ready", + ObservedGeneration: 1, }, }, } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cl := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(tc.objs...).Build() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { - testMgr := &testManager{ - client: cl, - scheme: scheme, - } + reconciler := &InstanceReconciler{} - mgr, err := mcmanager.WithMultiCluster(testMgr, nil) - if err != nil { - t.Fatalf("failed to create manager: %v", err) + networkFailureFunc := tt.networkFailureFunc + if networkFailureFunc == nil { + networkFailureFunc = func(ctx context.Context, upstreamClient client.Client, instance *computev1alpha.Instance) (bool, string, error) { + return false, "", nil + } } - reconciler := &InstanceReconciler{mgr} - _, err = reconciler.Reconcile(context.Background(), mcreconcile.Request{ - Request: tc.req, - ClusterName: "", - }) + changed, err := reconciler.reconcileInstanceReadyCondition( + ctx, + nil, + tt.instance, + networkFailureFunc, + ) - // Check error - if tc.expectedErr == "" { - assert.NoError(t, err) - } else { - assert.ErrorContains(t, err, tc.expectedErr) - } + require.NoError(t, err) + assert.Equal(t, tt.expectedChanged, changed) - // If labels are expected, fetch the instance and validate the labels. - if tc.expectedLabels != nil { - instance := &computev1alpha.Instance{} - if err := cl.Get(context.Background(), tc.req.NamespacedName, instance); err != nil { - t.Fatalf("failed to get instance: %v", err) - } - assert.Equal(t, tc.expectedLabels, instance.Labels) - } + readyCondition := apimeta.FindStatusCondition(tt.instance.Status.Conditions, computev1alpha.InstanceReady) + require.NotNil(t, readyCondition) + + assert.Equal(t, tt.expectedCondition.Type, readyCondition.Type) + assert.Equal(t, tt.expectedCondition.Status, readyCondition.Status) + assert.Equal(t, tt.expectedCondition.Reason, readyCondition.Reason) + assert.Equal(t, tt.expectedCondition.Message, readyCondition.Message) + assert.Equal(t, tt.expectedCondition.ObservedGeneration, readyCondition.ObservedGeneration) }) } } diff --git a/internal/controller/instancecontrol/controller_utils.go b/internal/controller/instancecontrol/controller_utils.go new file mode 100644 index 00000000..8dd890ee --- /dev/null +++ b/internal/controller/instancecontrol/controller_utils.go @@ -0,0 +1,30 @@ +package instancecontrol + +import ( + "fmt" + "hash" + "hash/fnv" + + "k8s.io/apimachinery/pkg/util/dump" + "k8s.io/apimachinery/pkg/util/rand" +) + +// ComputeHash returns a hash value calculated from pod template and +// a collisionCount to avoid hash collision. The hash will be safe encoded to +// avoid bad words. +// +// Inspired by: https://github.com/kubernetes/kubernetes/blob/8aae5398b3885dc271d407c4d661e19653daaf88/pkg/controller/controller_utils.go#L1295 +func ComputeHash(obj interface{}) string { + hasher := fnv.New32a() + DeepHashObject(hasher, obj) + + return rand.SafeEncodeString(fmt.Sprint(hasher.Sum32())) +} + +// DeepHashObject writes specified object to hash using the spew library +// which follows pointers and prints actual values of the nested objects +// ensuring the hash does not change when a pointer changes. +func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) { + hasher.Reset() + _, _ = fmt.Fprintf(hasher, "%v", dump.ForHash(objectToWrite)) +} diff --git a/internal/controller/instancecontrol/instancecontrol.go b/internal/controller/instancecontrol/instancecontrol.go new file mode 100644 index 00000000..a217e4a2 --- /dev/null +++ b/internal/controller/instancecontrol/instancecontrol.go @@ -0,0 +1,106 @@ +package instancecontrol + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "go.datum.net/workload-operator/api/v1alpha" +) + +type Strategy interface { + // GetActions returns a set of actions that should be taken in order to drive + // the workload deployment to the desired state. Some actions may be informational, + // such as providing context into pending actions when instance management + // policies may require waiting for pod readiness. + GetActions( + ctx context.Context, + scheme *runtime.Scheme, + deployment *v1alpha.WorkloadDeployment, + currentInstances []v1alpha.Instance, + ) ([]Action, error) +} + +type ActionType string + +const ( + ActionTypeCreate ActionType = "Create" + ActionTypeUpdate ActionType = "Update" + ActionTypeDelete ActionType = "Delete" + ActionTypeWait ActionType = "Wait" +) + +type Action struct { + Object client.Object + actionType ActionType + skipExecution bool + fn func(ctx context.Context, c client.Client) error +} + +func (a Action) Execute(ctx context.Context, c client.Client) error { + if a.skipExecution { + return nil + } + + return a.fn(ctx, c) +} + +func (a *Action) SkipExecution() { + a.skipExecution = true +} + +func (a Action) IsSkipped() bool { + return a.skipExecution +} + +func (a Action) ActionType() ActionType { + return a.actionType +} + +func NewCreateAction(object client.Object) Action { + return Action{ + Object: object, + actionType: ActionTypeCreate, + fn: func(ctx context.Context, c client.Client) error { + if err := c.Create(ctx, object); err != nil { + return fmt.Errorf("failed to create %T: %w", object, err) + } + + return nil + }, + } +} + +func NewUpdateAction(object client.Object) Action { + return Action{ + Object: object, + actionType: ActionTypeUpdate, + fn: func(ctx context.Context, c client.Client) error { + if err := c.Update(ctx, object); err != nil { + return fmt.Errorf("failed to update %T: %w", object, err) + } + + return nil + }, + } +} + +func NewDeleteAction(object client.Object) Action { + return Action{ + Object: object, + actionType: ActionTypeDelete, + fn: func(ctx context.Context, c client.Client) error { + return c.Delete(ctx, object) + }, + } +} + +func NewWaitAction(object client.Object) Action { + return Action{ + Object: object, + actionType: ActionTypeWait, + fn: func(ctx context.Context, c client.Client) error { return nil }, + } +} diff --git a/internal/controller/instancecontrol/scheduling_gates.go b/internal/controller/instancecontrol/scheduling_gates.go new file mode 100644 index 00000000..5ff5ce9a --- /dev/null +++ b/internal/controller/instancecontrol/scheduling_gates.go @@ -0,0 +1,11 @@ +package instancecontrol + +type SchedulingGate string + +const ( + NetworkSchedulingGate SchedulingGate = "Network" +) + +func (s SchedulingGate) String() string { + return string(s) +} diff --git a/internal/controller/instancecontrol/stateful/stateful_control.go b/internal/controller/instancecontrol/stateful/stateful_control.go new file mode 100644 index 00000000..05bb9ee0 --- /dev/null +++ b/internal/controller/instancecontrol/stateful/stateful_control.go @@ -0,0 +1,159 @@ +package stateful + +import ( + "context" + "fmt" + "slices" + "strconv" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "go.datum.net/workload-operator/api/v1alpha" + "go.datum.net/workload-operator/internal/controller/instancecontrol" +) + +// Behavior inspired by https://github.com/kubernetes/kubernetes/tree/master/pkg/controller/statefulset +// Does not currently implement exact behavior. +type statefulControl struct { +} + +func New() instancecontrol.Strategy { + return &statefulControl{} +} + +func (c *statefulControl) GetActions( + ctx context.Context, + scheme *runtime.Scheme, + deployment *v1alpha.WorkloadDeployment, + currentInstances []v1alpha.Instance, +) ([]instancecontrol.Action, error) { + instanceTemplateHash := instancecontrol.ComputeHash(deployment.Spec.Template) + + // lowest -> highest + var createActions []instancecontrol.Action + var waitActions []instancecontrol.Action + + // highest -> lowest + var updateActions []instancecontrol.Action + + // highest -> lowest + var deleteActions []instancecontrol.Action + + // Instances that are desired to exist. We do not currently support the + // concept of a partition, so will fill the entire slice. + desiredInstances := make([]*v1alpha.Instance, deployment.Spec.ScaleSettings.MinReplicas) + + for _, instance := range currentInstances { + instanceIndex := getInstanceOrdinal(instance.Name) + if instanceIndex >= len(desiredInstances) { + deleteActions = append(deleteActions, instancecontrol.NewDeleteAction(&instance)) + } else { + desiredInstances[instanceIndex] = &instance + } + } + + // It's possible that the incoming currentInstances will have gaps in + // instances, so fill them in. + for i := range deployment.Spec.ScaleSettings.MinReplicas { + if desiredInstances[i] == nil { + desiredInstances[i] = &v1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Labels: deployment.Spec.Template.Labels, + Annotations: deployment.Spec.Template.Annotations, + Name: fmt.Sprintf("%s-%d", deployment.Name, i), + Namespace: deployment.Namespace, + }, + Spec: deployment.Spec.Template.Spec, + } + desiredInstances[i].Spec.Location = deployment.Status.Location + + // TODO(jreese) consider adding scheduling gates via mutating webhooks + desiredInstances[i].Spec.Controller = &v1alpha.InstanceController{ + TemplateHash: instanceTemplateHash, + SchedulingGates: []v1alpha.SchedulingGate{ + { + Name: instancecontrol.NetworkSchedulingGate.String(), + }, + }, + } + + addInstanceControllerLabels(desiredInstances[i], getInstanceOrdinal(desiredInstances[i].Name), deployment) + + if err := controllerutil.SetControllerReference(deployment, desiredInstances[i], scheme); err != nil { + return nil, fmt.Errorf("failed to set controller reference: %w", err) + } + } + } + + for _, instance := range desiredInstances { + if instance.CreationTimestamp.IsZero() { + action := instancecontrol.NewCreateAction(instance) + + createActions = append(createActions, action) + } else if !instance.DeletionTimestamp.IsZero() { + // Wait for graceful deletion before continuing processing additional + // instances. + waitActions = append(waitActions, instancecontrol.NewWaitAction(instance)) + + } else if instance.DeletionTimestamp.IsZero() { + // Wait for the instance to be ready before continuing processing + if !apimeta.IsStatusConditionTrue(instance.Status.Conditions, v1alpha.InstanceReady) { + waitActions = append(waitActions, instancecontrol.NewWaitAction(instance)) + } else if needsUpdate(instance, instanceTemplateHash) { + updatedInstance := instance.DeepCopy() + updatedInstance.Annotations = deployment.Spec.Template.Annotations + updatedInstance.Labels = deployment.Spec.Template.Labels + + addInstanceControllerLabels(updatedInstance, getInstanceOrdinal(updatedInstance.Name), deployment) + + updatedInstance.Spec = deployment.Spec.Template.Spec + updateActions = append(updateActions, instancecontrol.NewUpdateAction(updatedInstance)) + } + } + } + + slices.SortFunc(updateActions, descendingOrdinal) + slices.SortFunc(deleteActions, descendingOrdinal) + + actions := make([]instancecontrol.Action, 0, len(createActions)+len(waitActions)+len(updateActions)+len(deleteActions)) + + switch deployment.Spec.ScaleSettings.InstanceManagementPolicy { + case v1alpha.OrderedReadyInstanceManagementPolicyType: + + // Add create and wait actions, and sort by ordinal. This allows us to wait + // for instances to be processed in the correct order. + // + // For instance, we may have instance 0 that needs to wait to be ready, but + // instance 1 wants to be created. + actions = append(actions, createActions...) + actions = append(actions, waitActions...) + + slices.SortFunc(actions, ascendingOrdinal) + + actions = append(actions, updateActions...) + actions = append(actions, deleteActions...) + + // Skip all actions except the first one. + for i := range actions { + if i > 0 { + actions[i].SkipExecution() + } + } + + } + + return actions, nil +} + +func addInstanceControllerLabels(instance *v1alpha.Instance, index int, deployment *v1alpha.WorkloadDeployment) { + if instance.Labels == nil { + instance.Labels = map[string]string{} + } + + instance.Labels[v1alpha.InstanceIndexLabel] = strconv.Itoa(index) + instance.Labels[v1alpha.WorkloadUIDLabel] = string(deployment.Spec.WorkloadRef.UID) + instance.Labels[v1alpha.WorkloadDeploymentUIDLabel] = string(deployment.GetUID()) +} diff --git a/internal/controller/instancecontrol/stateful/stateful_control_test.go b/internal/controller/instancecontrol/stateful/stateful_control_test.go new file mode 100644 index 00000000..ad99e749 --- /dev/null +++ b/internal/controller/instancecontrol/stateful/stateful_control_test.go @@ -0,0 +1,233 @@ +package stateful + +import ( + "context" + "fmt" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/utils/ptr" + + "go.datum.net/workload-operator/api/v1alpha" + "go.datum.net/workload-operator/internal/controller/instancecontrol" +) + +var ( + scheme = runtime.NewScheme() +) + +func init() { + utilruntime.Must(v1alpha.AddToScheme(scheme)) +} + +func TestFreshDeployment(t *testing.T) { + ctx := context.Background() + control := New() + + deployment := getWorkloadDeployment("test-fresh-deploy", 2) + + // No instances + var currentInstances []v1alpha.Instance + actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + + assert.NoError(t, err) + assert.Len(t, actions, 2) + + assert.Equal(t, "test-fresh-deploy-0", actions[0].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeCreate, actions[0].ActionType()) + assert.False(t, actions[0].IsSkipped()) + + assert.Equal(t, "test-fresh-deploy-1", actions[1].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeCreate, actions[1].ActionType()) + assert.True(t, actions[1].IsSkipped()) +} + +func TestUpdateWithAllReadyInstances(t *testing.T) { + ctx := context.Background() + control := New() + + deployment := getWorkloadDeployment("test-deploy", 2) + + var currentInstances []v1alpha.Instance + currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 0)) + currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 1)) + + deployment.Spec.Template.Spec.Runtime.Sandbox.Containers[0].Image = "test-image-update" + + actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + + assert.NoError(t, err) + assert.Len(t, actions, 2) + + assert.Equal(t, "test-deploy-1", actions[0].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeUpdate, actions[0].ActionType()) + assert.False(t, actions[0].IsSkipped()) + + assert.Equal(t, "test-deploy-0", actions[1].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeUpdate, actions[1].ActionType()) + assert.True(t, actions[1].IsSkipped()) +} + +func TestScaleUpWithNotReadyInstance(t *testing.T) { + ctx := context.Background() + control := New() + + deployment := getWorkloadDeployment("test-deploy", 3) + + var currentInstances []v1alpha.Instance + currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 0)) + + notReadyInstance := getInstanceForDeployment(deployment, 1) + apimeta.SetStatusCondition(¬ReadyInstance.Status.Conditions, metav1.Condition{ + Type: v1alpha.InstanceReady, + Status: metav1.ConditionFalse, + }) + currentInstances = append(currentInstances, *notReadyInstance) + + actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + + assert.NoError(t, err) + assert.Len(t, actions, 2) + + assert.Equal(t, "test-deploy-1", actions[0].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeWait, actions[0].ActionType()) + assert.False(t, actions[0].IsSkipped()) + + assert.Equal(t, "test-deploy-2", actions[1].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeCreate, actions[1].ActionType()) + assert.True(t, actions[1].IsSkipped()) +} + +func TestScaleUpWithDeletingReadyInstance(t *testing.T) { + ctx := context.Background() + control := New() + + deployment := getWorkloadDeployment("test-deploy", 3) + + var currentInstances []v1alpha.Instance + currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 0)) + + deletingInstance := getInstanceForDeployment(deployment, 1) + deletingInstance.DeletionTimestamp = ptr.To(metav1.Now()) + currentInstances = append(currentInstances, *deletingInstance) + + actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + + assert.NoError(t, err) + assert.Len(t, actions, 2) + + assert.Equal(t, "test-deploy-1", actions[0].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeWait, actions[0].ActionType()) + assert.False(t, actions[0].IsSkipped()) + + assert.Equal(t, "test-deploy-2", actions[1].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeCreate, actions[1].ActionType()) + assert.True(t, actions[1].IsSkipped()) +} + +func TestScaleDownWithAllReadyInstances(t *testing.T) { + ctx := context.Background() + control := New() + + deployment := getWorkloadDeployment("test-deploy", 1) + + var currentInstances []v1alpha.Instance + currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 0)) + currentInstances = append(currentInstances, *getInstanceForDeployment(deployment, 1)) + + actions, err := control.GetActions(ctx, scheme, deployment, currentInstances) + + assert.NoError(t, err) + assert.Len(t, actions, 1) + + assert.Equal(t, "test-deploy-1", actions[0].Object.GetName()) + assert.Equal(t, instancecontrol.ActionTypeDelete, actions[0].ActionType()) + assert.False(t, actions[0].IsSkipped()) +} + +// Add more test functions below for different scenarios. + +func getWorkloadDeployment(name string, minReplicas int32) *v1alpha.WorkloadDeployment { + instance := getInstanceTemplate(name, 0) + deployment := &v1alpha.WorkloadDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + Spec: v1alpha.WorkloadDeploymentSpec{ + ScaleSettings: v1alpha.HorizontalScaleSettings{ + MinReplicas: minReplicas, + InstanceManagementPolicy: v1alpha.OrderedReadyInstanceManagementPolicyType, + }, + Template: v1alpha.InstanceTemplateSpec{ + ObjectMeta: instance.ObjectMeta, + Spec: instance.Spec, + }, + }, + } + + return deployment +} + +func getInstanceForDeployment(deployment *v1alpha.WorkloadDeployment, ordinal int) *v1alpha.Instance { + instance := getInstance(deployment.Name, ordinal) + instance.Spec.Controller = &v1alpha.InstanceController{ + TemplateHash: instancecontrol.ComputeHash(deployment.Spec.Template), + } + + return instance +} + +func getInstance(name string, ordinal int) *v1alpha.Instance { + instance := getInstanceTemplate(name, ordinal) + instance.CreationTimestamp = metav1.Now() + instance.Labels = map[string]string{ + v1alpha.InstanceIndexLabel: strconv.Itoa(ordinal), + } + + instance.Status = v1alpha.InstanceStatus{ + Conditions: []metav1.Condition{ + { + Type: v1alpha.InstanceReady, + Status: metav1.ConditionTrue, + Reason: "Ready", + Message: "Instance is ready", + LastTransitionTime: metav1.Now(), + }, + }, + } + + return instance +} + +func getInstanceTemplate(name string, ordinal int) *v1alpha.Instance { + instanceName := fmt.Sprintf("%s-%d", name, ordinal) + instance := &v1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: instanceName, + Namespace: "default", + }, + Spec: v1alpha.InstanceSpec{ + Runtime: v1alpha.InstanceRuntimeSpec{ + Resources: v1alpha.InstanceRuntimeResources{ + InstanceType: "datumcloud/d1-standard-2", + }, + Sandbox: &v1alpha.SandboxRuntime{ + Containers: []v1alpha.SandboxContainer{ + { + Name: "test", + Image: "test", + }, + }, + }, + }, + }, + } + + return instance +} diff --git a/internal/controller/instancecontrol/stateful/stateful_control_util.go b/internal/controller/instancecontrol/stateful/stateful_control_util.go new file mode 100644 index 00000000..32c27e3d --- /dev/null +++ b/internal/controller/instancecontrol/stateful/stateful_control_util.go @@ -0,0 +1,46 @@ +package stateful + +import ( + "strconv" + "strings" + + "go.datum.net/workload-operator/api/v1alpha" + "go.datum.net/workload-operator/internal/controller/instancecontrol" +) + +func needsUpdate(instance *v1alpha.Instance, instanceTemplateHash string) bool { + return instance.Spec.Controller == nil || + instance.Spec.Controller.TemplateHash != instanceTemplateHash +} + +// getInstanceOrdinal returns the ordinal of the instance, or -1 if the instance +// does not have an ordinal. +func getInstanceOrdinal(name string) int { + lastDash := strings.LastIndex(name, "-") + if lastDash == -1 { + return -1 + } + + ordinal := -1 + if i, err := strconv.Atoi(name[lastDash+1:]); err == nil { + ordinal = i + } + + return ordinal +} + +func ascendingOrdinal(a, b instancecontrol.Action) int { + if getInstanceOrdinal(a.Object.GetName()) < getInstanceOrdinal(b.Object.GetName()) { + return -1 + } else { + return 1 + } +} + +func descendingOrdinal(a, b instancecontrol.Action) int { + if getInstanceOrdinal(a.Object.GetName()) > getInstanceOrdinal(b.Object.GetName()) { + return -1 + } else { + return 1 + } +} diff --git a/internal/controller/instancecontrol/stateful/stateful_control_util_test.go b/internal/controller/instancecontrol/stateful/stateful_control_util_test.go new file mode 100644 index 00000000..5456f8df --- /dev/null +++ b/internal/controller/instancecontrol/stateful/stateful_control_util_test.go @@ -0,0 +1,74 @@ +package stateful + +import ( + "fmt" + "math/rand/v2" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "go.datum.net/workload-operator/api/v1alpha" + "go.datum.net/workload-operator/internal/controller/instancecontrol" +) + +func TestGetInstanceOrdinal(t *testing.T) { + tests := []struct { + name string + objectName string + want int + }{ + { + name: "instance with ordinal 0", + objectName: "my-instance-0", + want: 0, + }, + { + name: "instance with ordinal 1", + objectName: "my-instance-1", + want: 1, + }, + { + name: "instance with unexpected suffix", + objectName: "my-instance-foo", + want: -1, + }, + { + name: "instance with no dash in name", + objectName: "myinstance", + want: -1, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := getInstanceOrdinal(test.objectName) + if got != test.want { + t.Errorf("getInstanceOrdinal(%q) = %d, want %d", test.objectName, got, test.want) + } + }) + } +} + +func TestDescendingOrdinal(t *testing.T) { + actions := make([]instancecontrol.Action, 0, 4) + + perm := rand.Perm(4) + for i := range perm { + actions = append(actions, instancecontrol.NewWaitAction( + &v1alpha.Instance{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("my-instance-%d", perm[i]), + }, + }, + )) + } + + slices.SortFunc(actions, descendingOrdinal) + + assert.Equal(t, actions[0].Object.GetName(), "my-instance-3") + assert.Equal(t, actions[1].Object.GetName(), "my-instance-2") + assert.Equal(t, actions[2].Object.GetName(), "my-instance-1") + assert.Equal(t, actions[3].Object.GetName(), "my-instance-0") +} diff --git a/internal/controller/workload_controller.go b/internal/controller/workload_controller.go index e07da676..9e8f628c 100644 --- a/internal/controller/workload_controller.go +++ b/internal/controller/workload_controller.go @@ -12,13 +12,17 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/finalizer" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" mccontext "sigs.k8s.io/multicluster-runtime/pkg/context" mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" @@ -29,7 +33,6 @@ import ( ) const workloadControllerFinalizer = "compute.datumapis.com/workload-controller" -const deploymentWorkloadUID = "spec.workloadRef.uid" // WorkloadReconciler reconciles a Workload object type WorkloadReconciler struct { @@ -174,7 +177,6 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req mcreconcile.Requ _, err := controllerutil.CreateOrUpdate(ctx, cl.GetClient(), deployment, func() error { if deployment.CreationTimestamp.IsZero() { logger.Info("creating deployment", "deployment_name", deployment.Name) - deployment.Finalizers = desiredDeployment.Finalizers if err := controllerutil.SetControllerReference(&workload, deployment, cl.GetScheme()); err != nil { return fmt.Errorf("failed to set controller on workload deployment: %w", err) } @@ -184,6 +186,8 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req mcreconcile.Requ deployment.Annotations = desiredDeployment.Annotations deployment.Labels = desiredDeployment.Labels + + // TODO(jreese) consider how this plays well with autoscaling deployment.Spec = desiredDeployment.Spec return nil }) @@ -213,6 +217,8 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( totalReplicas := int32(0) totalCurrentReplicas := int32(0) totalDesiredReplicas := int32(0) + totalReadyReplicas := int32(0) + totalDeployments := int32(0) availablePlacementFound := false @@ -231,7 +237,7 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( } } - availableCondition := metav1.Condition{ + placementAvailableCondition := metav1.Condition{ Type: "Available", Status: metav1.ConditionFalse, Reason: "NoAvailableDeployments", @@ -242,10 +248,13 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( replicas := int32(0) currentReplicas := int32(0) desiredReplicas := int32(0) + readyReplicas := int32(0) + totalDeployments += int32(len(placementDeployments)) for _, deployment := range placementDeployments { replicas += deployment.Status.Replicas - currentReplicas += deployment.Status.Replicas - desiredReplicas += deployment.Status.Replicas + currentReplicas += deployment.Status.CurrentReplicas + desiredReplicas += deployment.Status.DesiredReplicas + readyReplicas += deployment.Status.ReadyReplicas if apimeta.IsStatusConditionTrue(deployment.Status.Conditions, "Available") { foundAvailableDeployment = true @@ -254,19 +263,21 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( totalReplicas += replicas totalCurrentReplicas += currentReplicas totalDesiredReplicas += desiredReplicas + totalReadyReplicas += readyReplicas placementStatus.Replicas = replicas placementStatus.CurrentReplicas = currentReplicas placementStatus.DesiredReplicas = desiredReplicas + placementStatus.ReadyReplicas = readyReplicas if foundAvailableDeployment { - availableCondition.Status = metav1.ConditionTrue - availableCondition.Reason = "AvailableDeploymentFound" - availableCondition.Message = "At least one available deployment was found" + placementAvailableCondition.Status = metav1.ConditionTrue + placementAvailableCondition.Reason = "AvailableDeploymentFound" + placementAvailableCondition.Message = "At least one available deployment was found" availablePlacementFound = true } - apimeta.SetStatusCondition(&placementStatus.Conditions, availableCondition) + apimeta.SetStatusCondition(&placementStatus.Conditions, placementAvailableCondition) newWorkloadStatus.Placements = append(newWorkloadStatus.Placements, placementStatus) } @@ -286,9 +297,11 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( apimeta.SetStatusCondition(&newWorkloadStatus.Conditions, availableCondition) + newWorkloadStatus.Deployments = totalDeployments newWorkloadStatus.Replicas = totalReplicas newWorkloadStatus.CurrentReplicas = totalCurrentReplicas newWorkloadStatus.DesiredReplicas = totalDesiredReplicas + newWorkloadStatus.ReadyReplicas = totalReadyReplicas if equality.Semantic.DeepEqual(workload.Status, newWorkloadStatus) { return nil @@ -296,7 +309,7 @@ func (r *WorkloadReconciler) reconcileWorkloadStatus( workload.Status = *newWorkloadStatus if err := upstreamClient.Status().Update(ctx, workload); err != nil { - return fmt.Errorf("failed updating workload status") + return fmt.Errorf("failed updating workload status: %w", err) } return nil @@ -317,7 +330,7 @@ func (r *WorkloadReconciler) Finalize(ctx context.Context, obj client.Object) (f } listOpts := client.MatchingFields{ - deploymentWorkloadUID: string(obj.GetUID()), + deploymentWorkloadUIDIndex: string(obj.GetUID()), } var deployments computev1alpha.WorkloadDeploymentList if err := cl.GetClient().List(ctx, &deployments, listOpts); err != nil { @@ -334,21 +347,9 @@ func (r *WorkloadReconciler) Finalize(ctx context.Context, obj client.Object) (f if deployment.DeletionTimestamp.IsZero() { // Deletion will result in another reconcile of the workload, where we // will remove the finalizers. - if err := cl.GetClient().Delete(ctx, &deployment); err != nil { - if apierrors.IsNotFound(err) { - // List cache was not up to date - continue - } + if err := cl.GetClient().Delete(ctx, &deployment); client.IgnoreNotFound(err) != nil { return finalizer.Result{}, fmt.Errorf("failed deleting workload deployment: %w", err) } - } else { - // Remove the finalizer if it's still present - if controllerutil.RemoveFinalizer(&deployment, workloadControllerFinalizer) { - if err := cl.GetClient().Update(ctx, &deployment); err != nil { - return finalizer.Result{}, fmt.Errorf("failed removing finalizer from workload deployment: %w", err) - } - } - } } @@ -368,7 +369,7 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( ) (desired []computev1alpha.WorkloadDeployment, orphaned []computev1alpha.WorkloadDeployment, err error) { listOpts := client.MatchingFields{ - deploymentWorkloadUID: string(workload.UID), + deploymentWorkloadUIDIndex: string(workload.UID), } var deployments computev1alpha.WorkloadDeploymentList if err := upstreamClient.List(ctx, &deployments, listOpts); err != nil { @@ -423,9 +424,6 @@ func (r *WorkloadReconciler) getDeploymentsForWorkload( Labels: map[string]string{ computev1alpha.WorkloadUIDLabel: string(workload.UID), }, - Finalizers: []string{ - workloadControllerFinalizer, - }, }, Spec: computev1alpha.WorkloadDeploymentSpec{ WorkloadRef: computev1alpha.WorkloadReference{ @@ -462,22 +460,46 @@ func (r *WorkloadReconciler) SetupWithManager(mgr mcmanager.Manager) error { return fmt.Errorf("failed to register finalizer: %w", err) } - // TODO(jreese) move to indexer package - - err := mgr.GetFieldIndexer().IndexField(context.Background(), &computev1alpha.WorkloadDeployment{}, deploymentWorkloadUID, func(o client.Object) []string { - return []string{ - string(o.(*computev1alpha.WorkloadDeployment).Spec.WorkloadRef.UID), - } - }) - if err != nil { - return fmt.Errorf("failed to add workload deployment field indexer: %w", err) - } - - // TODO(jreese) add watch against networks that triggers a reconcile for - // workloads that are attached and are in an error state for networks not - // existing. return mcbuilder.ControllerManagedBy(mgr). For(&computev1alpha.Workload{}, mcbuilder.WithEngageWithLocalCluster(false)). Owns(&computev1alpha.WorkloadDeployment{}, mcbuilder.WithEngageWithLocalCluster(false)). + Watches(&networkingv1alpha.Network{}, func(clusterName string, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, network client.Object) []mcreconcile.Request { + logger := log.FromContext(ctx) + + cluster, err := mgr.GetCluster(ctx, clusterName) + if err != nil { + logger.Error(err, "failed to get cluster") + return nil + } + clusterClient := cluster.GetClient() + + networkName := client.ObjectKeyFromObject(network).String() + listOpts := client.MatchingFields{ + workloadNetworksIndex: networkName, + } + + var workloads computev1alpha.WorkloadList + if err := clusterClient.List(ctx, &workloads, listOpts); err != nil { + logger.Error(err, "failed to list workloads") + return nil + } + + var requests []mcreconcile.Request + for _, workload := range workloads.Items { + requests = append(requests, mcreconcile.Request{ + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: workload.Namespace, + Name: workload.Name, + }, + }, + ClusterName: clusterName, + }) + } + + return requests + }) + }). Complete(r) } diff --git a/internal/controller/workloaddeployment_controller.go b/internal/controller/workloaddeployment_controller.go index e3ff994a..30ae3a2f 100644 --- a/internal/controller/workloaddeployment_controller.go +++ b/internal/controller/workloaddeployment_controller.go @@ -4,14 +4,23 @@ package controller import ( "context" + "errors" "fmt" + "slices" apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + kerrors "k8s.io/apimachinery/pkg/util/errors" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/finalizer" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" mccontext "sigs.k8s.io/multicluster-runtime/pkg/context" mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" @@ -19,11 +28,15 @@ import ( networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" computev1alpha "go.datum.net/workload-operator/api/v1alpha" + + "go.datum.net/workload-operator/internal/controller/instancecontrol" + instancecontrolstateful "go.datum.net/workload-operator/internal/controller/instancecontrol/stateful" ) // WorkloadDeploymentReconciler reconciles a WorkloadDeployment object type WorkloadDeploymentReconciler struct { - mgr mcmanager.Manager + mgr mcmanager.Manager + finalizers finalizer.Finalizers } // +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloaddeployments,verbs=get;list;watch;create;update;patch;delete @@ -48,6 +61,24 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco return ctrl.Result{}, err } + finalizationResult, err := r.finalizers.Finalize(ctx, &deployment) + if err != nil { + if v, ok := err.(kerrors.Aggregate); ok && v.Is(errDeploymentHasInstances) { + // Don't produce an error in this case and let the watch on deployments + // result in another reconcile schedule. + logger.Info("deployment still has instances, waiting until removal") + return ctrl.Result{}, nil + } else { + return ctrl.Result{}, fmt.Errorf("failed to finalize: %w", err) + } + } + if finalizationResult.Updated { + if err = cl.GetClient().Update(ctx, &deployment); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to update based on finalization result: %w", err) + } + return ctrl.Result{}, nil + } + if !deployment.DeletionTimestamp.IsZero() { return ctrl.Result{}, nil } @@ -55,16 +86,140 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco logger.Info("reconciling deployment") defer logger.Info("reconcile complete") - // Ensure that a `NetworkBinding` is created for each network interface's - // network. - if deployment.Status.Location == nil { return ctrl.Result{}, nil } - // TODO(jreese) shortcut work on a status condition for network bindings - // being ready + // Collect all instances for this deployment + listOpts := client.MatchingLabels{ + computev1alpha.WorkloadDeploymentUIDLabel: string(deployment.GetUID()), + } + + var instances computev1alpha.InstanceList + if err := cl.GetClient().List(ctx, &instances, listOpts); err != nil { + return ctrl.Result{}, fmt.Errorf("failed listing instances: %w", err) + } + + instanceControl := instancecontrolstateful.New() + + actions, err := instanceControl.GetActions(ctx, cl.GetScheme(), &deployment, instances.Items) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed getting instance control actions: %w", err) + } + + logger.Info("collected instance control actions", "count", len(actions)) + for _, action := range actions { + // We don't need to actually check this, but it'll reduce log noise. + if action.IsSkipped() { + continue + } + + logger.Info("instance control action", "instance", action.Object.GetName(), "action", action.ActionType()) + + if err := action.Execute(ctx, cl.GetClient()); err != nil { + return ctrl.Result{}, fmt.Errorf("failed executing instance control action: %w", err) + } + } + + networkReady, err := r.reconcileNetworks(ctx, cl.GetClient(), &deployment) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed reconciling networks: %w", err) + } + + // Networks are all ready with subnets ready to use, remove any scheduling + // gates on instances. If any instances were created by actions above, those + // will result in this reconciler being processed again, so will properly have + // their gates removed. + + replicas := len(instances.Items) + currentReplicas := 0 + desiredReplicas := deployment.Spec.ScaleSettings.MinReplicas + if dt := deployment.DeletionTimestamp; !dt.IsZero() { + desiredReplicas = 0 + } + + readyReplicas := 0 + for _, instance := range instances.Items { + if networkReady && len(instance.Spec.Controller.SchedulingGates) > 0 { + newGates := slices.DeleteFunc(instance.Spec.Controller.SchedulingGates, func(gate computev1alpha.SchedulingGate) bool { + return gate.Name == instancecontrol.NetworkSchedulingGate.String() + }) + + if len(newGates) != len(instance.Spec.Controller.SchedulingGates) { + if _, err := controllerutil.CreateOrPatch(ctx, cl.GetClient(), &instance, func() error { + instance.Spec.Controller.SchedulingGates = newGates + return nil + }); err != nil { + return ctrl.Result{}, fmt.Errorf("failed updating instance: %w", err) + } + } + } + + if apimeta.IsStatusConditionTrue(instance.Status.Conditions, computev1alpha.InstanceProgrammed) { + if instance.Status.Controller.ObservedTemplateHash == instancecontrol.ComputeHash(deployment.Spec.Template) { + currentReplicas++ + } + } + + if apimeta.IsStatusConditionTrue(instance.Status.Conditions, computev1alpha.InstanceReady) { + readyReplicas++ + } + } + + patchResult, err := controllerutil.CreateOrPatch(ctx, cl.GetClient(), &deployment, func() error { + deployment.Status.Replicas = int32(replicas) + deployment.Status.CurrentReplicas = int32(currentReplicas) + deployment.Status.DesiredReplicas = desiredReplicas + deployment.Status.ReadyReplicas = int32(readyReplicas) + + if readyReplicas > 0 { + apimeta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ + Type: computev1alpha.WorkloadDeploymentAvailable, + Status: metav1.ConditionTrue, + Reason: "StableInstanceFound", + Message: fmt.Sprintf("%d/%d instances are ready", readyReplicas, replicas), + }) + } else if !networkReady { + apimeta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ + Type: computev1alpha.WorkloadDeploymentAvailable, + Status: metav1.ConditionFalse, + Reason: "ProvisioningNetwork", + Message: "Network is being provisioned", + }) + } else if replicas > 0 { + apimeta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{ + Type: computev1alpha.WorkloadDeploymentAvailable, + Status: metav1.ConditionFalse, + Reason: "ProvisioningInstances", + Message: "Instances are being provisioned", + }) + } + + return nil + }) + + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed updating deployment status: %w", err) + } + + logger.Info("deployment status processed", "operation_result", patchResult) + + return ctrl.Result{}, nil +} + +func (r *WorkloadDeploymentReconciler) reconcileNetworks( + ctx context.Context, + c client.Client, + deployment *computev1alpha.WorkloadDeployment, +) (bool, error) { + logger := log.FromContext(ctx) + + // First, ensure we have a NetworkBinding for each interface, and that the + // binding is ready before we move on to create SubnetClaims. + + var networkContextRefs []networkingv1alpha.NetworkContextRef + allNetworkBindingsReady := true for i, networkInterface := range deployment.Spec.Template.Spec.NetworkInterfaces { var networkBinding networkingv1alpha.NetworkBinding networkBindingObjectKey := client.ObjectKey{ @@ -72,8 +227,8 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco Name: fmt.Sprintf("%s-net-%d", deployment.Name, i), } - if err := cl.GetClient().Get(ctx, networkBindingObjectKey, &networkBinding); client.IgnoreNotFound(err) != nil { - return ctrl.Result{}, fmt.Errorf("failed checking for existing network binding: %w", err) + if err := c.Get(ctx, networkBindingObjectKey, &networkBinding); client.IgnoreNotFound(err) != nil { + return false, fmt.Errorf("failed checking for existing network binding: %w", err) } if networkBinding.CreationTimestamp.IsZero() { @@ -88,25 +243,255 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco }, } - if err := controllerutil.SetControllerReference(&deployment, &networkBinding, cl.GetScheme()); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to set controller on network binding: %w", err) + if err := controllerutil.SetControllerReference(deployment, &networkBinding, c.Scheme()); err != nil { + return false, fmt.Errorf("failed to set controller on network binding: %w", err) } - if err := cl.GetClient().Create(ctx, &networkBinding); err != nil { - return ctrl.Result{}, fmt.Errorf("failed creating network binding: %w", err) + if err := c.Create(ctx, &networkBinding); err != nil { + return false, fmt.Errorf("failed creating network binding: %w", err) } } + if !apimeta.IsStatusConditionTrue(networkBinding.Status.Conditions, networkingv1alpha.NetworkBindingReady) { + allNetworkBindingsReady = false + } else if networkBinding.Status.NetworkContextRef != nil { + networkContextRefs = append(networkContextRefs, *networkBinding.Status.NetworkContextRef) + } } - return ctrl.Result{}, nil + if !allNetworkBindingsReady { + logger.Info("waiting for network bindings to be ready") + return false, nil + } + + // TODO(jreese): Currently this makes a SubnetClaim that will be used by + // many instances. Move to a claim per instance interface, and allocate from + // a larger subnet. In addition, it does not handle allocation of more than + // one subnet per network context. We'll have a future IPAM controller in + // network-services-operator that will handle this. + // + // Also, only handling ipv4 + + for _, networkContextRef := range networkContextRefs { + var networkContext networkingv1alpha.NetworkContext + networkContextObjectKey := client.ObjectKey{ + Namespace: networkContextRef.Namespace, + Name: networkContextRef.Name, + } + + if err := c.Get(ctx, networkContextObjectKey, &networkContext); client.IgnoreNotFound(err) != nil { + return false, fmt.Errorf("failed checking for existing network context: %w", err) + } + + if !apimeta.IsStatusConditionTrue(networkContext.Status.Conditions, networkingv1alpha.NetworkContextReady) { + logger.Info("waiting for network context to be ready", "network_context", networkContext.Name) + return false, nil + } + + var subnetClaims networkingv1alpha.SubnetClaimList + listOpts := []client.ListOption{ + client.InNamespace(networkContext.Namespace), + } + + if err := c.List(ctx, &subnetClaims, listOpts...); err != nil { + return false, fmt.Errorf("failed listing subnet claims: %w", err) + } + + var subnetClaim networkingv1alpha.SubnetClaim + for _, claim := range subnetClaims.Items { + // If it's not the same subnet class, don't consider the subnet claim. + if claim.Spec.SubnetClass != "private" { + continue + } + + // If it's not ipv4, don't consider the subnet claim. + if claim.Spec.IPFamily != networkingv1alpha.IPv4Protocol { + continue + } + + // If it's not the same network context, don't consider the subnet claim. + if claim.Spec.NetworkContext.Name != networkContext.Name { + continue + } + + // If it's not the same location, don't consider the subnet claim. + if claim.Spec.Location.Namespace != deployment.Status.Location.Namespace || + claim.Spec.Location.Name != deployment.Status.Location.Name { + continue + } + + subnetClaim = claim + break + } + + if subnetClaim.CreationTimestamp.IsZero() { + subnetClaim = networkingv1alpha.SubnetClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: networkContext.Namespace, + // In the future, subnets will be created with an ordinal that increases. + // This ensures that we don't create duplicate subnet claims when the + // cache is not up to date. + Name: fmt.Sprintf("%s-0", networkContext.Name), + }, + Spec: networkingv1alpha.SubnetClaimSpec{ + SubnetClass: "private", + IPFamily: networkingv1alpha.IPv4Protocol, + NetworkContext: networkingv1alpha.LocalNetworkContextRef{ + Name: networkContext.Name, + }, + Location: *deployment.Status.Location, + }, + } + + if err := controllerutil.SetOwnerReference(&networkContext, &subnetClaim, c.Scheme()); err != nil { + return false, fmt.Errorf("failed to set controller on subnet claim: %w", err) + } + + if err := c.Create(ctx, &subnetClaim); err != nil { + return false, fmt.Errorf("failed creating subnet claim: %w", err) + } + + logger.Info("created subnet claim", "subnetClaim", subnetClaim.Name) + + return false, nil + } + + logger.Info("found subnet claim", "subnetClaim", subnetClaim.Name) + + if !apimeta.IsStatusConditionTrue(subnetClaim.Status.Conditions, "Ready") { + logger.Info("waiting for subnet claim to be ready", "subnetClaim", subnetClaim.Name) + return false, nil + } + + var subnet networkingv1alpha.Subnet + subnetObjectKey := client.ObjectKey{ + Namespace: subnetClaim.Namespace, + Name: subnetClaim.Status.SubnetRef.Name, + } + if err := c.Get(ctx, subnetObjectKey, &subnet); err != nil { + return false, fmt.Errorf("failed fetching subnet: %w", err) + } + + if !apimeta.IsStatusConditionTrue(subnet.Status.Conditions, "Ready") { + logger.Info("waiting for subnet to be ready", "subnet", subnet.Name) + return false, nil + } + + logger.Info("subnet is ready", "subnet", subnet.Name) + + } + + return true, nil +} + +var errDeploymentHasInstances = errors.New("deployment has instances") + +func (r *WorkloadDeploymentReconciler) Finalize(ctx context.Context, obj client.Object) (finalizer.Result, error) { + clusterName, ok := mccontext.ClusterFrom(ctx) + if !ok { + return finalizer.Result{}, fmt.Errorf("cluster name not found in context") + } + + cl, err := r.mgr.GetCluster(ctx, clusterName) + if err != nil { + return finalizer.Result{}, err + } + + var instanceList computev1alpha.InstanceList + listOpts := []client.ListOption{ + client.InNamespace(obj.GetNamespace()), + client.MatchingLabels{ + computev1alpha.WorkloadDeploymentUIDLabel: string(obj.GetUID()), + }, + } + + if err := cl.GetClient().List(ctx, &instanceList, listOpts...); err != nil { + return finalizer.Result{}, fmt.Errorf("failed listing instances: %w", err) + } + + if len(instanceList.Items) == 0 { + log.FromContext(ctx).Info("instances have been removed") + return finalizer.Result{}, nil + } + + // All instances need to be deleted before the deployment may be deleted + for _, instance := range instanceList.Items { + if instance.DeletionTimestamp.IsZero() { + if err := cl.GetClient().Delete(ctx, &instance); client.IgnoreNotFound(err) != nil { + return finalizer.Result{}, fmt.Errorf("failed deleting instance: %w", err) + } + } + } + + // Really don't like using errors for communication here. I think we'd need + // to move away from the finalizer helper to ensure we can wait on child + // resources to be gone before allowing the finalizer to be removed. + return finalizer.Result{}, errDeploymentHasInstances } // SetupWithManager sets up the controller with the Manager. func (r *WorkloadDeploymentReconciler) SetupWithManager(mgr mcmanager.Manager) error { r.mgr = mgr - // TODO(jreese) finalizers + r.finalizers = finalizer.NewFinalizers() + if err := r.finalizers.Register(workloadControllerFinalizer, r); err != nil { + return fmt.Errorf("failed to register finalizer: %w", err) + } return mcbuilder.ControllerManagedBy(mgr). For(&computev1alpha.WorkloadDeployment{}, mcbuilder.WithEngageWithLocalCluster(false)). + Owns(&computev1alpha.Instance{}). + Owns(&networkingv1alpha.NetworkBinding{}). + Watches(&networkingv1alpha.SubnetClaim{}, func(clusterName string, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []mcreconcile.Request { + subnetClaim := o.(*networkingv1alpha.SubnetClaim) + return enqueueWorkloadDeploymentByLocation(ctx, mgr, clusterName, subnetClaim.Spec.Location) + }) + }). + Watches(&networkingv1alpha.Subnet{}, func(clusterName string, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []mcreconcile.Request { + subnet := o.(*networkingv1alpha.Subnet) + return enqueueWorkloadDeploymentByLocation(ctx, mgr, clusterName, subnet.Spec.Location) + }) + }). Complete(r) } + +func enqueueWorkloadDeploymentByLocation(ctx context.Context, mgr mcmanager.Manager, clusterName string, locationRef networkingv1alpha.LocationReference) []mcreconcile.Request { + logger := log.FromContext(ctx) + + cluster, err := mgr.GetCluster(ctx, clusterName) + if err != nil { + logger.Error(err, "failed to get cluster") + return nil + } + clusterClient := cluster.GetClient() + + locationName := (types.NamespacedName{ + Namespace: locationRef.Namespace, + Name: locationRef.Name, + }).String() + listOpts := client.MatchingFields{ + deploymentLocationIndex: locationName, + } + + var workloadDeployments computev1alpha.WorkloadDeploymentList + + if err := clusterClient.List(ctx, &workloadDeployments, listOpts); err != nil { + logger.Error(err, "failed to list workloads") + return nil + } + + requests := make([]mcreconcile.Request, 0, len(workloadDeployments.Items)) + for _, workload := range workloadDeployments.Items { + requests = append(requests, mcreconcile.Request{ + Request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: workload.Namespace, + Name: workload.Name, + }, + }, + ClusterName: clusterName, + }) + } + + return requests +} diff --git a/internal/providers/datum/provider_test.go b/internal/providers/datum/provider_test.go index d748fb93..7064737e 100644 --- a/internal/providers/datum/provider_test.go +++ b/internal/providers/datum/provider_test.go @@ -43,7 +43,6 @@ func TestNotReadyProject(t *testing.T) { result, err := provider.Reconcile(context.Background(), req) assert.NoError(t, err, "unexpected error returned from reconciler") - assert.Equal(t, false, result.Requeue) assert.Zero(t, result.RequeueAfter) assert.Len(t, provider.projects, 0) } @@ -57,7 +56,6 @@ func TestReadyProject(t *testing.T) { result, err := provider.Reconcile(context.Background(), req) assert.NoError(t, err, "unexpected error returned from reconciler") - assert.Equal(t, false, result.Requeue) assert.Zero(t, result.RequeueAfter) assert.Len(t, provider.projects, 1) diff --git a/internal/webhook/v1alpha/workload_webhook.go b/internal/webhook/v1alpha/workload_webhook.go index 8692ba56..ebf27537 100644 --- a/internal/webhook/v1alpha/workload_webhook.go +++ b/internal/webhook/v1alpha/workload_webhook.go @@ -5,11 +5,11 @@ import ( "fmt" - "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" @@ -24,8 +24,7 @@ import ( func SetupWorkloadWebhookWithManager(mgr mcmanager.Manager) error { webhook := &workloadWebhook{ - mgr: mgr, - logger: mgr.GetLogger(), + mgr: mgr, } return ctrl.NewWebhookManagedBy(mgr.GetLocalManager()). @@ -38,8 +37,7 @@ func SetupWorkloadWebhookWithManager(mgr mcmanager.Manager) error { // +kubebuilder:webhook:path=/mutate-compute-datumapis-com-v1alpha-workload,mutating=true,failurePolicy=fail,sideEffects=None,groups=compute.datumapis.com,resources=workloads,verbs=create;update,versions=v1alpha,name=mworkload.kb.io,admissionReviewVersions=v1 type workloadWebhook struct { - mgr mcmanager.Manager - logger logr.Logger + mgr mcmanager.Manager } var _ admission.CustomDefaulter = &workloadWebhook{} @@ -91,6 +89,9 @@ func (r *workloadWebhook) ValidateCreate(ctx context.Context, obj runtime.Object } clusterClient := cluster.GetClient() + logger := logf.FromContext(ctx).WithValues("cluster", clusterName) + logger.Info("Validating Workload Create", "name", workload.GetName(), "cluster", clusterName) + req, err := admission.RequestFromContext(ctx) if err != nil { return nil, err