Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
computev1alpha "go.datum.net/compute/api/v1alpha"
"go.datum.net/compute/internal/config"
"go.datum.net/compute/internal/controller"
"go.datum.net/compute/internal/features"
quotametrics "go.datum.net/compute/internal/quota"
computewebhook "go.datum.net/compute/internal/webhook"
computev1alphawebhooks "go.datum.net/compute/internal/webhook/v1alpha"
Expand Down Expand Up @@ -114,6 +115,12 @@ func main() {
flag.BoolVar(&enableCellControllers, "enable-cell-controllers", false,
"Enable cell controllers (WorkloadDeploymentReconciler, InstanceReconciler).")

var featureGatesFlag string
flag.StringVar(&featureGatesFlag, "feature-gates", "",
"A set of key=value pairs that describe feature gates for the compute operator. "+
"Example: --feature-gates=NetworkingIntegration=false. "+
"Available features: NetworkingIntegration (default=true).")

opts := zap.Options{
Development: true,
}
Expand All @@ -123,6 +130,14 @@ func main() {
opts.BindFlags(flag.CommandLine)
flag.Parse()

if featureGatesFlag != "" {
if err := features.MutableFeatureGate.Set(featureGatesFlag); err != nil {
setupLog.Error(err, "unable to parse feature gates", "feature-gates", featureGatesFlag)
os.Exit(1)
}
}
setupLog.Info("feature gates", "NetworkingIntegration", features.FeatureGate.Enabled(features.NetworkingIntegration))

ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))

// Load the federation (Karmada) control plane REST config when
Expand Down Expand Up @@ -270,7 +285,9 @@ func main() {
}

if enableCellControllers {
if err = (&controller.WorkloadDeploymentReconciler{}).SetupWithManager(mgr); err != nil {
if err = (&controller.WorkloadDeploymentReconciler{
NetworkingEnabled: features.FeatureGate.Enabled(features.NetworkingIntegration),
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeployment")
os.Exit(1)
}
Expand Down
36 changes: 30 additions & 6 deletions internal/controller/instancecontrol/stateful/stateful_control.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,30 @@ import (
"go.datum.net/compute/internal/controller/instancecontrol"
)

// Options controls optional behaviours of the stateful instance control strategy.
type Options struct {
// NetworkingEnabled controls whether the Network scheduling gate is added to
// newly created Instances. Set to false when the networking integration is
// disabled so that Instances are not blocked waiting for a NetworkBinding.
// Defaults to true.
NetworkingEnabled bool
}

// Behavior inspired by https://github.com/kubernetes/kubernetes/tree/master/pkg/controller/statefulset
// Does not currently implement exact behavior.
type statefulControl struct {
opts Options
}

// New returns a stateful instance control strategy with networking enabled.
func New() instancecontrol.Strategy {
return &statefulControl{}
return NewWithOptions(Options{NetworkingEnabled: true})
}

// NewWithOptions returns a stateful instance control strategy with the given
// options.
func NewWithOptions(opts Options) instancecontrol.Strategy {
return &statefulControl{opts: opts}
}

func (c *statefulControl) GetActions(
Expand Down Expand Up @@ -69,12 +86,19 @@ func (c *statefulControl) GetActions(
Spec: deployment.Spec.Template.Spec,
}
// TODO(jreese) consider adding scheduling gates via mutating webhooks
desiredInstances[i].Spec.Controller = &v1alpha.InstanceController{
TemplateHash: instanceTemplateHash,
SchedulingGates: []v1alpha.SchedulingGate{
gates := []v1alpha.SchedulingGate{
{Name: instancecontrol.QuotaSchedulingGate.String()},
}
if c.opts.NetworkingEnabled {
// Prepend the Network gate so it is cleared first; quota is
// independent and evaluated in parallel by InstanceReconciler.
gates = append([]v1alpha.SchedulingGate{
{Name: instancecontrol.NetworkSchedulingGate.String()},
{Name: instancecontrol.QuotaSchedulingGate.String()},
},
}, gates...)
}
desiredInstances[i].Spec.Controller = &v1alpha.InstanceController{
TemplateHash: instanceTemplateHash,
SchedulingGates: gates,
}

addInstanceControllerLabels(desiredInstances[i], getInstanceOrdinal(desiredInstances[i].Name), deployment)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,67 @@ func TestScaleDownWithAllReadyInstances(t *testing.T) {
assert.False(t, actions[0].IsSkipped())
}

// TestNetworkingEnabledAddsNetworkGate verifies that when networking is enabled
// (the default), newly created Instances receive both the Network and Quota
// scheduling gates so that they are held until the network is provisioned.
func TestNetworkingEnabledAddsNetworkGate(t *testing.T) {
ctx := context.Background()
control := NewWithOptions(Options{NetworkingEnabled: true})

deployment := getWorkloadDeployment("test-deploy-net-on", 1)

var currentInstances []v1alpha.Instance
actions, err := control.GetActions(ctx, scheme, deployment, currentInstances)

assert.NoError(t, err)
assert.Len(t, actions, 1)
assert.Equal(t, instancecontrol.ActionTypeCreate, actions[0].ActionType())

instance, ok := actions[0].Object.(*v1alpha.Instance)
assert.True(t, ok)
assert.NotNil(t, instance.Spec.Controller)

gateNames := make([]string, 0, len(instance.Spec.Controller.SchedulingGates))
for _, g := range instance.Spec.Controller.SchedulingGates {
gateNames = append(gateNames, g.Name)
}
assert.Contains(t, gateNames, instancecontrol.NetworkSchedulingGate.String(),
"Network gate must be present when networking is enabled")
assert.Contains(t, gateNames, instancecontrol.QuotaSchedulingGate.String(),
"Quota gate must be present")
}

// TestNetworkingDisabledOmitsNetworkGate verifies that when networking is
// disabled, newly created Instances do NOT receive the Network scheduling gate,
// so they are not blocked on network provisioning. The Quota gate is still
// added so quota enforcement remains active.
func TestNetworkingDisabledOmitsNetworkGate(t *testing.T) {
ctx := context.Background()
control := NewWithOptions(Options{NetworkingEnabled: false})

deployment := getWorkloadDeployment("test-deploy-net-off", 1)

var currentInstances []v1alpha.Instance
actions, err := control.GetActions(ctx, scheme, deployment, currentInstances)

assert.NoError(t, err)
assert.Len(t, actions, 1)
assert.Equal(t, instancecontrol.ActionTypeCreate, actions[0].ActionType())

instance, ok := actions[0].Object.(*v1alpha.Instance)
assert.True(t, ok)
assert.NotNil(t, instance.Spec.Controller)

gateNames := make([]string, 0, len(instance.Spec.Controller.SchedulingGates))
for _, g := range instance.Spec.Controller.SchedulingGates {
gateNames = append(gateNames, g.Name)
}
assert.NotContains(t, gateNames, instancecontrol.NetworkSchedulingGate.String(),
"Network gate must NOT be present when networking is disabled")
assert.Contains(t, gateNames, instancecontrol.QuotaSchedulingGate.String(),
"Quota gate must still be present when networking is disabled")
}

// Add more test functions below for different scenarios.

func getWorkloadDeployment(name string, minReplicas int32) *v1alpha.WorkloadDeployment {
Expand Down
65 changes: 46 additions & 19 deletions internal/controller/workloaddeployment_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,13 @@ type WorkloadDeploymentReconciler struct {
// the Karmada namespace after each reconcile so the WorkloadDeploymentFederator
// can aggregate it into the project-namespace object. Set to nil to disable.
KarmadaClient client.Client

// NetworkingEnabled controls whether the networking integration with
// network-services-operator is active. When false, NetworkBinding creation is
// skipped, the Network scheduling gate is never added to Instances (and is
// actively removed if present), and the networking step is treated as
// immediately ready. Defaults to true.
NetworkingEnabled bool
}

// +kubebuilder:rbac:groups=compute.datumapis.com,resources=workloaddeployments,verbs=get;list;watch;create;update;patch;delete
Expand Down Expand Up @@ -107,7 +114,9 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco
return ctrl.Result{}, fmt.Errorf("failed listing instances: %w", err)
}

instanceControl := instancecontrolstateful.New()
instanceControl := instancecontrolstateful.NewWithOptions(instancecontrolstateful.Options{
NetworkingEnabled: r.NetworkingEnabled,
})

actions, err := instanceControl.GetActions(ctx, cl.GetScheme(), &deployment, instances.Items)
if err != nil {
Expand All @@ -129,9 +138,18 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco
}
}

networkReady, err := r.reconcileNetworks(ctx, cl.GetClient(), &deployment)
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed reconciling networks: %w", err)
// When networking is disabled, bypass the entire network provisioning path.
// The Network scheduling gate is treated as cleared and no NetworkBindings
// are created. This lets Instances reach the runtime on cells where
// network-services-operator (VPC) is not yet available.
var networkReady bool
if !r.NetworkingEnabled {
networkReady = true
} else {
networkReady, err = r.reconcileNetworks(ctx, cl.GetClient(), &deployment)
if err != nil {
return ctrl.Result{}, fmt.Errorf("failed reconciling networks: %w", err)
}
}

// Networks are all ready with subnets ready to use, remove any scheduling
Expand Down Expand Up @@ -527,23 +545,32 @@ func (r *WorkloadDeploymentReconciler) SetupWithManager(mgr mcmanager.Manager) e
if err := r.finalizers.Register(workloadControllerFinalizer, r); err != nil {
return fmt.Errorf("failed to register finalizer: %w", err)
}
return mcbuilder.ControllerManagedBy(mgr).

b := mcbuilder.ControllerManagedBy(mgr).
For(&computev1alpha.WorkloadDeployment{}, mcbuilder.WithEngageWithLocalCluster(false)).
Owns(&computev1alpha.Instance{}).
Owns(&networkingv1alpha.NetworkBinding{}).
Watches(&networkingv1alpha.SubnetClaim{}, func(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] {
return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []mcreconcile.Request {
subnetClaim := o.(*networkingv1alpha.SubnetClaim)
return enqueueWorkloadDeploymentByLocation(ctx, mgr, clusterName, subnetClaim.Spec.Location)
})
}).
Watches(&networkingv1alpha.Subnet{}, func(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] {
return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []mcreconcile.Request {
subnet := o.(*networkingv1alpha.Subnet)
return enqueueWorkloadDeploymentByLocation(ctx, mgr, clusterName, subnet.Spec.Location)
Owns(&computev1alpha.Instance{})

// Only watch networking resources when the networking integration is enabled.
// On cells without network-services-operator these watches would log spurious
// errors for missing CRDs.
if r.NetworkingEnabled {
b = b.
Owns(&networkingv1alpha.NetworkBinding{}).
Watches(&networkingv1alpha.SubnetClaim{}, func(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] {
return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []mcreconcile.Request {
subnetClaim := o.(*networkingv1alpha.SubnetClaim)
return enqueueWorkloadDeploymentByLocation(ctx, mgr, clusterName, subnetClaim.Spec.Location)
})
}).
Watches(&networkingv1alpha.Subnet{}, func(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] {
return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []mcreconcile.Request {
subnet := o.(*networkingv1alpha.Subnet)
return enqueueWorkloadDeploymentByLocation(ctx, mgr, clusterName, subnet.Spec.Location)
})
})
}).
Complete(r)
}

return b.Complete(r)
}

func enqueueWorkloadDeploymentByLocation(ctx context.Context, mgr mcmanager.Manager, clusterName multicluster.ClusterName, locationRef networkingv1alpha.LocationReference) []mcreconcile.Request {
Expand Down
59 changes: 59 additions & 0 deletions internal/features/features.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// SPDX-License-Identifier: AGPL-3.0-only

// Package features defines the feature gates for the compute operator. Feature
// gates follow the Kubernetes component-base convention: each feature is
// declared as a Feature constant, registered with a FeatureSpec that includes
// its default enablement state, and toggled at runtime via the --feature-gates
// flag exposed by the binary.
//
// Usage in cmd/main.go:
//
// features.MutableFeatureGate.AddFlag(flag.CommandLine)
//
// Usage in controllers:
//
// if features.MutableFeatureGate.Enabled(features.NetworkingIntegration) { ... }
package features

import (
"k8s.io/component-base/featuregate"
)

const (
// NetworkingIntegration controls whether the compute operator integrates with
// the network-services-operator (VPC) for NetworkBinding provisioning and the
// Network scheduling gate on Instances.
//
// When disabled:
// - No NetworkBinding objects are created.
// - The Network scheduling gate is not added to newly created Instances.
// - Any existing Network scheduling gate is actively removed.
// - The networking step is treated as immediately ready so Instances
// proceed to the runtime without a NetworkBinding.
//
// This flag exists so operators can run compute on edge/lab cells where
// VPC/NSO is not yet functional. The default is true (enabled) so that
// existing production deployments are unaffected.
//
// alpha: v0.1
NetworkingIntegration featuregate.Feature = "NetworkingIntegration"
)

// MutableFeatureGate is the mutable feature gate for the compute operator.
// Call MutableFeatureGate.AddFlag to register the --feature-gates flag before
// flag.Parse(). Controllers should read from FeatureGate (the read-only view)
// after startup.
var MutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate()

// FeatureGate is the read-only view of the compute operator feature gate.
// Use this in controllers and reconcilers rather than MutableFeatureGate to
// avoid accidental mutations after startup.
var FeatureGate featuregate.FeatureGate = MutableFeatureGate

func init() {
if err := MutableFeatureGate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
NetworkingIntegration: {Default: true, PreRelease: featuregate.Alpha},
}); err != nil {
panic(err)
}
}
43 changes: 43 additions & 0 deletions internal/features/features_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// SPDX-License-Identifier: AGPL-3.0-only

package features

import (
"testing"
)

// TestNetworkingIntegration_DefaultEnabled verifies that the NetworkingIntegration
// feature gate defaults to enabled so that existing production deployments are
// unaffected when the flag is not set.
func TestNetworkingIntegration_DefaultEnabled(t *testing.T) {
// Use a fresh gate so this test is independent of any global state mutations.
gate := MutableFeatureGate.DeepCopy()
if !gate.Enabled(NetworkingIntegration) {
t.Error("NetworkingIntegration default = false, want true")
}
}

// TestNetworkingIntegration_CanBeDisabled verifies that setting
// NetworkingIntegration=false via the feature gate string disables the
// integration, allowing operators to run compute without VPC/NSO.
func TestNetworkingIntegration_CanBeDisabled(t *testing.T) {
gate := MutableFeatureGate.DeepCopy()
if err := gate.Set("NetworkingIntegration=false"); err != nil {
t.Fatalf("Set(NetworkingIntegration=false): %v", err)
}
if gate.Enabled(NetworkingIntegration) {
t.Error("NetworkingIntegration = true after Set=false, want false")
}
}

// TestNetworkingIntegration_ExplicitlyEnabled verifies that the gate can be
// explicitly set to true (round-trip).
func TestNetworkingIntegration_ExplicitlyEnabled(t *testing.T) {
gate := MutableFeatureGate.DeepCopy()
if err := gate.Set("NetworkingIntegration=true"); err != nil {
t.Fatalf("Set(NetworkingIntegration=true): %v", err)
}
if !gate.Enabled(NetworkingIntegration) {
t.Error("NetworkingIntegration = false after Set=true, want true")
}
}
Loading