From 97f87e1c2de7e7c112cd53c46422526443bd8675 Mon Sep 17 00:00:00 2001 From: Xavi Garcia Date: Tue, 17 Mar 2026 10:03:41 +0100 Subject: [PATCH] agentmanagement. Ports config controler to controller-runtime This PR is the first of a series to port the `agentmanagement` component to use controller-runtime. It creates the foundation to the rest of changes and marks the way iterations will be executed. The first step is to add testing coverage to the component to be ported, still running with `wrangler` then it ports the component and, without changing the tests, it verifies that nothing was broken. Signed-off-by: Xavi Garcia --- .../agentmanagement/config/config_test.go | 77 +++++++++++++++++++ .../agentmanagement/config/suite_test.go | 76 ++++++++++++++++++ .../controllers/config/reconciler.go | 71 +++++++++++++++++ .../controllers/controllers.go | 9 --- .../cmd/controller/agentmanagement/start.go | 41 ++++++++++ 5 files changed, 265 insertions(+), 9 deletions(-) create mode 100644 integrationtests/agentmanagement/config/config_test.go create mode 100644 integrationtests/agentmanagement/config/suite_test.go create mode 100644 internal/cmd/controller/agentmanagement/controllers/config/reconciler.go diff --git a/integrationtests/agentmanagement/config/config_test.go b/integrationtests/agentmanagement/config/config_test.go new file mode 100644 index 0000000000..d140dcb501 --- /dev/null +++ b/integrationtests/agentmanagement/config/config_test.go @@ -0,0 +1,77 @@ +package config_test + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/rancher/fleet/internal/config" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("ConfigReconciler", func() { + var cm *corev1.ConfigMap + + BeforeEach(func() { + cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: config.ManagerConfigName, + Namespace: systemNamespace, + }, + } + }) + + AfterEach(func() { + _ = k8sClient.Delete(ctx, cm) + }) + + It("loads config when ConfigMap is created", func() { + data, err := json.Marshal(config.Config{ + AgentImage: "rancher/fleet-agent:test", + }) + Expect(err).NotTo(HaveOccurred()) + + cm.Data = map[string]string{config.Key: string(data)} + Expect(k8sClient.Create(ctx, cm)).To(Succeed()) + + Eventually(func(g Gomega) { + g.Expect(config.Get().AgentImage).To(Equal("rancher/fleet-agent:test")) + }).Should(Succeed()) + }) + + It("reloads config when ConfigMap is updated", func() { + data, err := json.Marshal(config.Config{ + AgentImage: "rancher/fleet-agent:v1", + }) + Expect(err).NotTo(HaveOccurred()) + + cm.Data = map[string]string{config.Key: string(data)} + Expect(k8sClient.Create(ctx, cm)).To(Succeed()) + + Eventually(func(g Gomega) { + g.Expect(config.Get().AgentImage).To(Equal("rancher/fleet-agent:v1")) + }).Should(Succeed()) + + // Update the ConfigMap to a new value + data, err = json.Marshal(config.Config{ + AgentImage: "rancher/fleet-agent:v2", + }) + Expect(err).NotTo(HaveOccurred()) + + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Namespace: systemNamespace, + Name: config.ManagerConfigName, + }, cm)).To(Succeed()) + + cm.Data = map[string]string{config.Key: string(data)} + Expect(k8sClient.Update(ctx, cm)).To(Succeed()) + + Eventually(func(g Gomega) { + g.Expect(config.Get().AgentImage).To(Equal("rancher/fleet-agent:v2")) + }).Should(Succeed()) + }) +}) diff --git a/integrationtests/agentmanagement/config/suite_test.go b/integrationtests/agentmanagement/config/suite_test.go new file mode 100644 index 0000000000..e1af86f4d8 --- /dev/null +++ b/integrationtests/agentmanagement/config/suite_test.go @@ -0,0 +1,76 @@ +package config_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/rancher/fleet/integrationtests/utils" + agentconfig "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/config" + "github.com/rancher/fleet/internal/config" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +const systemNamespace = "cattle-fleet-system" + +var ( + cfg *rest.Config + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc + k8sClient client.Client +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "AgentManagement Config Suite") +} + +var _ = BeforeSuite(func() { + ctx, cancel = context.WithCancel(context.Background()) + testEnv = utils.NewEnvTest("../../..") + + var err error + cfg, err = utils.StartTestEnv(testEnv) + Expect(err).NotTo(HaveOccurred()) + + k8sClient, err = utils.NewClient(cfg) + Expect(err).NotTo(HaveOccurred()) + + // Initialize global config to prevent config.Get() panics during test setup. + config.Set(config.DefaultConfig()) + + // Create system namespace before starting the manager + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: systemNamespace}, + } + Expect(k8sClient.Create(ctx, ns)).To(Succeed()) + + mgr, err := utils.NewManager(cfg) + Expect(err).NotTo(HaveOccurred()) + + err = (&agentconfig.ConfigReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SystemNamespace: systemNamespace, + }).SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() +}) + +var _ = AfterSuite(func() { + cancel() + Expect(testEnv.Stop()).ToNot(HaveOccurred()) +}) diff --git a/internal/cmd/controller/agentmanagement/controllers/config/reconciler.go b/internal/cmd/controller/agentmanagement/controllers/config/reconciler.go new file mode 100644 index 0000000000..c9d6829484 --- /dev/null +++ b/internal/cmd/controller/agentmanagement/controllers/config/reconciler.go @@ -0,0 +1,71 @@ +// Package config reads the initial global configuration. +package config + +import ( + "context" + + "github.com/rancher/fleet/internal/config" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// ConfigReconciler reconciles the Fleet config object for agentmanagement, +// by reloading the config on change. +type ConfigReconciler struct { + client.Client + Scheme *runtime.Scheme + + SystemNamespace string +} + +// SetupWithManager sets up the controller with the Manager. +func (r *ConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.ConfigMap{}). + WithEventFilter( + predicate.And( + predicate.NewPredicateFuncs(func(object client.Object) bool { + return object.GetNamespace() == r.SystemNamespace && + object.GetName() == config.ManagerConfigName + }), + predicate.Or( + predicate.ResourceVersionChangedPredicate{}, + predicate.GenerationChangedPredicate{}, + predicate.AnnotationChangedPredicate{}, + predicate.LabelChangedPredicate{}, + ), + ), + ). + Complete(r) +} + +// Reconcile reloads the Fleet config from the ConfigMap when it changes. +func (r *ConfigReconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx).WithName("agentmanagement-config") + ctx = log.IntoContext(ctx, logger) + + cm := &corev1.ConfigMap{} + err := r.Get(ctx, types.NamespacedName{Namespace: r.SystemNamespace, Name: config.ManagerConfigName}, cm) + if client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err + } + + logger.V(1).Info("Reconciling config configmap, loading config") + + cfg, err := config.ReadConfig(cm) + if err != nil { + return ctrl.Result{}, err + } + + // SetAndTrigger is used during the wrangler-to-CR migration to ensure + // wrangler components (bootstrap, cluster/import) that register config.OnChange + // callbacks still receive config change notifications. + // TODO: Switch to config.Set() once those wrangler components are ported (Phases 3, 8). + return ctrl.Result{}, config.SetAndTrigger(cfg) +} diff --git a/internal/cmd/controller/agentmanagement/controllers/controllers.go b/internal/cmd/controller/agentmanagement/controllers/controllers.go index b930ab57eb..954794d499 100644 --- a/internal/cmd/controller/agentmanagement/controllers/controllers.go +++ b/internal/cmd/controller/agentmanagement/controllers/controllers.go @@ -7,7 +7,6 @@ import ( "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/cluster" "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/clusterregistration" "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/clusterregistrationtoken" - "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/config" "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/manageagent" "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/resources" fleetns "github.com/rancher/fleet/internal/cmd/controller/namespace" @@ -58,14 +57,6 @@ func (a *AppContext) Start(ctx context.Context) error { func Register(ctx context.Context, appCtx *AppContext, systemNamespace string, disableBootstrap bool) error { systemRegistrationNamespace := fleetns.SystemRegistrationNamespace(systemNamespace) - // config should be registered first to ensure the global - // config is available to all components - if err := config.Register(ctx, - systemNamespace, - appCtx.Core.ConfigMap()); err != nil { - return err - } - if err := resources.ApplyBootstrapResources( systemNamespace, systemRegistrationNamespace, diff --git a/internal/cmd/controller/agentmanagement/start.go b/internal/cmd/controller/agentmanagement/start.go index 48eebbb12e..1c904853c2 100644 --- a/internal/cmd/controller/agentmanagement/start.go +++ b/internal/cmd/controller/agentmanagement/start.go @@ -4,6 +4,8 @@ import ( "context" "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers" + agentconfig "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/config" + fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1" "github.com/rancher/wrangler/v3/pkg/kubeconfig" "github.com/rancher/wrangler/v3/pkg/leader" @@ -14,10 +16,22 @@ import ( v1 "k8s.io/api/apps/v1" policyv1 "k8s.io/api/policy/v1" schedulingv1 "k8s.io/api/scheduling/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) +var agentScheme = runtime.NewScheme() + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(agentScheme)) + utilruntime.Must(fleet.AddToScheme(agentScheme)) +} + func start(ctx context.Context, kubeConfig, namespace string, disableBootstrap bool) error { clientConfig := kubeconfig.GetNonInteractiveClientConfig(kubeConfig) kc, err := clientConfig.ClientConfig() @@ -47,6 +61,33 @@ func start(ctx context.Context, kubeConfig, namespace string, disableBootstrap b } leader.RunOrDie(ctx, namespace, "fleet-agentmanagement-lock", k8s, func(ctx context.Context) { + // Create controller-runtime manager. Leader election is disabled because + // wrangler's leader.RunOrDie already holds the lease; the manager starts + // inside the leader callback. + mgr, err := ctrl.NewManager(kc, ctrl.Options{ + Scheme: agentScheme, + LeaderElection: false, + Metrics: metricsserver.Options{BindAddress: "0"}, + HealthProbeBindAddress: "", + }) + if err != nil { + logrus.Fatal(err) + } + + if err := (&agentconfig.ConfigReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SystemNamespace: namespace, + }).SetupWithManager(mgr); err != nil { + logrus.Fatal(err) + } + + go func() { + if err := mgr.Start(ctx); err != nil { + logrus.Fatal(err) + } + }() + appCtx, err := controllers.NewAppContext(clientConfig) if err != nil { logrus.Fatal(err)