diff --git a/integrationtests/agentmanagement/config/config_test.go b/integrationtests/agentmanagement/config/config_test.go new file mode 100644 index 0000000000..d140dcb501 --- /dev/null +++ b/integrationtests/agentmanagement/config/config_test.go @@ -0,0 +1,77 @@ +package config_test + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/rancher/fleet/internal/config" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("ConfigReconciler", func() { + var cm *corev1.ConfigMap + + BeforeEach(func() { + cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: config.ManagerConfigName, + Namespace: systemNamespace, + }, + } + }) + + AfterEach(func() { + _ = k8sClient.Delete(ctx, cm) + }) + + It("loads config when ConfigMap is created", func() { + data, err := json.Marshal(config.Config{ + AgentImage: "rancher/fleet-agent:test", + }) + Expect(err).NotTo(HaveOccurred()) + + cm.Data = map[string]string{config.Key: string(data)} + Expect(k8sClient.Create(ctx, cm)).To(Succeed()) + + Eventually(func(g Gomega) { + g.Expect(config.Get().AgentImage).To(Equal("rancher/fleet-agent:test")) + }).Should(Succeed()) + }) + + It("reloads config when ConfigMap is updated", func() { + data, err := json.Marshal(config.Config{ + AgentImage: "rancher/fleet-agent:v1", + }) + Expect(err).NotTo(HaveOccurred()) + + cm.Data = map[string]string{config.Key: string(data)} + Expect(k8sClient.Create(ctx, cm)).To(Succeed()) + + Eventually(func(g Gomega) { + g.Expect(config.Get().AgentImage).To(Equal("rancher/fleet-agent:v1")) + }).Should(Succeed()) + + // Update the ConfigMap to a new value + data, err = json.Marshal(config.Config{ + AgentImage: "rancher/fleet-agent:v2", + }) + Expect(err).NotTo(HaveOccurred()) + + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Namespace: systemNamespace, + Name: config.ManagerConfigName, + }, cm)).To(Succeed()) + + cm.Data = map[string]string{config.Key: string(data)} + Expect(k8sClient.Update(ctx, cm)).To(Succeed()) + + Eventually(func(g Gomega) { + g.Expect(config.Get().AgentImage).To(Equal("rancher/fleet-agent:v2")) + }).Should(Succeed()) + }) +}) diff --git a/integrationtests/agentmanagement/config/suite_test.go b/integrationtests/agentmanagement/config/suite_test.go new file mode 100644 index 0000000000..e1af86f4d8 --- /dev/null +++ b/integrationtests/agentmanagement/config/suite_test.go @@ -0,0 +1,76 @@ +package config_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/rancher/fleet/integrationtests/utils" + agentconfig "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/config" + "github.com/rancher/fleet/internal/config" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +const systemNamespace = "cattle-fleet-system" + +var ( + cfg *rest.Config + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc + k8sClient client.Client +) + +func TestController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "AgentManagement Config Suite") +} + +var _ = BeforeSuite(func() { + ctx, cancel = context.WithCancel(context.Background()) + testEnv = utils.NewEnvTest("../../..") + + var err error + cfg, err = utils.StartTestEnv(testEnv) + Expect(err).NotTo(HaveOccurred()) + + k8sClient, err = utils.NewClient(cfg) + Expect(err).NotTo(HaveOccurred()) + + // Initialize global config to prevent config.Get() panics during test setup. + config.Set(config.DefaultConfig()) + + // Create system namespace before starting the manager + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: systemNamespace}, + } + Expect(k8sClient.Create(ctx, ns)).To(Succeed()) + + mgr, err := utils.NewManager(cfg) + Expect(err).NotTo(HaveOccurred()) + + err = (&agentconfig.ConfigReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SystemNamespace: systemNamespace, + }).SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() +}) + +var _ = AfterSuite(func() { + cancel() + Expect(testEnv.Stop()).ToNot(HaveOccurred()) +}) diff --git a/internal/cmd/controller/agentmanagement/controllers/config/reconciler.go b/internal/cmd/controller/agentmanagement/controllers/config/reconciler.go new file mode 100644 index 0000000000..c9d6829484 --- /dev/null +++ b/internal/cmd/controller/agentmanagement/controllers/config/reconciler.go @@ -0,0 +1,71 @@ +// Package config reads the initial global configuration. +package config + +import ( + "context" + + "github.com/rancher/fleet/internal/config" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// ConfigReconciler reconciles the Fleet config object for agentmanagement, +// by reloading the config on change. +type ConfigReconciler struct { + client.Client + Scheme *runtime.Scheme + + SystemNamespace string +} + +// SetupWithManager sets up the controller with the Manager. +func (r *ConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.ConfigMap{}). + WithEventFilter( + predicate.And( + predicate.NewPredicateFuncs(func(object client.Object) bool { + return object.GetNamespace() == r.SystemNamespace && + object.GetName() == config.ManagerConfigName + }), + predicate.Or( + predicate.ResourceVersionChangedPredicate{}, + predicate.GenerationChangedPredicate{}, + predicate.AnnotationChangedPredicate{}, + predicate.LabelChangedPredicate{}, + ), + ), + ). + Complete(r) +} + +// Reconcile reloads the Fleet config from the ConfigMap when it changes. +func (r *ConfigReconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx).WithName("agentmanagement-config") + ctx = log.IntoContext(ctx, logger) + + cm := &corev1.ConfigMap{} + err := r.Get(ctx, types.NamespacedName{Namespace: r.SystemNamespace, Name: config.ManagerConfigName}, cm) + if client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err + } + + logger.V(1).Info("Reconciling config configmap, loading config") + + cfg, err := config.ReadConfig(cm) + if err != nil { + return ctrl.Result{}, err + } + + // SetAndTrigger is used during the wrangler-to-CR migration to ensure + // wrangler components (bootstrap, cluster/import) that register config.OnChange + // callbacks still receive config change notifications. + // TODO: Switch to config.Set() once those wrangler components are ported (Phases 3, 8). + return ctrl.Result{}, config.SetAndTrigger(cfg) +} diff --git a/internal/cmd/controller/agentmanagement/controllers/controllers.go b/internal/cmd/controller/agentmanagement/controllers/controllers.go index b930ab57eb..954794d499 100644 --- a/internal/cmd/controller/agentmanagement/controllers/controllers.go +++ b/internal/cmd/controller/agentmanagement/controllers/controllers.go @@ -7,7 +7,6 @@ import ( "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/cluster" "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/clusterregistration" "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/clusterregistrationtoken" - "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/config" "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/manageagent" "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/resources" fleetns "github.com/rancher/fleet/internal/cmd/controller/namespace" @@ -58,14 +57,6 @@ func (a *AppContext) Start(ctx context.Context) error { func Register(ctx context.Context, appCtx *AppContext, systemNamespace string, disableBootstrap bool) error { systemRegistrationNamespace := fleetns.SystemRegistrationNamespace(systemNamespace) - // config should be registered first to ensure the global - // config is available to all components - if err := config.Register(ctx, - systemNamespace, - appCtx.Core.ConfigMap()); err != nil { - return err - } - if err := resources.ApplyBootstrapResources( systemNamespace, systemRegistrationNamespace, diff --git a/internal/cmd/controller/agentmanagement/start.go b/internal/cmd/controller/agentmanagement/start.go index 48eebbb12e..1c904853c2 100644 --- a/internal/cmd/controller/agentmanagement/start.go +++ b/internal/cmd/controller/agentmanagement/start.go @@ -4,6 +4,8 @@ import ( "context" "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers" + agentconfig "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/controllers/config" + fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1" "github.com/rancher/wrangler/v3/pkg/kubeconfig" "github.com/rancher/wrangler/v3/pkg/leader" @@ -14,10 +16,22 @@ import ( v1 "k8s.io/api/apps/v1" policyv1 "k8s.io/api/policy/v1" schedulingv1 "k8s.io/api/scheduling/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) +var agentScheme = runtime.NewScheme() + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(agentScheme)) + utilruntime.Must(fleet.AddToScheme(agentScheme)) +} + func start(ctx context.Context, kubeConfig, namespace string, disableBootstrap bool) error { clientConfig := kubeconfig.GetNonInteractiveClientConfig(kubeConfig) kc, err := clientConfig.ClientConfig() @@ -47,6 +61,33 @@ func start(ctx context.Context, kubeConfig, namespace string, disableBootstrap b } leader.RunOrDie(ctx, namespace, "fleet-agentmanagement-lock", k8s, func(ctx context.Context) { + // Create controller-runtime manager. Leader election is disabled because + // wrangler's leader.RunOrDie already holds the lease; the manager starts + // inside the leader callback. + mgr, err := ctrl.NewManager(kc, ctrl.Options{ + Scheme: agentScheme, + LeaderElection: false, + Metrics: metricsserver.Options{BindAddress: "0"}, + HealthProbeBindAddress: "", + }) + if err != nil { + logrus.Fatal(err) + } + + if err := (&agentconfig.ConfigReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SystemNamespace: namespace, + }).SetupWithManager(mgr); err != nil { + logrus.Fatal(err) + } + + go func() { + if err := mgr.Start(ctx); err != nil { + logrus.Fatal(err) + } + }() + appCtx, err := controllers.NewAppContext(clientConfig) if err != nil { logrus.Fatal(err)