Adds tolerations from fleet-controller when bootstraping and when running the fleet apply job (rancher#3362)

Adding the toleration needed to the helm chart in Fleet is not enough when running the agent and the fleet apply job.

This PR adds the tolerations found in the `fleet-controller` deployment to the agent and to the fleet apply job.

Refers to: rancher#3313

Signed-off-by: Xavi Garcia <xavi.garcia@suse.com>

Author: 0xavi0

charts/fleet/templates/rbac.yaml

@@ -38,6 +38,13 @@ rules:
     - 'events'
   verbs:
     - '*'
+- apiGroups:
+    - "apps"
+  resources:
+    - 'deployments'
+  verbs:
+    - 'list'
+    - 'get'
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

charts/fleet/templates/rbac_gitjob.yaml

@@ -99,7 +99,14 @@ rules:
       - list
       - watch
       - update
-
+  - apiGroups:
+      - "apps"
+    resources:
+      - 'deployments'
+    verbs:
+      - 'list'
+      - 'get'
+      - 'watch'
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

integrationtests/gitjob/controller/suite_test.go

@@ -14,6 +14,8 @@ import (
 	gomegatypes "github.com/onsi/gomega/types"
 	"github.com/reugn/go-quartz/quartz"
 	"go.uber.org/mock/gomock"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 
 	"github.com/rancher/fleet/internal/cmd/controller/gitops/reconciler"
 	ctrlreconciler "github.com/rancher/fleet/internal/cmd/controller/reconciler"
@@ -100,20 +102,53 @@ var _ = BeforeSuite(func() {
 		},
 	)
 
+	// fleet-controller deployment
+	err = k8sClient.Create(ctx, &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      config.ManagerConfigName,
+			Namespace: "default",
+		},
+		Spec: appsv1.DeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"app": "fleet-controller",
+				},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"app": "fleet-controller",
+					},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "test",
+							Image: "test", // value is required. but we don't need a real deployment for the test
+
+						},
+					},
+				},
+			},
+		},
+	})
+	Expect(err).ToNot(HaveOccurred())
+
 	sched := quartz.NewStdScheduler()
 	Expect(sched).ToNot(BeNil())
 
 	config.Set(&config.Config{})
 
 	err = (&reconciler.GitJobReconciler{
-		Client:     mgr.GetClient(),
-		Scheme:     mgr.GetScheme(),
-		Image:      "image",
-		Scheduler:  sched,
-		GitFetcher: fetcherMock,
-		Clock:      reconciler.RealClock{},
-		Recorder:   mgr.GetEventRecorderFor("gitjob-controller"),
-		Workers:    50,
+		Client:          mgr.GetClient(),
+		Scheme:          mgr.GetScheme(),
+		Image:           "image",
+		Scheduler:       sched,
+		GitFetcher:      fetcherMock,
+		Clock:           reconciler.RealClock{},
+		Recorder:        mgr.GetEventRecorderFor("gitjob-controller"),
+		Workers:         50,
+		SystemNamespace: "default",
 	}).SetupWithManager(mgr)
 	Expect(err).ToNot(HaveOccurred())
 

internal/cmd/controller/agentmanagement/controllers/bootstrap/bootstrap.go

@@ -10,14 +10,16 @@ import (
 
 	secretutil "github.com/rancher/fleet/internal/cmd/controller/agentmanagement/secret"
 	fleetns "github.com/rancher/fleet/internal/cmd/controller/namespace"
-	"github.com/rancher/fleet/internal/config"
+	fleetconfig "github.com/rancher/fleet/internal/config"
 	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 	"github.com/rancher/wrangler/v3/pkg/apply"
+	appscontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/apps/v1"
 	corecontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -39,6 +41,7 @@ type handler struct {
 	serviceAccountCache corecontrollers.ServiceAccountCache
 	secretsCache        corecontrollers.SecretCache
 	secretsController   corecontrollers.SecretController
+	deploymentsCache    appscontrollers.DeploymentCache
 	cfg                 clientcmd.ClientConfig
 }
 
@@ -49,19 +52,21 @@ func Register(ctx context.Context,
 	serviceAccountCache corecontrollers.ServiceAccountCache,
 	secretsController corecontrollers.SecretController,
 	secretsCache corecontrollers.SecretCache,
+	deploymentCache appscontrollers.DeploymentCache,
 ) {
 	h := handler{
 		systemNamespace:     systemNamespace,
 		serviceAccountCache: serviceAccountCache,
 		secretsCache:        secretsCache,
 		secretsController:   secretsController,
+		deploymentsCache:    deploymentCache,
 		apply:               apply.WithSetID("fleet-bootstrap"),
 		cfg:                 cfg,
 	}
-	config.OnChange(ctx, h.OnConfig)
+	fleetconfig.OnChange(ctx, h.OnConfig)
 }
 
-func (h *handler) OnConfig(config *config.Config) error {
+func (h *handler) OnConfig(config *fleetconfig.Config) error {
 	logrus.Debugf("Bootstrap config set, building namespace '%s', secret, local cluster, cluster group, ...", config.Bootstrap.Namespace)
 
 	var objs []runtime.Object
@@ -74,6 +79,10 @@ func (h *handler) OnConfig(config *config.Config) error {
 	if err != nil {
 		return err
 	}
+	fleetControllerDeployment, err := h.deploymentsCache.Get(h.systemNamespace, fleetconfig.ManagerConfigName)
+	if err != nil {
+		return err
+	}
 	objs = append(objs, &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: config.Bootstrap.Namespace,
@@ -89,6 +98,8 @@ func (h *handler) OnConfig(config *config.Config) error {
 		Spec: fleet.ClusterSpec{
 			KubeConfigSecret: secret.Name,
 			AgentNamespace:   config.Bootstrap.AgentNamespace,
+			// copy tolerations from fleet-controller
+			AgentTolerations: fleetControllerDeployment.Spec.Template.Spec.Tolerations,
 		},
 	}, &fleet.ClusterGroup{
 		ObjectMeta: metav1.ObjectMeta{
@@ -164,9 +175,9 @@ func (h *handler) buildSecret(bootstrapNamespace string, cfg clientcmd.ClientCon
 			},
 		},
 		Data: map[string][]byte{
-			config.KubeConfigSecretValueKey: value,
-			config.APIServerURLKey:          []byte(host),
-			config.APIServerCAKey:           ca,
+			fleetconfig.KubeConfigSecretValueKey: value,
+			fleetconfig.APIServerURLKey:          []byte(host),
+			fleetconfig.APIServerCAKey:           ca,
 		},
 	}, nil
 }

internal/cmd/controller/agentmanagement/controllers/controllers.go

@@ -20,6 +20,8 @@ import (
 	"github.com/rancher/lasso/pkg/client"
 	"github.com/rancher/lasso/pkg/controller"
 	"github.com/rancher/wrangler/v3/pkg/apply"
+	"github.com/rancher/wrangler/v3/pkg/generated/controllers/apps"
+	appscontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/apps/v1"
 	"github.com/rancher/wrangler/v3/pkg/generated/controllers/core"
 	corecontrollers "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
 	"github.com/rancher/wrangler/v3/pkg/generated/controllers/rbac"
@@ -42,6 +44,7 @@ type AppContext struct {
 
 	K8s          kubernetes.Interface
 	Core         corecontrollers.Interface
+	Apps         appscontrollers.Interface
 	RBAC         rbaccontrollers.Interface
 	RESTMapper   meta.RESTMapper
 	Apply        apply.Apply
@@ -86,7 +89,8 @@ func Register(ctx context.Context, appCtx *AppContext, systemNamespace string, d
 			appCtx.ClientConfig,
 			appCtx.Core.ServiceAccount().Cache(),
 			appCtx.Core.Secret(),
-			appCtx.Core.Secret().Cache())
+			appCtx.Core.Secret().Cache(),
+			appCtx.Apps.Deployment().Cache())
 	}
 
 	cluster.Register(ctx,
@@ -182,6 +186,14 @@ func NewAppContext(cfg clientcmd.ClientConfig) (*AppContext, error) {
 	}
 	rbacv := rbac.Rbac().V1()
 
+	apps, err := apps.NewFactoryFromConfigWithOptions(client, &apps.FactoryOptions{
+		SharedControllerFactory: scf,
+	})
+	if err != nil {
+		return nil, err
+	}
+	appsv := apps.Apps().V1()
+
 	apply, err := apply.NewForConfig(client)
 	if err != nil {
 		return nil, err
@@ -197,6 +209,7 @@ func NewAppContext(cfg clientcmd.ClientConfig) (*AppContext, error) {
 		K8s:          k8s,
 		Interface:    fleetv,
 		Core:         corev,
+		Apps:         appsv,
 		RBAC:         rbacv,
 		Apply:        apply,
 		ClientConfig: cfg,

internal/cmd/controller/gitops/operator.go

@@ -144,6 +144,7 @@ func (g *GitOperator) Run(cmd *cobra.Command, args []string) error {
 		GitFetcher:      &git.Fetch{},
 		Clock:           reconciler.RealClock{},
 		Recorder:        mgr.GetEventRecorderFor(fmt.Sprintf("fleet-gitops%s", shardIDSuffix)),
+		SystemNamespace: namespace,
 	}
 
 	statusReconciler := &reconciler.StatusReconciler{

internal/cmd/controller/gitops/reconciler/gitjob_controller.go

@@ -18,6 +18,7 @@ import (
 	fleetutil "github.com/rancher/fleet/internal/cmd/controller/errorutil"
 	"github.com/rancher/fleet/internal/cmd/controller/finalize"
 	"github.com/rancher/fleet/internal/cmd/controller/imagescan"
+	"github.com/rancher/fleet/internal/config"
 	"github.com/rancher/fleet/internal/metrics"
 	"github.com/rancher/fleet/internal/names"
 	"github.com/rancher/fleet/internal/ociwrapper"
@@ -30,6 +31,7 @@ import (
 	"github.com/rancher/wrangler/v3/pkg/genericcondition"
 	"github.com/rancher/wrangler/v3/pkg/kstatus"
 
+	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -98,6 +100,7 @@ type GitJobReconciler struct {
 	GitFetcher      GitFetcher
 	Clock           TimeGetter
 	Recorder        record.EventRecorder
+	SystemNamespace string
 }
 
 func (r *GitJobReconciler) SetupWithManager(mgr ctrl.Manager) error {
@@ -562,7 +565,19 @@ func (r *GitJobReconciler) newGitJob(ctx context.Context, obj *v1alpha1.GitRepo)
 	if err != nil {
 		return nil, err
 	}
+	var fleetControllerDeployment appsv1.Deployment
+	if err := r.Get(ctx, types.NamespacedName{
+		Namespace: r.SystemNamespace,
+		Name:      config.ManagerConfigName,
+	}, &fleetControllerDeployment); err != nil {
+		return nil, err
+	}
 
+	// add tolerations from the fleet-controller deployment
+	jobSpec.Template.Spec.Tolerations = append(
+		jobSpec.Template.Spec.Tolerations,
+		fleetControllerDeployment.Spec.Template.Spec.Tolerations...,
+	)
 	job := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Annotations: map[string]string{