Make cluster-proxy work with multicluster-controlplane

Signed-off-by: Tamal Saha <tamal@appscode.com>

Author: tamalsaha

.gitignore

@@ -12,7 +12,7 @@
 *.out
 
 # Dependency directories (remove the comment below to include it)
-# vendor/
+vendor/
 
 /apiserver.local.config
 /bin

Makefile

@@ -1,9 +1,9 @@
 
 # Image URL to use all building/pushing image targets
-IMG ?= controller:latest
 IMAGE_REGISTRY_NAME ?= quay.io/open-cluster-management
 IMAGE_NAME = cluster-proxy
 IMAGE_TAG ?= latest
+IMG ?= $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG)
 E2E_TEST_CLUSTER_NAME ?= loopback
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:crdVersions={v1},allowDangerousTypes=true,generateEmbeddedObjectMeta=true"

charts/cluster-proxy/templates/manager-deployment.yaml

@@ -17,7 +17,7 @@ spec:
       containers:
         - name: manager
           image: {{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag | default (print "v" .Chart.Version) }}
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: Always
           command:
             - /manager
           args:

charts/cluster-proxy/values.yaml

@@ -1,19 +1,19 @@
 # Image registry
-registry: quay.io/open-cluster-management
+registry: ghcr.io/kluster-manager
 
 # Image of the cluster-gateway instances
 image: cluster-proxy
 
 # Image tag
-tag:
+tag: latest
 
 # Number of replicas
 replicas: 1
 
 spokeAddonNamespace: "open-cluster-management-cluster-proxy"
 
-proxyServerImage: quay.io/open-cluster-management/cluster-proxy
-proxyAgentImage: quay.io/open-cluster-management/cluster-proxy
+proxyServerImage: ghcr.io/kluster-manager/cluster-proxy
+proxyAgentImage: ghcr.io/kluster-manager/cluster-proxy
 
 proxyServer:
   entrypointLoadBalancer: false

cmd/addon-manager/main.go

@@ -31,6 +31,8 @@ import (
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/textlogger"
 	"open-cluster-management.io/addon-framework/pkg/addonmanager"
@@ -46,6 +48,8 @@ import (
 	"open-cluster-management.io/cluster-proxy/pkg/proxyserver/controllers"
 	"open-cluster-management.io/cluster-proxy/pkg/proxyserver/operator/authentication/selfsigned"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	//+kubebuilder:scaffold:imports
@@ -72,6 +76,7 @@ func main() {
 	var signerSecretNamespace, signerSecretName string
 	var agentInstallAll bool
 	var enableKubeApiProxy bool
+	var mcKubeconfig string
 
 	logger := textlogger.NewLogger(textlogger.NewConfig())
 	klog.SetOutput(os.Stdout)
@@ -95,13 +100,30 @@ func main() {
 		"Configure the install strategy of agent on managed clusters. "+
 			"Enabling this will automatically install agent on all managed cluster.")
 	flag.BoolVar(&enableKubeApiProxy, "enable-kube-api-proxy", true, "Enable proxy to agent kube-apiserver")
+	flag.StringVar(&mcKubeconfig, "multicluster-kubeconfig", "",
+		"The path to multicluster-controlplane kubeconfig")
 
 	flag.Parse()
 
 	// pipe controller-runtime logs to klog
 	ctrl.SetLogger(logger)
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+	var mcConfig, hostConfig *rest.Config
+
+	if mcKubeconfig != "" {
+		var err error
+		mcConfig, err = clientcmd.BuildConfigFromFlags("", mcKubeconfig)
+		if err != nil {
+			setupLog.Error(err, "unable to build multicluster rest config")
+			os.Exit(1)
+		}
+		hostConfig = ctrl.GetConfigOrDie()
+	} else {
+		hostConfig = ctrl.GetConfigOrDie()
+		mcConfig = hostConfig
+	}
+
+	mgr, err := ctrl.NewManager(mcConfig, ctrl.Options{
 		Scheme:                 scheme,
 		Metrics:                metricsserver.Options{BindAddress: metricsAddr},
 		HealthProbeBindAddress: probeAddr,
@@ -119,6 +141,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	hostClient, err := kubernetes.NewForConfig(hostConfig)
+	if err != nil {
+		setupLog.Error(err, "unable to set up host kubernetes native client")
+		os.Exit(1)
+	}
+
 	nativeClient, err := kubernetes.NewForConfig(mgr.GetConfig())
 	if err != nil {
 		setupLog.Error(err, "unable to set up kubernetes native client")
@@ -147,22 +175,30 @@ func main() {
 	}
 
 	informerFactory := externalversions.NewSharedInformerFactory(client, 0)
-	nativeInformer := informers.NewSharedInformerFactoryWithOptions(nativeClient, 0)
+	hostInformer := informers.NewSharedInformerFactoryWithOptions(hostClient, 0, informers.WithNamespace(signerSecretNamespace))
 
 	// loading self-signer
 	selfSigner, err := selfsigned.NewSelfSignerFromSecretOrGenerate(
-		nativeClient, signerSecretNamespace, signerSecretName)
+		hostClient, signerSecretNamespace, signerSecretName)
 	if err != nil {
 		setupLog.Error(err, "failed loading self-signer")
 		os.Exit(1)
 	}
 
+	hostKubeClient, err := newHostClient(hostConfig)
+	if err != nil {
+		setupLog.Error(err, "failed create host KubeClient")
+		os.Exit(1)
+	}
+
 	if err := controllers.RegisterClusterManagementAddonReconciler(
 		mgr,
 		selfSigner,
-		nativeClient,
-		nativeInformer.Core().V1().Secrets(),
+		hostKubeClient,
+		hostClient,
+		hostInformer.Core().V1().Secrets(),
 		supportsV1CSR,
+		mcKubeconfig != "",
 	); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ClusterManagementAddonReconciler")
 		os.Exit(1)
@@ -195,6 +231,7 @@ func main() {
 		supportsV1CSR,
 		mgr.GetClient(),
 		nativeClient,
+		hostClient,
 		agentInstallAll,
 		enableKubeApiProxy,
 		addonClient,
@@ -212,7 +249,7 @@ func main() {
 	ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
 	defer cancel()
 	go informerFactory.Start(ctx.Done())
-	go nativeInformer.Start(ctx.Done())
+	go hostInformer.Start(ctx.Done())
 	go func() {
 		if err := addonManager.Start(ctx); err != nil {
 			setupLog.Error(err, "unable to start addon manager")
@@ -225,3 +262,18 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func newHostClient(hostConfig *rest.Config) (client.Client, error) {
+	hc, err := rest.HTTPClientFor(hostConfig)
+	if err != nil {
+		return nil, err
+	}
+	mapper, err := apiutil.NewDynamicRESTMapper(hostConfig, hc)
+	if err != nil {
+		return nil, err
+	}
+	return client.New(hostConfig, client.Options{
+		Scheme: clientgoscheme.Scheme,
+		Mapper: mapper,
+	})
+}

pkg/proxyagent/agent/agent.go

@@ -53,6 +53,7 @@ func NewAgentAddon(
 	v1CSRSupported bool,
 	runtimeClient client.Client,
 	nativeClient kubernetes.Interface,
+	hostClient kubernetes.Interface,
 	agentInstallAll bool,
 	enableKubeApiProxy bool,
 	addonClient addonclient.Interface) (agent.AgentAddon, error) {
@@ -134,7 +135,7 @@ func NewAgentAddon(
 			utils.AddOnDeploymentConfigGVR,
 		).
 		WithGetValuesFuncs(
-			GetClusterProxyValueFunc(runtimeClient, nativeClient, signerNamespace, caCertData, v1CSRSupported, enableKubeApiProxy),
+			GetClusterProxyValueFunc(runtimeClient, hostClient, signerNamespace, caCertData, v1CSRSupported, enableKubeApiProxy),
 			addonfactory.GetAddOnDeploymentConfigValues(
 				utils.NewAddOnDeploymentConfigGetter(addonClient),
 				toAgentAddOnChartValues(caCertData),

pkg/proxyagent/agent/manifests/charts/addon-agent/templates/addon-agent-deployment.yaml

@@ -38,7 +38,7 @@ spec:
       containers:
         - name: proxy-agent
           image: {{ .Values.proxyAgentImage }}
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: Always
           command:
             - /proxy-agent
           args:
@@ -90,7 +90,7 @@ spec:
               cpu: "300m"
         - name: addon-agent
           image: {{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }}
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: Always
           command:
             - /agent
           args:

pkg/proxyserver/controllers/managedproxyconfiguration_controller.go

@@ -18,6 +18,7 @@ import (
 	"github.com/openshift/library-go/pkg/operator/events"
 	"github.com/pkg/errors"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -47,12 +48,15 @@ var log = ctrl.Log.WithName("ClusterManagementAddonReconciler")
 func RegisterClusterManagementAddonReconciler(
 	mgr manager.Manager,
 	selfSigner selfsigned.SelfSigner,
+	hostClient client.Client,
 	nativeClient kubernetes.Interface,
 	secretInformer informercorev1.SecretInformer,
 	supportsV1CSR bool,
+	mcMode bool,
 ) error {
 	r := &ManagedProxyConfigurationReconciler{
 		Client:     mgr.GetClient(),
+		HostClient: hostClient,
 		SelfSigner: selfSigner,
 		CAPair:     selfSigner.CA(),
 		newCertRotatorFunc: func(namespace, name string, sans ...string) selfsigned.CertRotation {
@@ -72,12 +76,14 @@ func RegisterClusterManagementAddonReconciler(
 		EventRecorder:    events.NewInMemoryRecorder("ClusterManagementAddonReconciler"),
 
 		supportsV1CSR: supportsV1CSR,
+		mcMode:        mcMode,
 	}
 	return r.SetupWithManager(mgr)
 }
 
 type ManagedProxyConfigurationReconciler struct {
-	client.Client
+	Client           client.Client
+	HostClient       client.Client
 	SelfSigner       selfsigned.SelfSigner
 	CAPair           *crypto.CA
 	SecretLister     corev1listers.SecretLister
@@ -88,6 +94,7 @@ type ManagedProxyConfigurationReconciler struct {
 
 	newCertRotatorFunc func(namespace, name string, sans ...string) selfsigned.CertRotation
 	supportsV1CSR      bool
+	mcMode             bool
 }
 
 func (c *ManagedProxyConfigurationReconciler) SetupWithManager(mgr ctrl.Manager) error {
@@ -171,12 +178,40 @@ func (c *ManagedProxyConfigurationReconciler) deployProxyServer(config *proxyv1a
 		newProxyServerRole(config),
 		newProxyServerRoleBinding(config),
 	}
+	if c.mcMode {
+		var manager appsv1.Deployment
+		key := client.ObjectKey{
+			Namespace: config.Spec.ProxyServer.Namespace,
+			Name:      "cluster-proxy-addon-manager",
+		}
+		if err := c.HostClient.Get(context.Background(), key, &manager); err == nil {
+			ownerRef := metav1.NewControllerRef(&manager, schema.GroupVersionKind{
+				Group:   "apps",
+				Version: "v1",
+				Kind:    "Deployment",
+			})
+			for i, resource := range resources {
+				resource.SetOwnerReferences([]metav1.OwnerReference{
+					*ownerRef,
+				})
+				resources[i] = resource
+			}
+		}
+	} else {
+		for i, resource := range resources {
+			resource.SetOwnerReferences([]metav1.OwnerReference{
+				newOwnerReference(config),
+			})
+			resources[i] = resource
+		}
+	}
+
 	anyCreated := false
 	createdKinds := sets.NewString()
 	anyUpdated := false
 	updatedKinds := sets.NewString()
 	for _, resource := range resources {
-		gvks, _, err := c.Scheme().ObjectKinds(resource)
+		gvks, _, err := c.HostClient.Scheme().ObjectKinds(resource)
 		if err != nil {
 			return false, err
 		}
@@ -224,7 +259,7 @@ func (c *ManagedProxyConfigurationReconciler) ensure(incomingGeneration int64, g
 	// create if it doesn't exist
 	current := &unstructured.Unstructured{}
 	current.SetGroupVersionKind(gvk)
-	if err := c.Client.Get(
+	if err := c.HostClient.Get(
 		context.TODO(),
 		types.NamespacedName{
 			Namespace: resource.GetNamespace(),
@@ -239,7 +274,7 @@ func (c *ManagedProxyConfigurationReconciler) ensure(incomingGeneration int64, g
 			)
 		}
 		// if not found, then create
-		if err := c.Client.Create(context.TODO(), resource); err != nil {
+		if err := c.HostClient.Create(context.TODO(), resource); err != nil {
 			if !apierrors.IsAlreadyExists(err) {
 				return false, false, errors.Wrapf(err,
 					"failed to create resource kind: %s, namespace: %s, name %s",
@@ -270,7 +305,7 @@ func (c *ManagedProxyConfigurationReconciler) ensure(incomingGeneration int64, g
 	// update if generation bumped
 	if !created && int(incomingGeneration) > currentGeneration {
 		resource.SetResourceVersion(current.GetResourceVersion())
-		if err := c.Client.Update(context.TODO(), resource); err != nil {
+		if err := c.HostClient.Update(context.TODO(), resource); err != nil {
 			if apierrors.IsConflict(err) {
 				return c.ensure(incomingGeneration, gvk, resource)
 			}
@@ -353,7 +388,7 @@ func (c *ManagedProxyConfigurationReconciler) ensureEntrypoint(config *proxyv1al
 				},
 			},
 		}
-		if err := c.Client.Create(context.TODO(), proxyService); err != nil {
+		if err := c.HostClient.Create(context.TODO(), proxyService); err != nil {
 			if !apierrors.IsAlreadyExists(err) {
 				return "", errors.Wrapf(err, "failed to ensure entrypoint service for proxy-server")
 			}
@@ -451,11 +486,11 @@ func (c *ManagedProxyConfigurationReconciler) ensureBasicResources(config *proxy
 }
 
 func (c *ManagedProxyConfigurationReconciler) ensureNamespace(config *proxyv1alpha1.ManagedProxyConfiguration) error {
-	if err := c.Client.Get(context.TODO(), types.NamespacedName{
+	if err := c.HostClient.Get(context.TODO(), types.NamespacedName{
 		Name: config.Spec.ProxyServer.Namespace,
 	}, &corev1.Namespace{}); err != nil {
 		if apierrors.IsNotFound(err) {
-			if err := c.Client.Create(context.TODO(), &corev1.Namespace{
+			if err := c.HostClient.Create(context.TODO(), &corev1.Namespace{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: config.Spec.ProxyServer.Namespace,
 				},