flag to use host api server host & port

Author: Sharathmk99

cmd/virtual-kubelet/root/flag.go

@@ -72,6 +72,10 @@ func InstallFlags(flags *pflag.FlagSet, o *Opts) {
 	flags.BoolVar(&o.EnableStorage, "enable-storage", false, "Enable the Liqo storage reflection")
 	flags.StringVar(&o.VirtualStorageClassName, "virtual-storage-class-name", "liqo", "Name of the virtual storage class")
 	flags.StringVar(&o.RemoteRealStorageClassName, "remote-real-storage-class-name", "", "Name of the real storage class to use for the actual volumes")
+	flags.StringVar(&o.HomeAPIServerHost, "home-api-server-host", "",
+		"Home cluster API server HOST, this parameter is optional and required only to override the default values")
+	flags.StringVar(&o.HomeAPIServerPort, "home-api-server-port", "",
+		"Home cluster API server PORT, this parameter is optional and required only to override the default values")
 
 	flagset := flag.NewFlagSet("klog", flag.PanicOnError)
 	klog.InitFlags(flagset)

cmd/virtual-kubelet/root/opts.go

@@ -96,6 +96,9 @@ type Opts struct {
 	EnableStorage              bool
 	VirtualStorageClassName    string
 	RemoteRealStorageClassName string
+
+	HomeAPIServerHost string
+	HomeAPIServerPort string
 }
 
 // NewOpts returns an Opts struct with the default values set.

cmd/virtual-kubelet/root/root.go

@@ -110,6 +110,9 @@ func runRootCommand(ctx context.Context, c *Opts) error {
 		EnableStorage:              c.EnableStorage,
 		VirtualStorageClassName:    c.VirtualStorageClassName,
 		RemoteRealStorageClassName: c.RemoteRealStorageClassName,
+
+		HomeAPIServerHost: c.HomeAPIServerHost,
+		HomeAPIServerPort: c.HomeAPIServerPort,
 	}
 
 	eb := record.NewBroadcaster()

pkg/virtualKubelet/forge/pods.go

@@ -228,7 +228,7 @@ func RemotePodSpec(creation bool, local, remote *corev1.PodSpec, mutators ...Rem
 
 // APIServerSupportMutator is a mutator which implements the support to enable offloaded pods to interact back with the local Kubernetes API server.
 func APIServerSupportMutator(apiServerSupport APIServerSupportType, saName string, saSecretRetriever SASecretRetriever,
-	kubernetesServiceIPRetriever KubernetesServiceIPGetter) RemotePodSpecMutator {
+	kubernetesServiceIPRetriever KubernetesServiceIPGetter, homeAPIServerHost string, homeAPIServerPort string) RemotePodSpecMutator {
 	return func(remote *corev1.PodSpec) {
 		// The mutation of the service account related volume needs to be performed regardless of whether this feature is enabled.
 		remote.Volumes = RemoteVolumes(remote.Volumes, apiServerSupport, func() string { return saSecretRetriever(saName) })
@@ -239,11 +239,13 @@ func APIServerSupportMutator(apiServerSupport APIServerSupportType, saName strin
 		}
 
 		// Mutate the environment variables of the containers concerning the target API server hostname, and the service account name.
-		remote.Containers = RemoteContainersAPIServerSupport(remote.Containers, saName)
-		remote.InitContainers = RemoteContainersAPIServerSupport(remote.InitContainers, saName)
+		remote.Containers = RemoteContainersAPIServerSupport(remote.Containers, saName, homeAPIServerHost, homeAPIServerPort)
+		remote.InitContainers = RemoteContainersAPIServerSupport(remote.InitContainers, saName, homeAPIServerHost, homeAPIServerPort)
 
 		// Add a custom host alias to reach "kubernetes.default" through the remapped IP address.
-		remote.HostAliases = RemoteHostAliasesAPIServerSupport(remote.HostAliases, kubernetesServiceIPRetriever)
+		if homeAPIServerHost == "" && homeAPIServerPort == "" {
+			remote.HostAliases = RemoteHostAliasesAPIServerSupport(remote.HostAliases, kubernetesServiceIPRetriever)
+		}
 	}
 }
 
@@ -303,9 +305,9 @@ func FilterAntiAffinityLabels(labels map[string]string, whitelist string) map[st
 
 // RemoteContainersAPIServerSupport forges the containers for a reflected pod, appropriately adding the environment variables
 // to enable the offloaded containers to contact back the local API server, instead of the remote one.
-func RemoteContainersAPIServerSupport(containers []corev1.Container, saName string) []corev1.Container {
+func RemoteContainersAPIServerSupport(containers []corev1.Container, saName, homeAPIServerHost, homeAPIServerPort string) []corev1.Container {
 	for i := range containers {
-		containers[i].Env = RemoteContainerEnvVariablesAPIServerSupport(containers[i].Env, saName)
+		containers[i].Env = RemoteContainerEnvVariablesAPIServerSupport(containers[i].Env, saName, homeAPIServerHost, homeAPIServerPort)
 	}
 
 	return containers
@@ -314,7 +316,7 @@ func RemoteContainersAPIServerSupport(containers []corev1.Container, saName stri
 // RemoteContainerEnvVariablesAPIServerSupport forges the environment variables to enable offloaded containers to
 // contact back the local API server, instead of the remote one. In addition, it also hardcodes the
 // service account name in case it was retrieved from the pod spec, as it is not reflected remotely.
-func RemoteContainerEnvVariablesAPIServerSupport(envs []corev1.EnvVar, saName string) []corev1.EnvVar {
+func RemoteContainerEnvVariablesAPIServerSupport(envs []corev1.EnvVar, saName, homeAPIServerHost, homeAPIServerPort string) []corev1.EnvVar {
 	for i := range envs {
 		if envs[i].ValueFrom != nil && envs[i].ValueFrom.FieldRef != nil &&
 			envs[i].ValueFrom.FieldRef.FieldPath == "spec.serviceAccountName" {
@@ -323,19 +325,24 @@ func RemoteContainerEnvVariablesAPIServerSupport(envs []corev1.EnvVar, saName st
 			envs[i].ValueFrom = nil
 		}
 	}
-
-	hostport := "tcp://" + net.JoinHostPort(kubernetesAPIService, KubernetesServicePort)
+	actualKubernetesAPIService := kubernetesAPIService
+	actualKubernetesAPIServicePort := KubernetesServicePort
+	if homeAPIServerHost != "" && homeAPIServerPort != "" {
+		actualKubernetesAPIService = homeAPIServerHost
+		actualKubernetesAPIServicePort = homeAPIServerPort
+	}
+	hostport := "tcp://" + net.JoinHostPort(actualKubernetesAPIService, actualKubernetesAPIServicePort)
 	return append(envs,
 		// We replace the correct IP address with the kubernetes.default hostname (which is associated with the remapped
 		// IP through an appropriate host alias), since directly using the remapped IP address would lead to TLS errors
 		// as it is not included in the certificate.
-		corev1.EnvVar{Name: "KUBERNETES_SERVICE_HOST", Value: kubernetesAPIService},
-		corev1.EnvVar{Name: "KUBERNETES_SERVICE_PORT", Value: KubernetesServicePort},
+		corev1.EnvVar{Name: "KUBERNETES_SERVICE_HOST", Value: actualKubernetesAPIService},
+		corev1.EnvVar{Name: "KUBERNETES_SERVICE_PORT", Value: actualKubernetesAPIServicePort},
 		corev1.EnvVar{Name: "KUBERNETES_PORT", Value: hostport},
-		corev1.EnvVar{Name: fmt.Sprintf("KUBERNETES_PORT_%s_TCP", KubernetesServicePort), Value: hostport},
-		corev1.EnvVar{Name: fmt.Sprintf("KUBERNETES_PORT_%s_TCP_PROTO", KubernetesServicePort), Value: "tcp"},
-		corev1.EnvVar{Name: fmt.Sprintf("KUBERNETES_PORT_%s_TCP_ADDR", KubernetesServicePort), Value: kubernetesAPIService},
-		corev1.EnvVar{Name: fmt.Sprintf("KUBERNETES_PORT_%s_TCP_PORT", KubernetesServicePort), Value: KubernetesServicePort},
+		corev1.EnvVar{Name: fmt.Sprintf("KUBERNETES_PORT_%s_TCP", actualKubernetesAPIServicePort), Value: hostport},
+		corev1.EnvVar{Name: fmt.Sprintf("KUBERNETES_PORT_%s_TCP_PROTO", actualKubernetesAPIServicePort), Value: "tcp"},
+		corev1.EnvVar{Name: fmt.Sprintf("KUBERNETES_PORT_%s_TCP_ADDR", actualKubernetesAPIServicePort), Value: actualKubernetesAPIService},
+		corev1.EnvVar{Name: fmt.Sprintf("KUBERNETES_PORT_%s_TCP_PORT", actualKubernetesAPIServicePort), Value: actualKubernetesAPIServicePort},
 	)
 }
 

pkg/virtualKubelet/forge/pods_test.go

@@ -15,6 +15,7 @@
 package forge_test
 
 import (
+	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -200,8 +201,9 @@ var _ = Describe("Pod forging", func() {
 		const saName = "service-account"
 
 		var (
-			apiServerSupport forge.APIServerSupportType
-			remote, original *corev1.PodSpec
+			apiServerSupport                     forge.APIServerSupportType
+			remote, original                     *corev1.PodSpec
+			homeAPIServerHost, homeAPIServerPort string
 		)
 
 		BeforeEach(func() {
@@ -217,7 +219,7 @@ var _ = Describe("Pod forging", func() {
 
 		JustBeforeEach(func() {
 			original = remote.DeepCopy()
-			forge.APIServerSupportMutator(apiServerSupport, saName, SASecretRetriever, KubernetesServiceIPGetter)(remote)
+			forge.APIServerSupportMutator(apiServerSupport, saName, SASecretRetriever, KubernetesServiceIPGetter, homeAPIServerHost, homeAPIServerPort)(remote)
 		})
 
 		When("API server support is enabled", func() {
@@ -228,11 +230,12 @@ var _ = Describe("Pod forging", func() {
 				})
 
 				It("should appropriately mutate the remote containers", func() {
-					Expect(remote.Containers).To(Equal(forge.RemoteContainersAPIServerSupport(original.Containers, saName)))
+					Expect(remote.Containers).To(Equal(forge.RemoteContainersAPIServerSupport(original.Containers, saName, homeAPIServerHost, homeAPIServerPort)))
 				})
 
 				It("should appropriately mutate the remote init containers", func() {
-					Expect(remote.InitContainers).To(Equal(forge.RemoteContainersAPIServerSupport(original.InitContainers, saName)))
+					Expect(remote.InitContainers).To(Equal(forge.RemoteContainersAPIServerSupport(original.InitContainers, saName,
+						homeAPIServerHost, homeAPIServerPort)))
 				})
 
 				It("should appropriately mutate host aliases", func() {
@@ -241,12 +244,57 @@ var _ = Describe("Pod forging", func() {
 			}
 
 			When("legacy mode is set", func() {
-				BeforeEach(func() { apiServerSupport = forge.APIServerSupportLegacy })
+				BeforeEach(func() {
+					apiServerSupport = forge.APIServerSupportLegacy
+					homeAPIServerHost = ""
+					homeAPIServerPort = ""
+				})
 				WhenBody()
 			})
 
 			When("token API mode is set", func() {
-				BeforeEach(func() { apiServerSupport = forge.APIServerSupportTokenAPI })
+				BeforeEach(func() {
+					apiServerSupport = forge.APIServerSupportTokenAPI
+					homeAPIServerHost = ""
+					homeAPIServerPort = ""
+				})
+				WhenBody()
+			})
+		})
+
+		When("API server support is enabled with custom host API server Host & Port", func() {
+
+			WhenBody := func() {
+				It("should correctly mutate the volumes", func() {
+					Expect(remote.Volumes).To(Equal(forge.RemoteVolumes(original.Volumes, apiServerSupport,
+						func() string { return SASecretRetriever(saName) })))
+				})
+
+				It("should appropriately mutate the remote containers", func() {
+					Expect(remote.Containers).To(Equal(forge.RemoteContainersAPIServerSupport(original.Containers, saName, homeAPIServerHost, homeAPIServerPort)))
+				})
+
+				It("should appropriately mutate the remote init containers", func() {
+					Expect(remote.InitContainers).To(Equal(forge.RemoteContainersAPIServerSupport(original.InitContainers, saName,
+						homeAPIServerHost, homeAPIServerPort)))
+				})
+			}
+
+			When("legacy mode is set with custom host API server Host & Port", func() {
+				BeforeEach(func() {
+					apiServerSupport = forge.APIServerSupportLegacy
+					homeAPIServerHost = "custom.apiserver.com"
+					homeAPIServerPort = "6443"
+				})
+				WhenBody()
+			})
+
+			When("token API mode is set with custom host API server Host & Port", func() {
+				BeforeEach(func() {
+					apiServerSupport = forge.APIServerSupportTokenAPI
+					homeAPIServerHost = "custom1.apiserver.com"
+					homeAPIServerPort = "6443"
+				})
 				WhenBody()
 			})
 		})
@@ -392,6 +440,7 @@ var _ = Describe("Pod forging", func() {
 	Describe("the RemoteContainersAPIServerSupport function", func() {
 		var container corev1.Container
 		var output []corev1.Container
+		var homeAPIServerHost, homeAPIServerPort = "", ""
 
 		BeforeEach(func() {
 			container = corev1.Container{
@@ -410,7 +459,7 @@ var _ = Describe("Pod forging", func() {
 		})
 
 		JustBeforeEach(func() {
-			output = forge.RemoteContainersAPIServerSupport([]corev1.Container{container}, "service-account-name")
+			output = forge.RemoteContainersAPIServerSupport([]corev1.Container{container}, "service-account-name", homeAPIServerHost, homeAPIServerPort)
 		})
 
 		It("should propagate all container values", func() {
@@ -443,6 +492,62 @@ var _ = Describe("Pod forging", func() {
 		})
 	})
 
+	Describe("the RemoteContainersAPIServerSupport function with custom host API server Host & Port", func() {
+		var container corev1.Container
+		var output []corev1.Container
+		var homeAPIServerHost, homeAPIServerPort = "custom.apiserver.com", "6443"
+
+		BeforeEach(func() {
+			container = corev1.Container{
+				Name:  "foo",
+				Image: "foo/bar:v0.1-alpha3",
+				Args:  []string{"--first", "--second"},
+				Ports: []corev1.ContainerPort{{Name: "foo-port", ContainerPort: 8080}},
+				Env: []corev1.EnvVar{
+					{Name: "ENV_1", Value: "VALUE_1"},
+					{Name: "ENV_2", Value: "VALUE_2"},
+					{Name: "ENV_3", ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{Key: "foo"}}},
+					{Name: "ENV_4", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}},
+					{Name: "ENV_5", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.serviceAccountName"}}},
+				},
+			}
+		})
+
+		JustBeforeEach(func() {
+			output = forge.RemoteContainersAPIServerSupport([]corev1.Container{container}, "service-account-name", homeAPIServerHost, homeAPIServerPort)
+		})
+
+		It("should propagate all container values", func() {
+			// Remove the environment variables from the containers, as checked in the test below.
+			envcleaner := func(c corev1.Container) corev1.Container {
+				c.Env = nil
+				return c
+			}
+
+			Expect(output).To(HaveLen(1))
+			Expect(envcleaner(output[0])).To(Equal(envcleaner(container)))
+		})
+
+		It("should configure the appropriate environment variables", func() {
+			Expect(output).To(HaveLen(1))
+			Expect(output[0].Env).To(ConsistOf(
+				corev1.EnvVar{Name: "ENV_1", Value: "VALUE_1"},
+				corev1.EnvVar{Name: "ENV_2", Value: "VALUE_2"},
+				corev1.EnvVar{Name: "ENV_3", ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{Key: "foo"}}},
+				corev1.EnvVar{Name: "ENV_4", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}}},
+				corev1.EnvVar{Name: "ENV_5", Value: "service-account-name"},
+				corev1.EnvVar{Name: "KUBERNETES_SERVICE_PORT", Value: homeAPIServerPort},
+				corev1.EnvVar{Name: "KUBERNETES_SERVICE_HOST", Value: homeAPIServerHost},
+				corev1.EnvVar{Name: "KUBERNETES_PORT", Value: fmt.Sprintf("tcp://%s:%s", homeAPIServerHost, homeAPIServerPort)},
+				corev1.EnvVar{Name: fmt.Sprintf("KUBERNETES_PORT_%s_TCP", homeAPIServerPort),
+					Value: fmt.Sprintf("tcp://%s:%s", homeAPIServerHost, homeAPIServerPort)},
+				corev1.EnvVar{Name: fmt.Sprintf("KUBERNETES_PORT_%s_TCP_PROTO", homeAPIServerPort), Value: "tcp"},
+				corev1.EnvVar{Name: fmt.Sprintf("KUBERNETES_PORT_%s_TCP_PORT", homeAPIServerPort), Value: homeAPIServerPort},
+				corev1.EnvVar{Name: fmt.Sprintf("KUBERNETES_PORT_%s_TCP_ADDR", homeAPIServerPort), Value: homeAPIServerHost},
+			))
+		})
+	})
+
 	Describe("the RemoteTolerations function", func() {
 		var (
 			included, excluded corev1.Toleration

pkg/virtualKubelet/provider/provider.go

@@ -75,6 +75,9 @@ type InitConfig struct {
 	EnableStorage              bool
 	VirtualStorageClassName    string
 	RemoteRealStorageClassName string
+
+	HomeAPIServerHost string
+	HomeAPIServerPort string
 }
 
 // LiqoProvider implements the virtual-kubelet provider interface and stores pods in memory.
@@ -122,6 +125,8 @@ func NewLiqoProvider(ctx context.Context, cfg *InitConfig, eb record.EventBroadc
 	podReflectorConfig := workload.PodReflectorConfig{
 		APIServerSupport:    apiServerSupport,
 		DisableIPReflection: cfg.DisableIPReflection,
+		HomeAPIServerHost:   cfg.HomeAPIServerHost,
+		HomeAPIServerPort:   cfg.HomeAPIServerPort,
 	}
 
 	reflectionManager := manager.New(localClient, remoteClient, localLiqoClient, remoteLiqoClient, cfg.InformerResyncPeriod, eb)

pkg/virtualKubelet/reflection/workload/pod.go

@@ -93,6 +93,8 @@ type PodReflector struct {
 type PodReflectorConfig struct {
 	APIServerSupport    forge.APIServerSupportType
 	DisableIPReflection bool
+	HomeAPIServerHost   string
+	HomeAPIServerPort   string
 }
 
 // FallbackPodReflector handles the "orphan" pods outside the managed namespaces.

pkg/virtualKubelet/reflection/workload/pod_test.go

@@ -42,7 +42,7 @@ import (
 var _ = Describe("Pod Reflection Tests", func() {
 	Describe("the NewPodReflector function", func() {
 		It("should not return a nil reflector", func() {
-			reflector := workload.NewPodReflector(nil, nil, nil, &workload.PodReflectorConfig{forge.APIServerSupportDisabled, false}, 0)
+			reflector := workload.NewPodReflector(nil, nil, nil, &workload.PodReflectorConfig{forge.APIServerSupportDisabled, false, "", ""}, 0)
 			Expect(reflector).ToNot(BeNil())
 			Expect(reflector.Reflector).ToNot(BeNil())
 		})
@@ -59,7 +59,7 @@ var _ = Describe("Pod Reflection Tests", func() {
 		BeforeEach(func() {
 			ipam := fakeipam.NewIPAMClient("192.168.200.0/24", "192.168.201.0/24", true)
 			metricsFactory := func(string) metricsv1beta1.PodMetricsInterface { return nil }
-			reflector := workload.NewPodReflector(nil, metricsFactory, ipam, &workload.PodReflectorConfig{forge.APIServerSupportDisabled, false}, 0)
+			reflector := workload.NewPodReflector(nil, metricsFactory, ipam, &workload.PodReflectorConfig{forge.APIServerSupportDisabled, false, "", ""}, 0)
 			kubernetesServiceIPGetter = reflector.KubernetesServiceIPGetter()
 		})
 
@@ -106,7 +106,7 @@ var _ = Describe("Pod Reflection Tests", func() {
 			client = fake.NewSimpleClientset(&local)
 			factory := informers.NewSharedInformerFactory(client, 10*time.Hour)
 
-			reflector = workload.NewPodReflector(nil, nil, nil, &workload.PodReflectorConfig{forge.APIServerSupportDisabled, false}, 0)
+			reflector = workload.NewPodReflector(nil, nil, nil, &workload.PodReflectorConfig{forge.APIServerSupportDisabled, false, "", ""}, 0)
 
 			opts := options.New(client, factory.Core().V1().Pods()).
 				WithHandlerFactory(FakeEventHandler).

pkg/virtualKubelet/reflection/workload/podns.go

@@ -319,7 +319,8 @@ func (npr *NamespacedPodReflector) ForgeShadowPod(ctx context.Context, local *co
 
 	// Forge the target shadowpod object.
 	target := forge.RemoteShadowPod(local, shadow, npr.RemoteNamespace(),
-		forge.APIServerSupportMutator(npr.config.APIServerSupport, pod.ServiceAccountName(local), saSecretRetriever, ipGetter))
+		forge.APIServerSupportMutator(npr.config.APIServerSupport, pod.ServiceAccountName(local), saSecretRetriever,
+			ipGetter, npr.config.HomeAPIServerHost, npr.config.HomeAPIServerPort))
 
 	// Check whether an error occurred during secret name retrieval.
 	if saerr != nil {

pkg/virtualKubelet/reflection/workload/podns_test.go

@@ -72,7 +72,7 @@ var _ = Describe("Namespaced Pod Reflection Tests", func() {
 
 			broadcaster := record.NewBroadcaster()
 			metricsFactory := func(string) metricsv1beta1.PodMetricsInterface { return nil }
-			rfl := workload.NewPodReflector(nil, metricsFactory, ipam, &workload.PodReflectorConfig{forge.APIServerSupportTokenAPI, false}, 0)
+			rfl := workload.NewPodReflector(nil, metricsFactory, ipam, &workload.PodReflectorConfig{forge.APIServerSupportTokenAPI, false, "", ""}, 0)
 			rfl.Start(ctx, options.New(client, factory.Core().V1().Pods()).WithEventBroadcaster(broadcaster))
 			reflector = rfl.NewNamespaced(options.NewNamespaced().
 				WithLocal(LocalNamespace, client, factory).WithLiqoLocal(liqoClient, liqoFactory).