✨ POC for mounts (Master PR, not for merging)

cmd/kcp/kcp.go

@@ -81,7 +81,7 @@ func main() {
 				additionalMappingsFile = os.Args[i+1]
 			} // else let normal flag processing fail
 		} else if strings.HasPrefix(f, "--miniproxy-mapping-file") {
-			additionalMappingsFile = strings.TrimPrefix(f, "--mapping-file=")
+			additionalMappingsFile = strings.TrimPrefix(f, "--miniproxy-mapping-file=")
 		}
 	}
 

contrib/mounts-virtualworkspace/Makefile

@@ -0,0 +1,86 @@
+
+GO_INSTALL = ./hack/go-install.sh
+
+TOOLS_DIR=hack/tools
+TOOLS_GOBIN_DIR := $(abspath $(TOOLS_DIR))
+GOBIN_DIR=$(abspath ./bin)
+PATH := $(GOBIN_DIR):$(TOOLS_GOBIN_DIR):$(PATH)
+TMPDIR := $(shell mktemp -d)
+KIND_CLUSTER_NAME ?= kind
+
+CONTROLLER_GEN_VER := v0.16.1
+CONTROLLER_GEN_BIN := controller-gen
+CONTROLLER_GEN := $(TOOLS_DIR)/$(CONTROLLER_GEN_BIN)-$(CONTROLLER_GEN_VER)
+export CONTROLLER_GEN # so hack scripts can use it
+
+
+CODE_GENERATOR_VER := v2.1.0
+CODE_GENERATOR_BIN := code-generator
+CODE_GENERATOR := $(TOOLS_GOBIN_DIR)/$(CODE_GENERATOR_BIN)-$(CODE_GENERATOR_VER)
+export CODE_GENERATOR # so hack scripts can use it
+
+KCP_APIGEN_VER := v0.26.0
+KCP_APIGEN_BIN := apigen
+KCP_APIGEN_GEN := $(TOOLS_DIR)/$(KCP_APIGEN_BIN)-$(KCP_APIGEN_VER)
+export KCP_APIGEN_GEN # so hack scripts can use it
+
+OPENSHIFT_GOIMPORTS_VER := c72f1dc2e3aacfa00aece3391d938c9bc734e791
+OPENSHIFT_GOIMPORTS_BIN := openshift-goimports
+OPENSHIFT_GOIMPORTS := $(TOOLS_DIR)/$(OPENSHIFT_GOIMPORTS_BIN)-$(OPENSHIFT_GOIMPORTS_VER)
+export OPENSHIFT_GOIMPORTS # so hack scripts can use it
+
+VERSION ?= $(shell git describe --tags --always --dirty)
+
+LDFLAGS= -s -w \
+	-extldflags '-static'
+
+all: build
+.PHONY: all
+
+ldflags:
+	@echo $(LDFLAGS)
+
+tools: $(CONTROLLER_GEN) $(KCP_APIGEN_GEN) $ $(CODE_GENERATOR) $(OPENSHIFT_GOIMPORTS) ## Install tools
+.PHONY: tools
+
+$(CONTROLLER_GEN):
+	GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) sigs.k8s.io/controller-tools/cmd/controller-gen $(CONTROLLER_GEN_BIN) $(CONTROLLER_GEN_VER)
+
+$(CODE_GENERATOR):
+	GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) github.com/kcp-dev/code-generator/v2 $(CODE_GENERATOR_BIN) $(CODE_GENERATOR_VER)
+
+$(KCP_APIGEN_GEN):
+	GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) github.com/kcp-dev/kcp/sdk/cmd/apigen $(KCP_APIGEN_BIN) $(KCP_APIGEN_VER)
+
+$(OPENSHIFT_GOIMPORTS):
+	GOBIN=$(TOOLS_GOBIN_DIR) $(GO_INSTALL) github.com/openshift-eng/openshift-goimports $(OPENSHIFT_GOIMPORTS_BIN) $(OPENSHIFT_GOIMPORTS_VER)
+
+.PHONY: require-%
+require-%:
+	@if ! command -v $* 1> /dev/null 2>&1; then echo "$* not found in ${PATH}"; exit 1; fi
+
+.PHONY: verify-go-versions
+verify-go-versions: ## Verify go versions
+	hack/verify-go-versions.sh
+
+build: WHAT ?= ./cmd/...
+build: require-jq require-go require-git verify-go-versions ## Build the project
+	GOOS=$(OS) GOARCH=$(ARCH) CGO_ENABLED=0 go build $(BUILDFLAGS) -ldflags="$(LDFLAGS)" -o bin $(WHAT)
+.PHONY: build
+
+crds: $(CONTROLLER_GEN) $(YAML_PATCH) ## Generate crds
+	./hack/update-codegen-crds.sh
+.PHONY: crds
+
+codegen: crds $(CODE_GENERATOR) ## Generate all
+	go mod download
+	./hack/update-codegen-clients.sh
+	$(MAKE) imports
+
+.PHONY: imports
+imports: $(OPENSHIFT_GOIMPORTS) verify-go-versions
+	$(OPENSHIFT_GOIMPORTS) -m github.com/kcp-dev/kcp/contrib/mounts-virtualworkspace
+
+test:
+	go test ./...
+

contrib/mounts-virtualworkspace/README.md

@@ -0,0 +1,149 @@
+# Remote mounts virtual workspace example
+
+This is an example of how to use the mounts feature in kcp to mount a target cluster into a workspace.
+Mount happens at front-proxy level via reverse proxy. You need to implement your own VirtualWorkspace to handle this.
+and this example shows how to do it.
+
+## This is example codebase, and should be used only as example.
+
+# Step by step guide
+
+Step by step guide how to setup this example.
+
+1. Start kcp with mounts feature gate enabled:
+
+```
+go run ./cmd/kcp start --miniproxy-mapping-file=./contrib/mounts-virtualworkspace/assets/path-mapping.yaml --feature-gates=WorkspaceMounts=true
+```
+
+2. Setup all required workspaces and exports for virtual workspace to run:
+
+Provider workspace where all the target cluster will be defined with secrets.
+These clusters can be mounted later on by the any workspace.
+
+Setup providers:
+
+```
+cd ./contrib/mounts-virtualworkspace/
+
+# Set kcp KUBECONFIG
+export KUBECONFIG=../../.kcp/admin.kubeconfig
+
+kubectl ws use :root
+# create provider workspaces
+kubectl ws create providers --enter
+kubectl ws create mounts --enter
+
+# create exports
+kubectl create -f config/mounts/resources/apiresourceschema-targetkubeclusters.targets.contrib.kcp.io.yaml
+kubectl create -f config/mounts/resources/apiresourceschema-kubeclusters.mounts.contrib.kcp.io.yaml
+kubectl create -f config/mounts/resources/apiresourceschema-targetvclusters.targets.contrib.kcp.io.yaml
+kubectl create -f config/mounts/resources/apiresourceschema-vclusters.mounts.contrib.kcp.io.yaml
+kubectl create -f config/mounts/resources/apiexport-mounts.contrib.kcp.io.yaml
+kubectl create -f config/mounts/resources/apiexport-targets.contrib.kcp.io.yaml
+
+```
+
+3. Start virtual workspace process:
+```
+ go run ./cmd/virtual-workspaces/ start \
+ --kubeconfig=../../.kcp/admin.kubeconfig  \
+ --tls-cert-file=../../.kcp/apiserver.crt \
+ --tls-private-key-file=../../.kcp/apiserver.key \
+ --authentication-kubeconfig=../../.kcp/admin.kubeconfig \
+ --virtual-workspaces-proxy-hostname=https://localhost:6444 \
+ -v=8
+```
+
+4. Continue bootstrapping the mounts example:
+
+```
+# create operators namespace where platforms operators will create objects. This could be many of them.
+# for this example we will use only one.
+
+kubectl ws use :root
+kubectl ws create operators --enter
+kubectl ws create mounts --enter
+
+# bind the exports
+# see https://github.com/kcp-dev/kcp/issues/3189
+kubectl create -f config/mounts/resources/apibinding-targets.yaml
+
+# Create a target cluster to `kind` cluster locally:
+
+# create kind cluster if not already created
+kind create cluster --name kind --kubeconfig kind.kubeconfig
+
+#create secret with kubeconfig:
+kubectl ws use root:operators:mounts
+kubectl create secret generic kind-kubeconfig --from-file=kubeconfig=kind.kubeconfig
+
+# create target cluster:
+kubectl create -f config/mounts/resources/example-target-cluster.yaml
+
+# create vcluster target:
+kubectl create -f config/mounts/resources/example-target-vcluster.yaml
+
+# get secret string:
+kubectl get TargetKubeCluster proxy-cluster -o jsonpath='{.status.secretString}'
+HS7kd992rQNedzcY
+
+ cluster: YJfI5DIEoQ0ySamJ
+
+# Create a consumer workspace for mounts:
+kubectl ws use :root
+kubectl ws create consumer --enter
+kubectl ws create kind-cluster
+
+kubectl create -f config/mounts/resources/apibinding-mounts.yaml
+
+# !!!!! replace secrets string first in the file bellow :
+kubectl create -f config/mounts/resources/example-mount-cluster.yaml
+
+# annotate the workspace with mount,  putting the intent that this should be mounted:
+ kubectl annotate workspace kind-cluster  experimental.tenancy.kcp.io/mount='{"spec":{"ref":{"kind":"KubeCluster","name":"proxy-cluster","apiVersion":"mounts.contrib.kcp.io/v1alpha1"}}}'
+```
+
+5. Check the mounts reconciler logs:
+
+Now workspace should be backed by mountpoint from front-proxy:
+
+```
+kubectl ws use kind-cluster
+k get pods -A
+NAMESPACE            NAME                                         READY   STATUS    RESTARTS   AGE
+kube-system          coredns-7db6d8ff4d-4l625                     1/1     Running   0          22h
+kube-system          coredns-7db6d8ff4d-ntf95                     1/1     Running   0          22h
+kube-system          etcd-kind-control-plane                      1/1     Running   0          22h
+kube-system          kindnet-vv872                                1/1     Running   0          22h
+kube-system          kube-apiserver-kind-control-plane            1/1     Running   0          22h
+kube-system          kube-controller-manager-kind-control-plane   1/1     Running   0          22h
+kube-system          kube-proxy-lkv29                             1/1     Running   0          22h
+kube-system          kube-scheduler-kind-control-plane            1/1     Running   0          22h
+local-path-storage   local-path-provisioner-988d74bc-dqnd7        1/1     Running   0          22h
+```
+
+# Vclusters example
+
+vCluster are backed by vCluster mounts. This is a way to create a virtual cluster that is backed by a real cluster.
+You can either provide a kubeconfig or a target cluster to back the vCluster or secretString for "target" in the system.
+
+kubectl ws create vcluster
+kubectl create -f config/mounts/resources/example-mount-vcluster.yaml
+
+# annotate the workspace with mount,  putting the intent that this should be mounted:
+ kubectl annotate workspace vcluster experimental.tenancy.kcp.io/mount='{"spec":{"ref":{"kind":"VCluster","name":"virtual-cluster","apiVersion":"mounts.contrib.kcp.io/v1alpha1"}}}'
+
+ kubectl annotate workspace vcluster-3 experimental.tenancy.kcp.io/mount='{"spec":{"ref":{"kind":"VCluster","name":"virtual-cluster-3","apiVersion":"mounts.contrib.kcp.io/v1alpha1"}}}'
+
+
+# Known issues
+
+1. `TargetKubeCluster` changes do not propagate to `KubeCluster` need to wire them up.
+Challenge is that when these 2 objects are in separate bindings, its more machinery to make them work together.
+
+2. VirtualWorkspace is not yet fully shards aware. Ideally it should be 1 per each shard, and handle only its
+own workspaces.
+
+3. KubeCluster changes not applied to Workspaces. This might be a bug in core. Need to validate.
+

contrib/mounts-virtualworkspace/apis/mounts/OWNERS

@@ -0,0 +1,2 @@
+reviewers:
+- mjudeikis

contrib/mounts-virtualworkspace/apis/mounts/register.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2023 The KCP Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mounts
+
+const (
+	GroupName       = "mounts.contrib.kcp.io"
+	KubeClusterKind = "KubeCluster"
+	VClusterKind    = "VCluster"
+)

contrib/mounts-virtualworkspace/apis/mounts/v1alpha1/doc.go

@@ -0,0 +1,20 @@
+/*
+Copyright 2024 The KCP Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:deepcopy-gen=package,register
+// +groupName=mounts.contrib.kcp.io
+// +k8s:openapi-gen=true
+package v1alpha1

contrib/mounts-virtualworkspace/apis/mounts/v1alpha1/register.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2023 The KCP Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	"github.com/kcp-dev/kcp/contrib/mounts-virtualworkspace/apis/mounts"
+)
+
+// SchemeGroupVersion is group version used to register these objects.
+var SchemeGroupVersion = schema.GroupVersion{Group: mounts.GroupName, Version: "v1alpha1"}
+
+// Kind takes an unqualified kind and returns back a Group qualified GroupKind.
+func Kind(kind string) schema.GroupKind {
+	return SchemeGroupVersion.WithKind(kind).GroupKind()
+}
+
+// Resource takes an unqualified resource and returns a Group qualified GroupResource.
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+var (
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	AddToScheme   = SchemeBuilder.AddToScheme
+)
+
+// Adds the list of known types to Scheme.
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&KubeCluster{},
+		&KubeClusterList{},
+		&VCluster{},
+		&VClusterList{},
+	)
+	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
+	return nil
+}

contrib/mounts-virtualworkspace/apis/mounts/v1alpha1/types.go

@@ -0,0 +1,8 @@
+package v1alpha1
+
+const (
+
+	// InternalWorkspaceProxyKeyLabel is an internal label set on a WorkspaceProxy resource that contains the full hash of the WorkspaceProxyKey, generated with the ToProxyTargetKey(..)
+	// helper func, this label is used for reverse lookups of workspaceProxyKey to WorkspaceProxy.
+	InternalWorkspaceProxyKeyLabel = "internal.mounts.contrib.kcp.io/key"
+)