Add support for issuing IPSec tunnel certificates using cert-manager.

When cert-manager certificates are enabled, the controller no longer generates the IPSec CA cert or private key stored in the `ovn-ipsec-ca` secret. The secret should be populated with the same CA as configured with cert-manager. It still enables IPSec in OVN NB.

When cert-manager certificates are enabled the CNI daemon creates cert-manager CertificateRequest resources instead of CSRs. A cert-manager ClusterIssuer should be configured to approve and sign these CertificateRequests with a matching CA as configured in `ovn-ipsec-ca` secret. The name of the issuer to use is configurable in the CNI.

The CNI daemon now watches the `ovn-ipsec-ca` secret for changes allowing for rollout of a new trust bundle. It verifies the currently configured certificate is signed by the new bundle and if not then triggers a new certificate to be issued. The daemon now splits each certificate in the CA bundle into a separate file as strongswan is unable to parse multiple CAs from a single file.

The CNI daemon now requests a new certificate when the current certificate is at least half way to expiry based on the times in the certificate. When generating a new certificate the daemon also generates a new key just in case the previous one was leaked somehow. The certificate lifetime is also now configurable rather than lasting for a year. The CNI no longer restarts the ipsec or ovs-ipsec-monitor services when the certificate changes and just requests ipsec to reread the CA certs if they change.

To allow for the CNI daemon to keep track of the versions of its key, certificate, and CA cert files it now stores them with locally unique names on disk. Keys and certs are suffixed with the timestamp they were generated. CA files are suffixed with the k8s revision number of the `ovn-ipsec-ca` secret.

The cert manager validation webhook (if used) should be run in the host network to prevent the risk of certificate requests deadlocking in the event of a certificate expiry. The CNI pods and cert manager issuer interact with the API server over the host network to create and approve certificates but the API server calls the webhook of the service network which can be broken in the event of an expired certificate.

A new kind deployment is created using cert-manager issued certificates and a new e2e test is created that uses it. The e2e test runs through rotating the the CA.

Signed-off-by: Paul Cruickshank <pcruickshank@evroc.com>

Author: Paul Cruickshank

charts/kube-ovn-v2/templates/agent/agent-clusterrole.yaml

@@ -87,5 +87,9 @@ rules:
       - ""
     resources:
       - "secrets"
+    resourceNames:
+      - "ovn-ipsec-ca"
     verbs:
-      - "get"
+      - "get"
+      - "list"
+      - "watch"

charts/kube-ovn/templates/ovn-CR.yaml

@@ -273,7 +273,6 @@ rules:
     verbs:
       - get
       - list
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -352,12 +351,23 @@ rules:
       - "list"
       - "watch"
       - "delete"
-  - apiGroups:
-      - ""
-    resources:
-      - "secrets"
-    verbs:
-      - "get"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: secret-reader-ovn-ipsec
+  namespace: kube-system
+rules:
+- apiGroups: 
+    - ""
+  resources: 
+    - "secrets"
+  resourceNames: 
+    - "ovn-ipsec-ca"
+  verbs: 
+    - "get"
+    - "list"
+    - "watch"
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

charts/kube-ovn/templates/ovn-CRB.yaml

@@ -67,6 +67,20 @@ subjects:
     namespace: {{ .Values.namespace }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-ovn-cni-secret-reader
+  namespace: kube-system
+subjects:
+- kind: ServiceAccount
+  name: kube-ovn-cni
+  namespace: kube-system
+roleRef:
+  kind: Role
+  name: secret-reader-ovn-ipsec
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: kube-ovn-app

cmd/daemon/cniserver.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"errors"
+	"fmt"
 	"net"
 	"net/http"
 	"net/http/pprof"
@@ -95,7 +96,15 @@ func main() {
 		kubeovninformer.WithTweakListOptions(func(listOption *v1.ListOptions) {
 			listOption.AllowWatchBookmarks = true
 		}))
-	ctl, err := daemon.NewController(config, stopCh, podInformerFactory, nodeInformerFactory, kubeovnInformerFactory)
+
+	caSecretInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeClient, 0,
+		kubeinformers.WithTweakListOptions(func(listOption *v1.ListOptions) {
+			listOption.FieldSelector = fmt.Sprintf("metadata.name=%s", util.DefaultOVNIPSecCA)
+		}),
+		kubeinformers.WithNamespace("kube-system"),
+	)
+
+	ctl, err := daemon.NewController(config, stopCh, podInformerFactory, nodeInformerFactory, caSecretInformerFactory, kubeovnInformerFactory)
 	if err != nil {
 		util.LogFatalAndExit(err, "failed to create controller")
 	}

dist/images/install.sh

@@ -41,6 +41,9 @@ OVS_VSCTL_CONCURRENCY=${OVS_VSCTL_CONCURRENCY:-100}
 ENABLE_COMPACT=${ENABLE_COMPACT:-false}
 SECURE_SERVING=${SECURE_SERVING:-false}
 ENABLE_OVN_IPSEC=${ENABLE_OVN_IPSEC:-false}
+CERT_MANAGER_IPSEC_CERT=${CERT_MANAGER_IPSEC_CERT:-false}
+IPSEC_CERT_DURATION=${IPSEC_CERT_DURATION:-63072000} # 2 years in seconds
+CERT_MANAGER_ISSUER_NAME=${CERT_MANAGER_ISSUER_NAME:-kube-ovn}
 ENABLE_ANP=${ENABLE_ANP:-false}
 SET_VXLAN_TX_OFF=${SET_VXLAN_TX_OFF:-false}
 OVSDB_CON_TIMEOUT=${OVSDB_CON_TIMEOUT:-3}
@@ -3775,12 +3778,32 @@ rules:
       - "list"
       - "watch"
       - "delete"
-  - apiGroups:
-      - ""
-    resources:
-      - "secrets"
-    verbs:
-      - "get"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: secret-reader-ovn-ipsec
+  namespace: kube-system
+rules:
+- apiGroups: 
+    - ""
+  resources: 
+    - "secrets"
+  resourceNames:
+    - "ovn-ipsec-ca"
+  verbs: 
+    - "get"
+    - "list"
+    - "watch"
+- apiGroups: 
+    - "cert-manager.io"
+  resources: 
+    - "certificaterequests"
+  verbs: 
+    - "get"
+    - "list"
+    - "create"
+    - "delete"
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -3808,6 +3831,20 @@ subjects:
   - kind: ServiceAccount
     name: kube-ovn-cni
     namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-ovn-cni-secret-reader
+  namespace: kube-system
+subjects:
+- kind: ServiceAccount
+  name: kube-ovn-cni
+  namespace: kube-system
+roleRef:
+  kind: Role
+  name: secret-reader-ovn-ipsec
+  apiGroup: rbac.authorization.k8s.io
 EOF
 
 cat <<EOF > kube-ovn-app-sa.yaml
@@ -4610,6 +4647,7 @@ spec:
           - --enable-metrics=$ENABLE_METRICS
           - --node-local-dns-ip=$NODE_LOCAL_DNS_IP
           - --enable-ovn-ipsec=$ENABLE_OVN_IPSEC
+          - --cert-manager-ipsec-cert=$CERT_MANAGER_IPSEC_CERT
           - --secure-serving=${SECURE_SERVING}
           - --enable-anp=$ENABLE_ANP
           - --ovsdb-con-timeout=$OVSDB_CON_TIMEOUT
@@ -4805,6 +4843,9 @@ spec:
           - --ovs-vsctl-concurrency=$OVS_VSCTL_CONCURRENCY
           - --secure-serving=${SECURE_SERVING}
           - --enable-ovn-ipsec=$ENABLE_OVN_IPSEC
+          - --cert-manager-ipsec-cert=$CERT_MANAGER_IPSEC_CERT
+          - --ovn-ipsec-cert-duration=$IPSEC_CERT_DURATION
+          - --cert-manager-issuer-name=$CERT_MANAGER_ISSUER_NAME
           - --set-vxlan-tx-off=$SET_VXLAN_TX_OFF
         securityContext:
           runAsUser: 0

e2e.mk

@@ -248,6 +248,15 @@ kube-ovn-ipsec-e2e:
 	ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \
 		--focus=CNI:Kube-OVN ./test/e2e/ipsec/ipsec.test -- $(TEST_BIN_ARGS)
 
+.PHONY: kube-ovn-ipsec-e2e-cert-mgr
+kube-ovn-ipsec-e2e-cert-mgr:
+	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ipsec-cert-mgr
+	E2E_BRANCH=$(E2E_BRANCH) \
+	E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
+	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
+	ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \
+		--focus=CNI:Kube-OVN ./test/e2e/ipsec-cert-mgr/ipsec-cert-mgr.test -- $(TEST_BIN_ARGS)
+
 .PHONY: kube-ovn-anp-e2e
 kube-ovn-anp-e2e:
 	KUBECONFIG=$(KUBECONFIG) ./test/anp/conformance.sh

go.mod

@@ -75,6 +75,8 @@ require (
 	sigs.k8s.io/network-policy-api v0.1.5
 )
 
+require sigs.k8s.io/gateway-api v1.1.0 // indirect
+
 require (
 	cel.dev/expr v0.24.0 // indirect
 	cloud.google.com/go/compute/metadata v0.7.0 // indirect
@@ -93,6 +95,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cenkalti/hub v1.0.2 // indirect
 	github.com/cenkalti/rpc2 v1.0.4 // indirect
+	github.com/cert-manager/cert-manager v1.17.1
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chai2010/gettext-go v1.0.3 // indirect
 	github.com/container-storage-interface/spec v1.11.0 // indirect
@@ -160,7 +163,7 @@ require (
 	github.com/httprunner/funplugin v0.5.5 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jinzhu/copier v0.4.0 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect

go.sum

@@ -54,6 +54,8 @@ github.com/cenkalti/hub v1.0.2/go.mod h1:8LAFAZcCasb83vfxatMUnZHRoQcffho2ELpHb+k
 github.com/cenkalti/rpc2 v1.0.4 h1:MJWmm7mbt8r/ZkQS+qr/e2KMMrhMLPr/62CYZIHybdI=
 github.com/cenkalti/rpc2 v1.0.4/go.mod h1:2yfU5b86vOr16+iY1jN3MvT6Kxc9Nf8j5iZWwUf7iaw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cert-manager/cert-manager v1.17.1 h1:Aig+lWMoLsmpGd9TOlTvO4t0Ah3D+/vGB37x/f+ZKt0=
+github.com/cert-manager/cert-manager v1.17.1/go.mod h1:zeG4D+AdzqA7hFMNpYCJgcQ2VOfFNBa+Jzm3kAwiDU4=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v1.0.3 h1:9liNh8t+u26xl5ddmWLmsOsdNLwkdRTg5AG+JnTiM80=
@@ -336,8 +338,8 @@ github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgf
 github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo=
 github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
 github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
-github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 h1:liMMTbpW34dhU4az1GN0pTPADwNmvoRSeoZ6PItiqnY=
+github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
@@ -1225,6 +1227,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.32.1 h1:Cf+ed5N8038zb
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.32.1/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
 sigs.k8s.io/controller-runtime v0.20.4 h1:X3c+Odnxz+iPTRobG4tp092+CvBU9UK0t/bRf+n0DGU=
 sigs.k8s.io/controller-runtime v0.20.4/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY=
+sigs.k8s.io/gateway-api v1.1.0 h1:DsLDXCi6jR+Xz8/xd0Z1PYl2Pn0TyaFMOPPZIj4inDM=
+sigs.k8s.io/gateway-api v1.1.0/go.mod h1:ZH4lHrL2sDi0FHZ9jjneb8kKnGzFWyrTya35sWUTrRs=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo=
 sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=

kind.mk

@@ -657,6 +657,34 @@ kind-install-kwok:
 kind-install-ovn-ipsec:
 	@$(MAKE) ENABLE_OVN_IPSEC=true kind-install
 
+.PHONY: kind-install-cert-manager
+kind-install-cert-manager:
+	$(call kind_load_image,kube-ovn,$(CERT_MANAGER_CONTROLLER),1)
+	kubectl apply -f "$(CERT_MANAGER_YAML)"
+	kubectl rollout status deployment/cert-manager -n cert-manager --timeout 120s
+
+.PHONY: kind-install-ovn-ipsec-cert-manager 
+kind-install-ovn-ipsec-cert-manager:
+	@$(MAKE) ENABLE_OVN_IPSEC=true CERT_MANAGER_IPSEC_CERT=true kind-install
+	@$(MAKE) kind-install-cert-manager
+	
+	kubectl rollout status deployment/cert-manager-webhook -n cert-manager --timeout 120s
+	
+	$(eval CA_KEY = $(shell mktemp))
+	$(shell openssl genrsa -out $(CA_KEY) 2048)
+	$(eval CA_CERT = $(shell openssl req -x509 -new -nodes \
+		-key "$(CA_KEY)" \
+		-days 3650 \
+		-subj "/C=US/ST=California/L=Palo Alto/O=Open vSwitch/OU=Open vSwitch Certification Authority/CN=OVS CA" | \
+		base64 -w 0 -))
+
+	$(eval CA_KEY64 = $(shell base64 -w 0 "$(CA_KEY)"))
+
+	sed -e 's/KUBE_OVN_CA_KEY/$(CA_KEY64)/g' \
+	    -e 's/KUBE_OVN_CA_CERT/$(CA_CERT)/g' yamls/ipsec-certs.yaml | \
+		kubectl apply -f -
+	rm $(CA_KEY)
+
 .PHONY: kind-install-anp
 kind-install-anp: kind-load-image
 	$(call kind_load_image,kube-ovn,$(ANP_TEST_IMAGE),1)

pkg/controller/config.go

@@ -97,6 +97,7 @@ type Configuration struct {
 	EnableMetrics               bool
 	EnableANP                   bool
 	EnableOVNIPSec              bool
+	CertManagerIPSECCert        bool
 	EnableLiveMigrationOptimize bool
 
 	ExternalGatewaySwitch   string
@@ -188,6 +189,7 @@ func ParseFlags() (*Configuration, error) {
 		argEnableMetrics               = pflag.Bool("enable-metrics", true, "Whether to support metrics query")
 		argEnableANP                   = pflag.Bool("enable-anp", false, "Enable support for admin network policy and baseline admin network policy")
 		argEnableOVNIPSec              = pflag.Bool("enable-ovn-ipsec", false, "Whether to enable ovn ipsec")
+		argCertManagerIPSECCert        = pflag.Bool("cert-manager-ipsec-cert", false, "Whether to use cert-manager for signing IPSec certificates")
 		argEnableLiveMigrationOptimize = pflag.Bool("enable-live-migration-optimize", true, "Whether to enable kubevirt live migration optimize")
 
 		argExternalGatewayConfigNS = pflag.String("external-gateway-config-ns", "kube-system", "The namespace of configmap external-gateway-config, default: kube-system")
@@ -290,6 +292,7 @@ func ParseFlags() (*Configuration, error) {
 		EnableOVNLBPreferLocal:         *argEnableOVNLBPreferLocal,
 		EnableMetrics:                  *argEnableMetrics,
 		EnableOVNIPSec:                 *argEnableOVNIPSec,
+		CertManagerIPSECCert:           *argCertManagerIPSECCert,
 		EnableLiveMigrationOptimize:    *argEnableLiveMigrationOptimize,
 		BfdMinTx:                       *argBfdMinTx,
 		BfdMinRx:                       *argBfdMinRx,