From d742af7080c34ed212f37ecfcead37f8b0284b72 Mon Sep 17 00:00:00 2001 From: Damyan Yordanov Date: Tue, 3 Dec 2024 11:17:14 +0100 Subject: [PATCH 1/9] Create ironcore IPv6 PoC --- .../ironcore/v1alpha1/types_infrastructure.go | 4 +- .../infrastructure/actuator_reconcile.go | 123 +++++++++++++++--- pkg/controller/worker/machines.go | 6 +- pkg/ironcore/types.go | 2 +- 4 files changed, 116 insertions(+), 19 deletions(-) diff --git a/pkg/apis/ironcore/v1alpha1/types_infrastructure.go b/pkg/apis/ironcore/v1alpha1/types_infrastructure.go index f205e092..ab05a7a3 100644 --- a/pkg/apis/ironcore/v1alpha1/types_infrastructure.go +++ b/pkg/apis/ironcore/v1alpha1/types_infrastructure.go @@ -35,8 +35,8 @@ type InfrastructureStatus struct { NetworkRef commonv1alpha1.LocalUIDReference `json:"networkRef,omitempty"` // NATGatewayRef is the reference to the NAT gateway used NATGatewayRef commonv1alpha1.LocalUIDReference `json:"natGatewayRef,omitempty"` - // PrefixRef is the reference to the Prefix used - PrefixRef commonv1alpha1.LocalUIDReference `json:"prefixRef,omitempty"` + // PrefixRefs is the reference to the Prefixes used + PrefixRefs []commonv1alpha1.LocalUIDReference `json:"prefixRefs,omitempty"` //NetworkPolicy is reference to the NetworkPolicy defined NetworkPolicyRef commonv1alpha1.LocalUIDReference `json:"networkPolicyRef,omitempty"` } diff --git a/pkg/controller/infrastructure/actuator_reconcile.go b/pkg/controller/infrastructure/actuator_reconcile.go index 58b563df..2809ddc6 100644 --- a/pkg/controller/infrastructure/actuator_reconcile.go +++ b/pkg/controller/infrastructure/actuator_reconcile.go @@ -8,11 +8,16 @@ import ( "fmt" "math/big" "net" + "net/netip" + "slices" + + "github.com/gardener/gardener/pkg/apis/core/v1beta1" "github.com/gardener/gardener/extensions/pkg/controller" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/go-logr/logr" "github.com/ironcore-dev/ironcore/api/common/v1alpha1" + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" ipamv1alpha1 "github.com/ironcore-dev/ironcore/api/ipam/v1alpha1" networkingv1alpha1 "github.com/ironcore-dev/ironcore/api/networking/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -61,7 +66,7 @@ func (a *actuator) reconcile(ctx context.Context, log logr.Logger, infra *extens return err } - prefix, err := a.applyPrefix(ctx, ironcoreClient, namespace, cluster) + prefixes, servicePrefix, err := a.applyPrefixes(ctx, ironcoreClient, namespace, cluster) if err != nil { return err } @@ -73,19 +78,18 @@ func (a *actuator) reconcile(ctx context.Context, log logr.Logger, infra *extens log.V(2).Info("Successfully reconciled infrastructure") - // update status - return a.updateProviderStatus(ctx, infra, network, natGateway, prefix, networkPolicy) + return a.updateProviderStatus(ctx, infra, network, natGateway, prefixes, networkPolicy, servicePrefix, cluster) } -func (a *actuator) applyPrefix(ctx context.Context, ironcoreClient client.Client, namespace string, cluster *controller.Cluster) (*ipamv1alpha1.Prefix, error) { - prefix := &ipamv1alpha1.Prefix{ +func (a *actuator) applyPrefixes(ctx context.Context, ironcoreClient client.Client, namespace string, cluster *controller.Cluster) ([]ipamv1alpha1.Prefix, *ipamv1alpha1.Prefix, error) { + prefixIPV4 := &ipamv1alpha1.Prefix{ TypeMeta: metav1.TypeMeta{ Kind: "Prefix", APIVersion: "ipam.ironcore.dev/v1alpha1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, - Name: generateResourceNameFromCluster(cluster), + Name: generateResourceNameFromCluster(cluster) + "-v4", }, Spec: ipamv1alpha1.PrefixSpec{ // TODO: for now we only support IPv4 until Gardener has support for IPv6 based Shoots @@ -94,14 +98,68 @@ func (a *actuator) applyPrefix(ctx context.Context, ironcoreClient client.Client } if nodeCIDR := cluster.Shoot.Spec.Networking.Nodes; nodeCIDR != nil { - prefix.Spec.Prefix = v1alpha1.MustParseNewIPPrefix(ptr.Deref[string](nodeCIDR, "")) + prefixIPV4.Spec.Prefix = v1alpha1.MustParseNewIPPrefix(ptr.Deref[string](nodeCIDR, "")) } - if _, err := controllerutil.CreateOrPatch(ctx, ironcoreClient, prefix, nil); err != nil { - return nil, fmt.Errorf("failed to apply prefix %s: %w", client.ObjectKeyFromObject(prefix), err) + if _, err := controllerutil.CreateOrPatch(ctx, ironcoreClient, prefixIPV4, nil); err != nil { + return nil, nil, fmt.Errorf("failed to apply prefix %s: %w", client.ObjectKeyFromObject(prefixIPV4), err) } - return prefix, nil + prefixes := []ipamv1alpha1.Prefix{*prefixIPV4} + var servicePrefix *ipamv1alpha1.Prefix + if slices.Contains(cluster.Shoot.Spec.Networking.IPFamilies, v1beta1.IPFamilyIPv6) { + // TODO: Get overlay IPv6 Block from Malte + nodesIPV6Prefix, err := netip.ParsePrefix("2a10:afc0:e010:cafe::/64") + if err != nil { + return nil, nil, fmt.Errorf("failed to parse IPv6 prefix: %w", err) + } + prefixIPV6 := &ipamv1alpha1.Prefix{ + TypeMeta: metav1.TypeMeta{ + Kind: "Prefix", + APIVersion: "ipam.ironcore.dev/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: generateResourceNameFromCluster(cluster) + "-v6", + }, + Spec: ipamv1alpha1.PrefixSpec{ + IPFamily: corev1.IPv6Protocol, + Prefix: &commonv1alpha1.IPPrefix{ + Prefix: nodesIPV6Prefix, + }, + }, + } + if _, err := controllerutil.CreateOrPatch(ctx, ironcoreClient, prefixIPV6, nil); err != nil { + return nil, nil, fmt.Errorf("failed to apply prefix %s: %w", client.ObjectKeyFromObject(prefixIPV6), err) + } + + prefixes = append(prefixes, *prefixIPV6) + + maxPrefixLength := 128 + servicePrefixLength := int32((maxPrefixLength-nodesIPV6Prefix.Bits())/2 + nodesIPV6Prefix.Bits()) + servicePrefix = &ipamv1alpha1.Prefix{ + TypeMeta: metav1.TypeMeta{ + Kind: "Prefix", + APIVersion: "ipam.ironcore.dev/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: generateResourceNameFromCluster(cluster) + "-services-v6", + }, + Spec: ipamv1alpha1.PrefixSpec{ + IPFamily: corev1.IPv6Protocol, + PrefixLength: servicePrefixLength, + ParentRef: &corev1.LocalObjectReference{ + Name: prefixIPV6.Name, + }, + }, + } + if _, err := controllerutil.CreateOrPatch(ctx, ironcoreClient, servicePrefix, nil); err != nil { + return nil, nil, fmt.Errorf("failed to apply prefix %s: %w", client.ObjectKeyFromObject(servicePrefix), err) + } + } + + return prefixes, servicePrefix, nil } func (a *actuator) applyNATGateway(ctx context.Context, config *api.InfrastructureConfig, ironcoreClient client.Client, namespace string, cluster *controller.Cluster, network *networkingv1alpha1.Network) (*networkingv1alpha1.NATGateway, error) { @@ -244,8 +302,10 @@ func (a *actuator) updateProviderStatus( infra *extensionsv1alpha1.Infrastructure, network *networkingv1alpha1.Network, natGateway *networkingv1alpha1.NATGateway, - prefix *ipamv1alpha1.Prefix, + prefixes []ipamv1alpha1.Prefix, networkPolicy *networkingv1alpha1.NetworkPolicy, + servicePrefix *ipamv1alpha1.Prefix, + cluster *controller.Cluster, ) error { infraStatus := &apiv1alpha1.InfrastructureStatus{ TypeMeta: metav1.TypeMeta{ @@ -260,15 +320,48 @@ func (a *actuator) updateProviderStatus( Name: natGateway.Name, UID: natGateway.UID, }, - PrefixRef: v1alpha1.LocalUIDReference{ - Name: prefix.Name, - UID: prefix.UID, - }, NetworkPolicyRef: v1alpha1.LocalUIDReference{ Name: networkPolicy.Name, UID: networkPolicy.UID, }, } + var ( + nodes []string + pods []string + services []string + ) + if cluster.Shoot.Spec.Networking.Pods != nil { + pods = []string{*cluster.Shoot.Spec.Networking.Pods} + } + if cluster.Shoot.Spec.Networking.Services != nil { + services = []string{*cluster.Shoot.Spec.Networking.Services} + } + for _, prefix := range prefixes { + infraStatus.PrefixRefs = append(infraStatus.PrefixRefs, + v1alpha1.LocalUIDReference{ + Name: prefix.Name, + UID: prefix.UID, + }, + ) + nodes = append(nodes, prefix.Spec.Prefix.Prefix.String()) + if prefix.Spec.IPFamily == corev1.IPv6Protocol { + // for IPv6 the pods and the nodes share the same prefix + pods = append(pods, prefix.Spec.Prefix.Prefix.String()) + } + } + if servicePrefix != nil { + if servicePrefix.Status.Phase != ipamv1alpha1.PrefixPhaseAllocated { + return fmt.Errorf("service prefix not yet allocated") + } + if servicePrefix.Spec.Prefix != nil { + services = append(services, servicePrefix.Spec.Prefix.Prefix.String()) + } + } + infra.Status.Networking = &extensionsv1alpha1.InfrastructureStatusNetworking{ + Nodes: nodes, + Pods: pods, + Services: services, + } infraBase := infra.DeepCopy() infra.Status.ProviderStatus = &runtime.RawExtension{ Object: infraStatus, diff --git a/pkg/controller/worker/machines.go b/pkg/controller/worker/machines.go index 6ba7f92d..14fc802d 100644 --- a/pkg/controller/worker/machines.go +++ b/pkg/controller/worker/machines.go @@ -150,8 +150,12 @@ func (w *workerDelegate) generateMachineClassAndSecrets(ctx context.Context) ([] } } + prefixNames := []string{} + for _, prefix := range infrastructureStatus.PrefixRefs { + prefixNames = append(prefixNames, prefix.Name) + } machineClassProviderSpec[ironcore.NetworkFieldName] = infrastructureStatus.NetworkRef.Name - machineClassProviderSpec[ironcore.PrefixFieldName] = infrastructureStatus.PrefixRef.Name + machineClassProviderSpec[ironcore.PrefixFieldName] = prefixNames machineClassProviderSpec[ironcore.LabelsFieldName] = map[string]string{ ironcore.ClusterNameLabel: w.cluster.ObjectMeta.Name, } diff --git a/pkg/ironcore/types.go b/pkg/ironcore/types.go index d990ed71..72ca48b7 100644 --- a/pkg/ironcore/types.go +++ b/pkg/ironcore/types.go @@ -51,7 +51,7 @@ const ( // NetworkFieldName is the name of network field NetworkFieldName = "networkName" // PrefixFieldName is the name of the prefix field - PrefixFieldName = "prefixName" + PrefixFieldName = "prefixNames" // ClusterFieldName is the name of the cluster field ClusterFieldName = "clusterName" // LabelsFieldName is the name of the labels field From 7355a810aa6a9cebc4e94c1483929307c846aa59 Mon Sep 17 00:00:00 2001 From: Damyan Yordanov Date: Tue, 3 Dec 2024 11:19:52 +0100 Subject: [PATCH 2/9] Push PR images to `ghcr.io` as well --- .github/workflows/publish-docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index beee80fe..93fd8e9e 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -73,7 +73,6 @@ jobs: version: latest endpoint: builders # self-hosted - name: Login to GHCR - if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: registry: ghcr.io @@ -85,7 +84,7 @@ jobs: with: context: . platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} target: ${{ matrix.image.target }} From 8f58f538ea6e623707cdb78bfb414813ac447f25 Mon Sep 17 00:00:00 2001 From: Damyan Yordanov Date: Tue, 3 Dec 2024 11:39:27 +0100 Subject: [PATCH 3/9] Regenerate types --- Makefile | 35 +------------------ hack/update-codegen.sh | 2 +- pkg/apis/ironcore/types_infrastructure.go | 4 +-- .../ironcore/v1alpha1/types_infrastructure.go | 2 +- .../v1alpha1/zz_generated.conversion.go | 14 ++------ .../v1alpha1/zz_generated.deepcopy.go | 6 +++- pkg/apis/ironcore/zz_generated.deepcopy.go | 2 +- .../infrastructure/actuator_reconcile.go | 3 +- 8 files changed, 14 insertions(+), 54 deletions(-) diff --git a/Makefile b/Makefile index 452dd26b..a80abcc5 100644 --- a/Makefile +++ b/Makefile @@ -136,7 +136,7 @@ check: $(GOIMPORTS) $(GOLANGCI_LINT) $(MOCKGEN) @REPO_ROOT=$(REPO_ROOT) bash $(GARDENER_HACK_DIR)/check-charts.sh ./charts .PHONY: generate -generate: deepcopy-gen defaulter-gen conversion-gen $(CONTROLLER_GEN) $(HELM) $(MOCKGEN) $(YQ) $(VGOPATH) +generate: $(CONTROLLER_GEN) $(HELM) $(MOCKGEN) $(YQ) $(VGOPATH) @GOPATH=$(GOPATH) VGOPATH=$(VGOPATH) \ MOCKGEN=$(MOCKGEN) \ DEEPCOPY_GEN=$(DEEPCOPY_GEN) \ @@ -184,9 +184,6 @@ $(LOCALBIN): mkdir -p $(LOCALBIN) ## Tool Binaries -DEEPCOPY_GEN ?= $(LOCALBIN)/deepcopy-gen -CONVERSION_GEN ?= $(LOCALBIN)/conversion-gen -DEFAULTER_GEN ?= $(LOCALBIN)/defaulter-gen ENVTEST ?= $(LOCALBIN)/setup-envtest ## Tool Versions @@ -201,33 +198,3 @@ envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) -.PHONY: deepcopy-gen -deepcopy-gen: $(DEEPCOPY_GEN) ## Download deepcopy-gen locally if necessary. -$(DEEPCOPY_GEN): $(LOCALBIN) - $(call go-install-tool,$(DEEPCOPY_GEN),k8s.io/code-generator/cmd/deepcopy-gen,$(CODE_GENERATOR_VERSION)) - -.PHONY: defaulter-gen -defaulter-gen: $(DEFAULTER_GEN) ## Download defaulter-gen locally if necessary. -$(DEFAULTER_GEN): $(LOCALBIN) - $(call go-install-tool,$(DEFAULTER_GEN),k8s.io/code-generator/cmd/defaulter-gen,$(CODE_GENERATOR_VERSION)) - -.PHONY: conversion-gen -conversion-gen: $(CONVERSION_GEN) ## Download conversion-gen locally if necessary. -$(CONVERSION_GEN): $(LOCALBIN) - $(call go-install-tool,$(CONVERSION_GEN),k8s.io/code-generator/cmd/conversion-gen,${CODE_GENERATOR_VERSION}) - -# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist -# $1 - target path with name of binary -# $2 - package url which can be installed -# $3 - specific version of package -define go-install-tool -@[ -f "$(1)-$(3)" ] && [ "$$(readlink -- "$(1)" 2>/dev/null)" = "$(1)-$(3)" ] || { \ -set -e; \ -package=$(2)@$(3) ;\ -echo "Downloading $${package}" ;\ -rm -f $(1) ;\ -GOBIN=$(LOCALBIN) go install $${package} ;\ -mv $(1) $(1)-$(3) ;\ -} ;\ -ln -sf $$(realpath $(1)-$(3)) $(1) -endef diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 7c8daf6b..9b15e994 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash set -o errexit set -o nounset diff --git a/pkg/apis/ironcore/types_infrastructure.go b/pkg/apis/ironcore/types_infrastructure.go index ba7685ad..f3175c49 100644 --- a/pkg/apis/ironcore/types_infrastructure.go +++ b/pkg/apis/ironcore/types_infrastructure.go @@ -35,8 +35,8 @@ type InfrastructureStatus struct { NetworkRef commonv1alpha1.LocalUIDReference // NATGatewayRef is the reference to the NAT gateway used NATGatewayRef commonv1alpha1.LocalUIDReference - // PrefixRef is the reference to the Prefix used - PrefixRef commonv1alpha1.LocalUIDReference + // PrefixRefs are the reference to the Prefixes used + PrefixRefs commonv1alpha1.LocalUIDReference //NetworkPolicy is reference to the NetworkPolicy defined NetworkPolicyRef commonv1alpha1.LocalUIDReference } diff --git a/pkg/apis/ironcore/v1alpha1/types_infrastructure.go b/pkg/apis/ironcore/v1alpha1/types_infrastructure.go index ab05a7a3..1da7f156 100644 --- a/pkg/apis/ironcore/v1alpha1/types_infrastructure.go +++ b/pkg/apis/ironcore/v1alpha1/types_infrastructure.go @@ -35,7 +35,7 @@ type InfrastructureStatus struct { NetworkRef commonv1alpha1.LocalUIDReference `json:"networkRef,omitempty"` // NATGatewayRef is the reference to the NAT gateway used NATGatewayRef commonv1alpha1.LocalUIDReference `json:"natGatewayRef,omitempty"` - // PrefixRefs is the reference to the Prefixes used + // PrefixRefs are the reference to the Prefixes used PrefixRefs []commonv1alpha1.LocalUIDReference `json:"prefixRefs,omitempty"` //NetworkPolicy is reference to the NetworkPolicy defined NetworkPolicyRef commonv1alpha1.LocalUIDReference `json:"networkPolicyRef,omitempty"` diff --git a/pkg/apis/ironcore/v1alpha1/zz_generated.conversion.go b/pkg/apis/ironcore/v1alpha1/zz_generated.conversion.go index fdc6c6e6..00e98625 100644 --- a/pkg/apis/ironcore/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/ironcore/v1alpha1/zz_generated.conversion.go @@ -244,29 +244,19 @@ func Convert_ironcore_InfrastructureConfig_To_v1alpha1_InfrastructureConfig(in * func autoConvert_v1alpha1_InfrastructureStatus_To_ironcore_InfrastructureStatus(in *InfrastructureStatus, out *ironcore.InfrastructureStatus, s conversion.Scope) error { out.NetworkRef = in.NetworkRef out.NATGatewayRef = in.NATGatewayRef - out.PrefixRef = in.PrefixRef + // WARNING: in.PrefixRefs requires manual conversion: inconvertible types ([]github.com/ironcore-dev/ironcore/api/common/v1alpha1.LocalUIDReference vs github.com/ironcore-dev/ironcore/api/common/v1alpha1.LocalUIDReference) out.NetworkPolicyRef = in.NetworkPolicyRef return nil } -// Convert_v1alpha1_InfrastructureStatus_To_ironcore_InfrastructureStatus is an autogenerated conversion function. -func Convert_v1alpha1_InfrastructureStatus_To_ironcore_InfrastructureStatus(in *InfrastructureStatus, out *ironcore.InfrastructureStatus, s conversion.Scope) error { - return autoConvert_v1alpha1_InfrastructureStatus_To_ironcore_InfrastructureStatus(in, out, s) -} - func autoConvert_ironcore_InfrastructureStatus_To_v1alpha1_InfrastructureStatus(in *ironcore.InfrastructureStatus, out *InfrastructureStatus, s conversion.Scope) error { out.NetworkRef = in.NetworkRef out.NATGatewayRef = in.NATGatewayRef - out.PrefixRef = in.PrefixRef + // WARNING: in.PrefixRefs requires manual conversion: inconvertible types (github.com/ironcore-dev/ironcore/api/common/v1alpha1.LocalUIDReference vs []github.com/ironcore-dev/ironcore/api/common/v1alpha1.LocalUIDReference) out.NetworkPolicyRef = in.NetworkPolicyRef return nil } -// Convert_ironcore_InfrastructureStatus_To_v1alpha1_InfrastructureStatus is an autogenerated conversion function. -func Convert_ironcore_InfrastructureStatus_To_v1alpha1_InfrastructureStatus(in *ironcore.InfrastructureStatus, out *InfrastructureStatus, s conversion.Scope) error { - return autoConvert_ironcore_InfrastructureStatus_To_v1alpha1_InfrastructureStatus(in, out, s) -} - func autoConvert_v1alpha1_MachineImage_To_ironcore_MachineImage(in *MachineImage, out *ironcore.MachineImage, s conversion.Scope) error { out.Name = in.Name out.Version = in.Version diff --git a/pkg/apis/ironcore/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/ironcore/v1alpha1/zz_generated.deepcopy.go index f8e11a03..dfcbf597 100644 --- a/pkg/apis/ironcore/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/ironcore/v1alpha1/zz_generated.deepcopy.go @@ -154,7 +154,11 @@ func (in *InfrastructureStatus) DeepCopyInto(out *InfrastructureStatus) { out.TypeMeta = in.TypeMeta out.NetworkRef = in.NetworkRef out.NATGatewayRef = in.NATGatewayRef - out.PrefixRef = in.PrefixRef + if in.PrefixRefs != nil { + in, out := &in.PrefixRefs, &out.PrefixRefs + *out = make([]commonv1alpha1.LocalUIDReference, len(*in)) + copy(*out, *in) + } out.NetworkPolicyRef = in.NetworkPolicyRef return } diff --git a/pkg/apis/ironcore/zz_generated.deepcopy.go b/pkg/apis/ironcore/zz_generated.deepcopy.go index 743506de..b8a8ed47 100644 --- a/pkg/apis/ironcore/zz_generated.deepcopy.go +++ b/pkg/apis/ironcore/zz_generated.deepcopy.go @@ -154,7 +154,7 @@ func (in *InfrastructureStatus) DeepCopyInto(out *InfrastructureStatus) { out.TypeMeta = in.TypeMeta out.NetworkRef = in.NetworkRef out.NATGatewayRef = in.NATGatewayRef - out.PrefixRef = in.PrefixRef + out.PrefixRefs = in.PrefixRefs out.NetworkPolicyRef = in.NetworkPolicyRef return } diff --git a/pkg/controller/infrastructure/actuator_reconcile.go b/pkg/controller/infrastructure/actuator_reconcile.go index 2809ddc6..2378f838 100644 --- a/pkg/controller/infrastructure/actuator_reconcile.go +++ b/pkg/controller/infrastructure/actuator_reconcile.go @@ -11,9 +11,8 @@ import ( "net/netip" "slices" - "github.com/gardener/gardener/pkg/apis/core/v1beta1" - "github.com/gardener/gardener/extensions/pkg/controller" + "github.com/gardener/gardener/pkg/apis/core/v1beta1" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/go-logr/logr" "github.com/ironcore-dev/ironcore/api/common/v1alpha1" From 53adf4eba7682c48073cd9d0cb55809423f56a16 Mon Sep 17 00:00:00 2001 From: Damyan Yordanov Date: Tue, 3 Dec 2024 14:42:46 +0100 Subject: [PATCH 4/9] Fix rest issues --- Makefile | 1 + pkg/apis/ironcore/types_infrastructure.go | 2 +- pkg/apis/ironcore/v1alpha1/types_infrastructure.go | 5 ++--- .../ironcore/v1alpha1/zz_generated.conversion.go | 14 ++++++++++++-- pkg/apis/ironcore/zz_generated.deepcopy.go | 6 +++++- pkg/controller/bastion/actuator_reconcile.go | 3 ++- pkg/controller/bastion/configvalidator.go | 6 ++++-- pkg/controller/controlplane/valuesprovider.go | 10 +++++++--- pkg/controller/worker/machines.go | 2 +- pkg/ironcore/types.go | 4 ++-- 10 files changed, 37 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index a80abcc5..3bb394c2 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ ADMISSION_NAME := admission-ironcore IMAGE_PREFIX := $(REGISTRY)/extensions REPO_ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) HACK_DIR := $(REPO_ROOT)/hack +VERSION := $(shell cat "$(REPO_ROOT)/VERSION") EFFECTIVE_VERSION := $(VERSION)-$(shell git rev-parse HEAD) LD_FLAGS := "-w $(shell bash $(GARDENER_HACK_DIR)/get-build-ld-flags.sh k8s.io/component-base $(REPO_ROOT)/VERSION $(EXTENSION_PREFIX))" LEADER_ELECTION := false diff --git a/pkg/apis/ironcore/types_infrastructure.go b/pkg/apis/ironcore/types_infrastructure.go index f3175c49..363cf1df 100644 --- a/pkg/apis/ironcore/types_infrastructure.go +++ b/pkg/apis/ironcore/types_infrastructure.go @@ -36,7 +36,7 @@ type InfrastructureStatus struct { // NATGatewayRef is the reference to the NAT gateway used NATGatewayRef commonv1alpha1.LocalUIDReference // PrefixRefs are the reference to the Prefixes used - PrefixRefs commonv1alpha1.LocalUIDReference + PrefixRefs []commonv1alpha1.LocalUIDReference //NetworkPolicy is reference to the NetworkPolicy defined NetworkPolicyRef commonv1alpha1.LocalUIDReference } diff --git a/pkg/apis/ironcore/v1alpha1/types_infrastructure.go b/pkg/apis/ironcore/v1alpha1/types_infrastructure.go index 1da7f156..c1f93240 100644 --- a/pkg/apis/ironcore/v1alpha1/types_infrastructure.go +++ b/pkg/apis/ironcore/v1alpha1/types_infrastructure.go @@ -34,9 +34,8 @@ type InfrastructureStatus struct { // NetworkRef is the reference to the networked used NetworkRef commonv1alpha1.LocalUIDReference `json:"networkRef,omitempty"` // NATGatewayRef is the reference to the NAT gateway used - NATGatewayRef commonv1alpha1.LocalUIDReference `json:"natGatewayRef,omitempty"` - // PrefixRefs are the reference to the Prefixes used - PrefixRefs []commonv1alpha1.LocalUIDReference `json:"prefixRefs,omitempty"` + NATGatewayRef commonv1alpha1.LocalUIDReference `json:"natGatewayRef,omitempty"` + PrefixRefs []commonv1alpha1.LocalUIDReference `json:"prefixRefs,omitempty"` //NetworkPolicy is reference to the NetworkPolicy defined NetworkPolicyRef commonv1alpha1.LocalUIDReference `json:"networkPolicyRef,omitempty"` } diff --git a/pkg/apis/ironcore/v1alpha1/zz_generated.conversion.go b/pkg/apis/ironcore/v1alpha1/zz_generated.conversion.go index 00e98625..ceb02289 100644 --- a/pkg/apis/ironcore/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/ironcore/v1alpha1/zz_generated.conversion.go @@ -244,19 +244,29 @@ func Convert_ironcore_InfrastructureConfig_To_v1alpha1_InfrastructureConfig(in * func autoConvert_v1alpha1_InfrastructureStatus_To_ironcore_InfrastructureStatus(in *InfrastructureStatus, out *ironcore.InfrastructureStatus, s conversion.Scope) error { out.NetworkRef = in.NetworkRef out.NATGatewayRef = in.NATGatewayRef - // WARNING: in.PrefixRefs requires manual conversion: inconvertible types ([]github.com/ironcore-dev/ironcore/api/common/v1alpha1.LocalUIDReference vs github.com/ironcore-dev/ironcore/api/common/v1alpha1.LocalUIDReference) + out.PrefixRefs = *(*[]commonv1alpha1.LocalUIDReference)(unsafe.Pointer(&in.PrefixRefs)) out.NetworkPolicyRef = in.NetworkPolicyRef return nil } +// Convert_v1alpha1_InfrastructureStatus_To_ironcore_InfrastructureStatus is an autogenerated conversion function. +func Convert_v1alpha1_InfrastructureStatus_To_ironcore_InfrastructureStatus(in *InfrastructureStatus, out *ironcore.InfrastructureStatus, s conversion.Scope) error { + return autoConvert_v1alpha1_InfrastructureStatus_To_ironcore_InfrastructureStatus(in, out, s) +} + func autoConvert_ironcore_InfrastructureStatus_To_v1alpha1_InfrastructureStatus(in *ironcore.InfrastructureStatus, out *InfrastructureStatus, s conversion.Scope) error { out.NetworkRef = in.NetworkRef out.NATGatewayRef = in.NATGatewayRef - // WARNING: in.PrefixRefs requires manual conversion: inconvertible types (github.com/ironcore-dev/ironcore/api/common/v1alpha1.LocalUIDReference vs []github.com/ironcore-dev/ironcore/api/common/v1alpha1.LocalUIDReference) + out.PrefixRefs = *(*[]commonv1alpha1.LocalUIDReference)(unsafe.Pointer(&in.PrefixRefs)) out.NetworkPolicyRef = in.NetworkPolicyRef return nil } +// Convert_ironcore_InfrastructureStatus_To_v1alpha1_InfrastructureStatus is an autogenerated conversion function. +func Convert_ironcore_InfrastructureStatus_To_v1alpha1_InfrastructureStatus(in *ironcore.InfrastructureStatus, out *InfrastructureStatus, s conversion.Scope) error { + return autoConvert_ironcore_InfrastructureStatus_To_v1alpha1_InfrastructureStatus(in, out, s) +} + func autoConvert_v1alpha1_MachineImage_To_ironcore_MachineImage(in *MachineImage, out *ironcore.MachineImage, s conversion.Scope) error { out.Name = in.Name out.Version = in.Version diff --git a/pkg/apis/ironcore/zz_generated.deepcopy.go b/pkg/apis/ironcore/zz_generated.deepcopy.go index b8a8ed47..c351a98d 100644 --- a/pkg/apis/ironcore/zz_generated.deepcopy.go +++ b/pkg/apis/ironcore/zz_generated.deepcopy.go @@ -154,7 +154,11 @@ func (in *InfrastructureStatus) DeepCopyInto(out *InfrastructureStatus) { out.TypeMeta = in.TypeMeta out.NetworkRef = in.NetworkRef out.NATGatewayRef = in.NATGatewayRef - out.PrefixRefs = in.PrefixRefs + if in.PrefixRefs != nil { + in, out := &in.PrefixRefs, &out.PrefixRefs + *out = make([]v1alpha1.LocalUIDReference, len(*in)) + copy(*out, *in) + } out.NetworkPolicyRef = in.NetworkPolicyRef return } diff --git a/pkg/controller/bastion/actuator_reconcile.go b/pkg/controller/bastion/actuator_reconcile.go index 876029a1..d3f762a4 100644 --- a/pkg/controller/bastion/actuator_reconcile.go +++ b/pkg/controller/bastion/actuator_reconcile.go @@ -252,8 +252,9 @@ func generateMachine(namespace string, bastionConfig *controllerconfig.BastionCo Spec: ipamv1alpha1.PrefixSpec{ // request single IP PrefixLength: 32, + // TODO do we need to generate IPv6 bastion hosts? ParentRef: &corev1.LocalObjectReference{ - Name: infraStatus.PrefixRef.Name, + Name: infraStatus.PrefixRefs[0].Name, }, }, }, diff --git a/pkg/controller/bastion/configvalidator.go b/pkg/controller/bastion/configvalidator.go index 8ac68983..ffda5fd9 100644 --- a/pkg/controller/bastion/configvalidator.go +++ b/pkg/controller/bastion/configvalidator.go @@ -122,8 +122,10 @@ func validateInfrastructureStatus(infrastructureStatus *api.InfrastructureStatus return fmt.Errorf("network ref must be not empty for infrastructure provider status") } - if infrastructureStatus.PrefixRef == emptyref { - return fmt.Errorf("prefix ref must be not empty for infrastructure provider status") + for _, prefixRef := range infrastructureStatus.PrefixRefs { + if prefixRef == emptyref { + return fmt.Errorf("prefix ref must be not empty for infrastructure provider status") + } } return nil diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index 59c0be7e..ae8deff9 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -215,10 +215,14 @@ func (vp *valuesProvider) GetConfigChartValues( return nil, fmt.Errorf("failed to decode infrastructure status: %w", err) } // Collect config chart values + var prefixNames []string + for _, prefix := range infrastructureStatus.PrefixRefs { + prefixNames = append(prefixNames, prefix.Name) + } return map[string]interface{}{ - ironcore.NetworkFieldName: infrastructureStatus.NetworkRef.Name, - ironcore.PrefixFieldName: infrastructureStatus.PrefixRef.Name, - ironcore.ClusterFieldName: cluster.ObjectMeta.Name, + ironcore.NetworkFieldName: infrastructureStatus.NetworkRef.Name, + ironcore.PrefixesFieldName: prefixNames, + ironcore.ClusterFieldName: cluster.ObjectMeta.Name, }, nil } diff --git a/pkg/controller/worker/machines.go b/pkg/controller/worker/machines.go index 14fc802d..69a76a3a 100644 --- a/pkg/controller/worker/machines.go +++ b/pkg/controller/worker/machines.go @@ -155,7 +155,7 @@ func (w *workerDelegate) generateMachineClassAndSecrets(ctx context.Context) ([] prefixNames = append(prefixNames, prefix.Name) } machineClassProviderSpec[ironcore.NetworkFieldName] = infrastructureStatus.NetworkRef.Name - machineClassProviderSpec[ironcore.PrefixFieldName] = prefixNames + machineClassProviderSpec[ironcore.PrefixesFieldName] = prefixNames machineClassProviderSpec[ironcore.LabelsFieldName] = map[string]string{ ironcore.ClusterNameLabel: w.cluster.ObjectMeta.Name, } diff --git a/pkg/ironcore/types.go b/pkg/ironcore/types.go index 72ca48b7..856c67ee 100644 --- a/pkg/ironcore/types.go +++ b/pkg/ironcore/types.go @@ -50,8 +50,8 @@ const ( TokenFieldName = "token" // NetworkFieldName is the name of network field NetworkFieldName = "networkName" - // PrefixFieldName is the name of the prefix field - PrefixFieldName = "prefixNames" + // PrefixesFieldName is the name of the prefix field + PrefixesFieldName = "prefixNames" // ClusterFieldName is the name of the cluster field ClusterFieldName = "clusterName" // LabelsFieldName is the name of the labels field From 4c889a05349f1af9e39ef6d41e5cf02381d05f3e Mon Sep 17 00:00:00 2001 From: Damyan Yordanov Date: Tue, 3 Dec 2024 15:06:06 +0100 Subject: [PATCH 5/9] Fix tests --- pkg/controller/bastion/suite_test.go | 10 ++++++---- .../controlplane/valuesprovider_test.go | 8 +++++--- pkg/controller/worker/machines_test.go | 15 +++++++++++---- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/pkg/controller/bastion/suite_test.go b/pkg/controller/bastion/suite_test.go index f97374a1..eef1ff70 100644 --- a/pkg/controller/bastion/suite_test.go +++ b/pkg/controller/bastion/suite_test.go @@ -256,9 +256,9 @@ func SetupTest() *corev1.Namespace { }, Minimum: 0, Name: "pool", + //UserData: []byte("some-data"), UserDataSecretRef: corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: userDataSecretName}, - Key: userDataSecretDataKey, }, Volume: &extensionsv1alpha1.Volume{ Name: &volumeName, @@ -285,9 +285,11 @@ func SetupTest() *corev1.Namespace { Name: "my-network", UID: "1234", }, - PrefixRef: commonv1alpha1.LocalUIDReference{ - Name: "my-prefix", - UID: "4321", + PrefixRefs: []commonv1alpha1.LocalUIDReference{ + { + Name: "my-prefix", + UID: "4321", + }, }, } worker.Spec.InfrastructureProviderStatus = &apiruntime.RawExtension{Object: infraStatus} diff --git a/pkg/controller/controlplane/valuesprovider_test.go b/pkg/controller/controlplane/valuesprovider_test.go index e7c8088c..56aaf572 100644 --- a/pkg/controller/controlplane/valuesprovider_test.go +++ b/pkg/controller/controlplane/valuesprovider_test.go @@ -84,9 +84,11 @@ var _ = Describe("Valueprovider Reconcile", func() { Name: "my-network", UID: "1234", }, - PrefixRef: v1alpha1.LocalUIDReference{ - Name: "my-prefix", - UID: "6789", + PrefixRefs: []v1alpha1.LocalUIDReference{ + { + Name: "my-prefix", + UID: "6789", + }, }, }), }, diff --git a/pkg/controller/worker/machines_test.go b/pkg/controller/worker/machines_test.go index cd7a1761..3eafa62e 100644 --- a/pkg/controller/worker/machines_test.go +++ b/pkg/controller/worker/machines_test.go @@ -40,9 +40,11 @@ var _ = Describe("Machines", func() { Name: "my-network", UID: "1234", }, - PrefixRef: commonv1alpha1.LocalUIDReference{ - Name: "my-prefix", - UID: "3766", + PrefixRefs: []commonv1alpha1.LocalUIDReference{ + { + Name: "my-prefix", + UID: "3766", + }, }, } w.Spec.InfrastructureProviderStatus = &runtime.RawExtension{Raw: encodeObject(infraStatus)} @@ -72,6 +74,11 @@ var _ = Describe("Machines", func() { }, } + var prefixNames []string + + for _, prefix := range infraStatus.PrefixRefs { + prefixNames = append(prefixNames, prefix.Name) + } machineClassProviderSpec := map[string]interface{}{ "image": "registry/my-os", "rootDisk": map[string]interface{}{ @@ -79,7 +86,7 @@ var _ = Describe("Machines", func() { "volumeClassName": pool.Volume.Type, }, "networkName": infraStatus.NetworkRef.Name, - "prefixName": infraStatus.PrefixRef.Name, + "prefixNames": prefixNames, "labels": map[string]interface{}{ ironcore.ClusterNameLabel: testCluster.ObjectMeta.Name, }, From 4f44303dd740eee24ab30195b1b01b1ab4bb7cbd Mon Sep 17 00:00:00 2001 From: Damyan Yordanov Date: Wed, 4 Dec 2024 17:32:39 +0100 Subject: [PATCH 6/9] Replace kube-controller-manager with cloud-controller-manager for pod IPAM --- pkg/webhook/controlplane/ensurer.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/webhook/controlplane/ensurer.go b/pkg/webhook/controlplane/ensurer.go index 65590a06..46fdec90 100644 --- a/pkg/webhook/controlplane/ensurer.go +++ b/pkg/webhook/controlplane/ensurer.go @@ -139,6 +139,9 @@ func ensureKubeAPIServerCommandLineArgs(c *corev1.Container) { func ensureKubeControllerManagerCommandLineArgs(c *corev1.Container) { c.Command = extensionswebhook.EnsureStringWithPrefix(c.Command, "--cloud-provider=", "external") c.Command = extensionswebhook.EnsureNoStringWithPrefix(c.Command, "--cloud-config=") + + // allocate-node-cidrs is a boolean flag and could be enabled by name without an explicit value passed. Therefore, we delete all prefixes (without including "=" in the prefix) + c.Command = extensionswebhook.EnsureNoStringWithPrefix(c.Command, "--allocate-node-cidrs") } // EnsureKubeletServiceUnitOptions ensures that the kubelet.service unit options conform to the provider requirements. From fdc8e0e77143ae3e762f174150c5b8597a460a1f Mon Sep 17 00:00:00 2001 From: Damyan Yordanov Date: Thu, 5 Dec 2024 09:57:42 +0100 Subject: [PATCH 7/9] Fix infrastructure status patch --- pkg/controller/infrastructure/actuator_reconcile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/infrastructure/actuator_reconcile.go b/pkg/controller/infrastructure/actuator_reconcile.go index 2378f838..1ef98b39 100644 --- a/pkg/controller/infrastructure/actuator_reconcile.go +++ b/pkg/controller/infrastructure/actuator_reconcile.go @@ -356,12 +356,12 @@ func (a *actuator) updateProviderStatus( services = append(services, servicePrefix.Spec.Prefix.Prefix.String()) } } + infraBase := infra.DeepCopy() infra.Status.Networking = &extensionsv1alpha1.InfrastructureStatusNetworking{ Nodes: nodes, Pods: pods, Services: services, } - infraBase := infra.DeepCopy() infra.Status.ProviderStatus = &runtime.RawExtension{ Object: infraStatus, } From 032632b9b7fdc5849909544e7111c77c0cdacb1c Mon Sep 17 00:00:00 2001 From: Damyan Yordanov Date: Thu, 5 Dec 2024 10:24:35 +0100 Subject: [PATCH 8/9] Fix service mask length --- pkg/controller/infrastructure/actuator_reconcile.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/controller/infrastructure/actuator_reconcile.go b/pkg/controller/infrastructure/actuator_reconcile.go index 1ef98b39..80f7f32f 100644 --- a/pkg/controller/infrastructure/actuator_reconcile.go +++ b/pkg/controller/infrastructure/actuator_reconcile.go @@ -134,8 +134,7 @@ func (a *actuator) applyPrefixes(ctx context.Context, ironcoreClient client.Clie prefixes = append(prefixes, *prefixIPV6) - maxPrefixLength := 128 - servicePrefixLength := int32((maxPrefixLength-nodesIPV6Prefix.Bits())/2 + nodesIPV6Prefix.Bits()) + servicePrefixLength := int32(112) servicePrefix = &ipamv1alpha1.Prefix{ TypeMeta: metav1.TypeMeta{ Kind: "Prefix", From 49caeb9ac1943fcf7224dd874ffa66dc5e99de95 Mon Sep 17 00:00:00 2001 From: Damyan Yordanov Date: Thu, 5 Dec 2024 14:17:47 +0100 Subject: [PATCH 9/9] Add multi shoot --- .../infrastructure/actuator_reconcile.go | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/pkg/controller/infrastructure/actuator_reconcile.go b/pkg/controller/infrastructure/actuator_reconcile.go index 80f7f32f..0527ed94 100644 --- a/pkg/controller/infrastructure/actuator_reconcile.go +++ b/pkg/controller/infrastructure/actuator_reconcile.go @@ -108,23 +108,43 @@ func (a *actuator) applyPrefixes(ctx context.Context, ironcoreClient client.Clie var servicePrefix *ipamv1alpha1.Prefix if slices.Contains(cluster.Shoot.Spec.Networking.IPFamilies, v1beta1.IPFamilyIPv6) { // TODO: Get overlay IPv6 Block from Malte - nodesIPV6Prefix, err := netip.ParsePrefix("2a10:afc0:e010:cafe::/64") + rootPrefix, err := netip.ParsePrefix("2a10:afc0:e010:cafe::/64") if err != nil { return nil, nil, fmt.Errorf("failed to parse IPv6 prefix: %w", err) } - prefixIPV6 := &ipamv1alpha1.Prefix{ + rootPrefixIPv6 := &ipamv1alpha1.Prefix{ TypeMeta: metav1.TypeMeta{ Kind: "Prefix", APIVersion: "ipam.ironcore.dev/v1alpha1", }, ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, - Name: generateResourceNameFromCluster(cluster) + "-v6", + Name: "root-prefix-v6-all-shoots", }, Spec: ipamv1alpha1.PrefixSpec{ IPFamily: corev1.IPv6Protocol, Prefix: &commonv1alpha1.IPPrefix{ - Prefix: nodesIPV6Prefix, + Prefix: rootPrefix, + }, + }, + } + if _, err := controllerutil.CreateOrPatch(ctx, ironcoreClient, rootPrefixIPv6, nil); err != nil { + return nil, nil, fmt.Errorf("failed to apply root prefix %s: %w", client.ObjectKeyFromObject(rootPrefixIPv6), err) + } + prefixIPV6 := &ipamv1alpha1.Prefix{ + TypeMeta: metav1.TypeMeta{ + Kind: "Prefix", + APIVersion: "ipam.ironcore.dev/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: generateResourceNameFromCluster(cluster) + "-v6", + }, + Spec: ipamv1alpha1.PrefixSpec{ + IPFamily: corev1.IPv6Protocol, + PrefixLength: 96, + ParentRef: &corev1.LocalObjectReference{ + Name: rootPrefixIPv6.Name, }, }, }