From 85f93977ba6cd11e178bb32899f3d045aef3e902 Mon Sep 17 00:00:00 2001 From: Richard Case Date: Wed, 12 Feb 2025 17:41:30 +0000 Subject: [PATCH 1/2] chore: graduate garbage collection This graduate the garbage collection feature from experimental to beta. Signed-off-by: Richard Case --- Makefile | 11 -- config/manager/manager.yaml | 86 +++++++------- docs/book/src/topics/external-resource-gc.md | 10 +- feature/feature.go | 4 +- scripts/ci-e2e-eks-gc.sh | 112 ------------------ scripts/ci-e2e-gc.sh | 112 ------------------ test/e2e/data/e2e_conf.yaml | 1 - test/e2e/data/e2e_eks_conf.yaml | 1 - .../gc_managed/gc_managed_suite_test.go | 77 ------------ .../gc_unmanaged/gc_unmanaged_suite_test.go | 64 ---------- .../gc_managed_test.go => managed/gc_test.go} | 42 ++----- .../gc_test.go} | 34 +----- 12 files changed, 62 insertions(+), 492 deletions(-) delete mode 100755 scripts/ci-e2e-eks-gc.sh delete mode 100755 scripts/ci-e2e-gc.sh delete mode 100644 test/e2e/suites/gc_managed/gc_managed_suite_test.go delete mode 100644 test/e2e/suites/gc_unmanaged/gc_unmanaged_suite_test.go rename test/e2e/suites/{gc_managed/gc_managed_test.go => managed/gc_test.go} (87%) rename test/e2e/suites/{gc_unmanaged/gc_unmanaged_test.go => unmanaged/gc_test.go} (85%) diff --git a/Makefile b/Makefile index 3b4a078c03..b84e8eb99e 100644 --- a/Makefile +++ b/Makefile @@ -433,15 +433,6 @@ test-e2e: $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) generate-test-flavors e2e-image ## test-e2e-eks: generate-test-flavors $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run eks e2e tests time go run github.com/onsi/ginkgo/v2/ginkgo -tags=e2e $(GINKGO_ARGS) ./test/e2e/suites/managed/... -- -config-path="$(E2E_EKS_CONF_PATH)" --source-template="$(EKS_SOURCE_TEMPLATE)" $(E2E_ARGS) $(EKS_E2E_ARGS) -.PHONY: test-e2e-gc ## Run garbage collection e2e tests using clusterctl -test-e2e-gc: generate-test-flavors $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run eks e2e tests - time go run github.com/onsi/ginkgo/v2/ginkgo -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" $(GINKGO_ARGS) -p ./test/e2e/suites/gc_unmanaged/... -- -config-path="$(E2E_CONF_PATH)" $(E2E_ARGS) - -.PHONY: test-e2e-eks-gc ## Run EKS garbage collection e2e tests using clusterctl -test-e2e-eks-gc: generate-test-flavors $(KIND) $(SSM_PLUGIN) $(KUSTOMIZE) e2e-image ## Run eks e2e tests - time go run github.com/onsi/ginkgo/v2/ginkgo -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" $(GINKGO_ARGS) ./test/e2e/suites/gc_managed/... -- -config-path="$(E2E_EKS_CONF_PATH)" --source-template="$(EKS_SOURCE_TEMPLATE)" $(E2E_ARGS) $(EKS_E2E_ARGS) - - CONFORMANCE_E2E_ARGS ?= -kubetest.config-file=$(KUBETEST_CONF_PATH) CONFORMANCE_E2E_ARGS += $(E2E_ARGS) CONFORMANCE_GINKGO_ARGS += $(GINKGO_ARGS) @@ -464,8 +455,6 @@ compile-e2e: ## Test e2e compilation go test -c -o /dev/null -tags=e2e ./test/e2e/suites/unmanaged go test -c -o /dev/null -tags=e2e ./test/e2e/suites/conformance go test -c -o /dev/null -tags=e2e ./test/e2e/suites/managed - go test -c -o /dev/null -tags=e2e ./test/e2e/suites/gc_managed - go test -c -o /dev/null -tags=e2e ./test/e2e/suites/gc_unmanaged .PHONY: docker-pull-e2e-preloads diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 10ae7d696c..18c5547f53 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -17,38 +17,38 @@ spec: control-plane: capa-controller-manager spec: containers: - - args: - - "--leader-elect" - - "--feature-gates=EKS=${CAPA_EKS:=true},EKSEnableIAM=${CAPA_EKS_IAM:=false},EKSAllowAddRoles=${CAPA_EKS_ADD_ROLES:=false},EKSFargate=${EXP_EKS_FARGATE:=false},MachinePool=${EXP_MACHINE_POOL:=false},EventBridgeInstanceState=${EVENT_BRIDGE_INSTANCE_STATE:=false},AutoControllerIdentityCreator=${AUTO_CONTROLLER_IDENTITY_CREATOR:=true},BootstrapFormatIgnition=${EXP_BOOTSTRAP_FORMAT_IGNITION:=false},ExternalResourceGC=${EXP_EXTERNAL_RESOURCE_GC:=false},AlternativeGCStrategy=${EXP_ALTERNATIVE_GC_STRATEGY:=false},TagUnmanagedNetworkResources=${TAG_UNMANAGED_NETWORK_RESOURCES:=true},ROSA=${EXP_ROSA:=false}" - - "--v=${CAPA_LOGLEVEL:=0}" - - "--diagnostics-address=${CAPA_DIAGNOSTICS_ADDRESS:=:8443}" - - "--insecure-diagnostics=${CAPA_INSECURE_DIAGNOSTICS:=false}" - image: controller:latest - imagePullPolicy: Always - name: manager - ports: - - containerPort: 9440 - name: healthz - protocol: TCP - - containerPort: 8443 - name: metrics - protocol: TCP - readinessProbe: - httpGet: - path: /readyz - port: healthz - livenessProbe: - httpGet: - path: /healthz - port: healthz - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsUser: 65532 - runAsGroup: 65532 - terminationMessagePolicy: FallbackToLogsOnError + - args: + - "--leader-elect" + - "--feature-gates=EKS=${CAPA_EKS:=true},EKSEnableIAM=${CAPA_EKS_IAM:=false},EKSAllowAddRoles=${CAPA_EKS_ADD_ROLES:=false},EKSFargate=${EXP_EKS_FARGATE:=false},MachinePool=${EXP_MACHINE_POOL:=false},EventBridgeInstanceState=${EVENT_BRIDGE_INSTANCE_STATE:=false},AutoControllerIdentityCreator=${AUTO_CONTROLLER_IDENTITY_CREATOR:=true},BootstrapFormatIgnition=${EXP_BOOTSTRAP_FORMAT_IGNITION:=false},ExternalResourceGC=${EXTERNAL_RESOURCE_GC:=true},AlternativeGCStrategy=${ALTERNATIVE_GC_STRATEGY:=false},TagUnmanagedNetworkResources=${TAG_UNMANAGED_NETWORK_RESOURCES:=true},ROSA=${EXP_ROSA:=false}" + - "--v=${CAPA_LOGLEVEL:=0}" + - "--diagnostics-address=${CAPA_DIAGNOSTICS_ADDRESS:=:8443}" + - "--insecure-diagnostics=${CAPA_INSECURE_DIAGNOSTICS:=false}" + image: controller:latest + imagePullPolicy: Always + name: manager + ports: + - containerPort: 9440 + name: healthz + protocol: TCP + - containerPort: 8443 + name: metrics + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: healthz + livenessProbe: + httpGet: + path: /healthz + port: healthz + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsUser: 65532 + runAsGroup: 65532 + terminationMessagePolicy: FallbackToLogsOnError securityContext: runAsNonRoot: true seccompProfile: @@ -62,15 +62,15 @@ spec: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - - weight: 10 - preference: - matchExpressions: - - key: ${K8S_CP_LABEL:=node-role.kubernetes.io/control-plane} - operator: Exists - # remove once usage of node-role.kubernetes.io/master is removed from Kubernetes - - weight: 10 - preference: - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists + - weight: 10 + preference: + matchExpressions: + - key: ${K8S_CP_LABEL:=node-role.kubernetes.io/control-plane} + operator: Exists + # remove once usage of node-role.kubernetes.io/master is removed from Kubernetes + - weight: 10 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists serviceAccountName: manager diff --git a/docs/book/src/topics/external-resource-gc.md b/docs/book/src/topics/external-resource-gc.md index 5f7ce6fdda..abe8f3ee50 100644 --- a/docs/book/src/topics/external-resource-gc.md +++ b/docs/book/src/topics/external-resource-gc.md @@ -1,6 +1,6 @@ # External Resource Garbage Collection -- **Feature status:** Experimental +- **Feature status:** Stable - **Feature gate (required):** ExternalResourceGC=true ## Overview @@ -21,16 +21,16 @@ We will look to support deleting EBS volumes in the future potentially. > Note: this feature will likely be superseded by an upstream CAPI feature in the future when [this issue](https://github.com/kubernetes-sigs/cluster-api/issues/3075) is resolved. -## Enabling +## Disabling -To enable garbage collection, you must set the `ExternalResourceGC` feature gate to `true` on the controller manager. The easiest way to do this is via an environment variable: +The garbage collection feature is enabled by default. If you want to disable the feature then you must set the `ExternalResourceGC` feature gate to `false` on the controller manager. The easiest way to do this is via an environment variable: ```bash -export EXP_EXTERNAL_RESOURCE_GC=true +export EXTERNAL_RESOURCE_GC=false clusterctl init --infrastructure aws ``` -> Note: if you enable this feature **ALL** clusters will be marked as requiring garbage collection. +> Note: if you disablw this feature **ALL** clusters will be marked as not requiring garbage collection. ## Operations diff --git a/feature/feature.go b/feature/feature.go index 110c75c5ba..0fef034790 100644 --- a/feature/feature.go +++ b/feature/feature.go @@ -111,8 +111,8 @@ var defaultCAPAFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ MachinePoolMachines: {Default: false, PreRelease: featuregate.Alpha}, AutoControllerIdentityCreator: {Default: true, PreRelease: featuregate.Alpha}, BootstrapFormatIgnition: {Default: false, PreRelease: featuregate.Alpha}, - ExternalResourceGC: {Default: false, PreRelease: featuregate.Alpha}, - AlternativeGCStrategy: {Default: false, PreRelease: featuregate.Alpha}, + ExternalResourceGC: {Default: true, PreRelease: featuregate.Beta}, + AlternativeGCStrategy: {Default: false, PreRelease: featuregate.Beta}, TagUnmanagedNetworkResources: {Default: true, PreRelease: featuregate.Alpha}, ROSA: {Default: false, PreRelease: featuregate.Alpha}, } diff --git a/scripts/ci-e2e-eks-gc.sh b/scripts/ci-e2e-eks-gc.sh deleted file mode 100755 index e9e9329631..0000000000 --- a/scripts/ci-e2e-eks-gc.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash - -# Copyright 2022 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -################################################################################ -# usage: e2e.sh -# This program runs the e2e tests. -# -# ENVIRONMENT VARIABLES -# JANITOR_ENABLED -# Set to 1 to run the aws-janitor command after running the e2e tests. -################################################################################ - -set -o nounset -set -o pipefail - -REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -cd "${REPO_ROOT}" || exit 1 - -# shellcheck source=../hack/ensure-go.sh -source "${REPO_ROOT}/hack/ensure-go.sh" -# shellcheck source=../hack/ensure-kind.sh -source "${REPO_ROOT}/hack/ensure-kind.sh" -# shellcheck source=../hack/ensure-kubectl.sh -source "${REPO_ROOT}/hack/ensure-kubectl.sh" - -ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}" -mkdir -p "$ARTIFACTS/logs/" - -# our exit handler (trap) -cleanup() { - # stop boskos heartbeat - [[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}" -} -trap cleanup EXIT - -#Install requests module explicitly for HTTP calls -apt update -apt install -y python3-pip python3-requests - -# If BOSKOS_HOST is set then acquire an AWS account from Boskos. -if [ -n "${BOSKOS_HOST:-}" ]; then - # Check out the account from Boskos and store the produced environment - # variables in a temporary file. - account_env_var_file="$(mktemp)" - python3 hack/boskos.py --get 1>"${account_env_var_file}" - checkout_account_status="${?}" - - # If the checkout process was a success then load the account's - # environment variables into this process. - # shellcheck disable=SC1090 - [ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}" - - # Always remove the account environment variable file. It contains - # sensitive information. - rm -f "${account_env_var_file}" - - if [ ! "${checkout_account_status}" = "0" ]; then - echo "error getting account from boskos" 1>&2 - exit "${checkout_account_status}" - fi - - # run the heart beat process to tell boskos that we are still - # using the checked out account periodically - python3 -u hack/boskos.py --heartbeat >>$ARTIFACTS/logs/boskos.log 2>&1 & - HEART_BEAT_PID=$(echo $!) -fi - -# Prevent a disallowed AWS key from being used. -if grep -iqF "$(echo "${AWS_ACCESS_KEY_ID-}" | - { md5sum 2>/dev/null || md5; } | - awk '{print $1}')" hack/e2e-aws-disallowed.txt; then - echo "The provided AWS key is not allowed" 1>&2 - exit 1 -fi - -EXP_EXTERNAL_RESOURCE_GC="true" GC_WORKLOAD="../../data/gcworkload.yaml" make test-e2e-eks-gc ARTIFACTS=$ARTIFACTS - -test_status="${?}" - -# If Boskos is being used then release the AWS account back to Boskos. -[ -z "${BOSKOS_HOST:-}" ] || python3 -u hack/boskos.py --release - -# The janitor is typically not run as part of the e2e process, but rather -# in a parallel process via a service on the same cluster that runs Prow and -# Boskos. -# -# However, setting JANITOR_ENABLED=1 tells this program to run the janitor -# after the e2e test is executed. -if [ "${JANITOR_ENABLED:-0}" = "1" ]; then - if ! command -v aws-janitor >/dev/null 2>&1; then - echo "skipping janitor; aws-janitor not found" 1>&2 - else - aws-janitor -all -v 2 - fi -else - echo "skipping janitor; JANITOR_ENABLED=${JANITOR_ENABLED:-0}" 1>&2 -fi - -exit "${test_status}" diff --git a/scripts/ci-e2e-gc.sh b/scripts/ci-e2e-gc.sh deleted file mode 100755 index baccf8525d..0000000000 --- a/scripts/ci-e2e-gc.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash - -# Copyright 2022 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -################################################################################ -# usage: e2e.sh -# This program runs the e2e tests. -# -# ENVIRONMENT VARIABLES -# JANITOR_ENABLED -# Set to 1 to run the aws-janitor command after running the e2e tests. -################################################################################ - -set -o nounset -set -o pipefail - -REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -cd "${REPO_ROOT}" || exit 1 - -# shellcheck source=../hack/ensure-go.sh -source "${REPO_ROOT}/hack/ensure-go.sh" -# shellcheck source=../hack/ensure-kind.sh -source "${REPO_ROOT}/hack/ensure-kind.sh" -# shellcheck source=../hack/ensure-kubectl.sh -source "${REPO_ROOT}/hack/ensure-kubectl.sh" - -ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}" -mkdir -p "$ARTIFACTS/logs/" - -# our exit handler (trap) -cleanup() { - # stop boskos heartbeat - [[ -z ${HEART_BEAT_PID:-} ]] || kill -9 "${HEART_BEAT_PID}" -} -trap cleanup EXIT - -#Install requests module explicitly for HTTP calls -apt update -apt install -y python3-pip python3-requests - -# If BOSKOS_HOST is set then acquire an AWS account from Boskos. -if [ -n "${BOSKOS_HOST:-}" ]; then - # Check out the account from Boskos and store the produced environment - # variables in a temporary file. - account_env_var_file="$(mktemp)" - python3 hack/boskos.py --get 1>"${account_env_var_file}" - checkout_account_status="${?}" - - # If the checkout process was a success then load the account's - # environment variables into this process. - # shellcheck disable=SC1090 - [ "${checkout_account_status}" = "0" ] && . "${account_env_var_file}" - - # Always remove the account environment variable file. It contains - # sensitive information. - rm -f "${account_env_var_file}" - - if [ ! "${checkout_account_status}" = "0" ]; then - echo "error getting account from boskos" 1>&2 - exit "${checkout_account_status}" - fi - - # run the heart beat process to tell boskos that we are still - # using the checked out account periodically - python3 -u hack/boskos.py --heartbeat >>$ARTIFACTS/logs/boskos.log 2>&1 & - HEART_BEAT_PID=$(echo $!) -fi - -# Prevent a disallowed AWS key from being used. -if grep -iqF "$(echo "${AWS_ACCESS_KEY_ID-}" | - { md5sum 2>/dev/null || md5; } | - awk '{print $1}')" hack/e2e-aws-disallowed.txt; then - echo "The provided AWS key is not allowed" 1>&2 - exit 1 -fi - -EXP_EXTERNAL_RESOURCE_GC="true" GC_WORKLOAD="../../data/gcworkload.yaml" make test-e2e-gc ARTIFACTS=$ARTIFACTS - -test_status="${?}" - -# If Boskos is being used then release the AWS account back to Boskos. -[ -z "${BOSKOS_HOST:-}" ] || python3 -u hack/boskos.py --release - -# The janitor is typically not run as part of the e2e process, but rather -# in a parallel process via a service on the same cluster that runs Prow and -# Boskos. -# -# However, setting JANITOR_ENABLED=1 tells this program to run the janitor -# after the e2e test is executed. -if [ "${JANITOR_ENABLED:-0}" = "1" ]; then - if ! command -v aws-janitor >/dev/null 2>&1; then - echo "skipping janitor; aws-janitor not found" 1>&2 - else - aws-janitor -all -v 2 - fi -else - echo "skipping janitor; JANITOR_ENABLED=${JANITOR_ENABLED:-0}" 1>&2 -fi - -exit "${test_status}" diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml index 231c837340..a22fd6f2f7 100644 --- a/test/e2e/data/e2e_conf.yaml +++ b/test/e2e/data/e2e_conf.yaml @@ -216,7 +216,6 @@ variables: INIT_WITH_KUBERNETES_VERSION: "v1.30.8" EXP_BOOTSTRAP_FORMAT_IGNITION: "true" EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true" - EXP_EXTERNAL_RESOURCE_GC: "true" GC_WORKLOAD: "../../data/gcworkload.yaml" CLASSICELB_TEST_KUBERNETES_VERSION_FROM: "v1.29.9" CLASSICELB_TEST_KUBERNETES_VERSION_TO: "v1.30.8" diff --git a/test/e2e/data/e2e_eks_conf.yaml b/test/e2e/data/e2e_eks_conf.yaml index c3f893afae..ec1769eb02 100644 --- a/test/e2e/data/e2e_eks_conf.yaml +++ b/test/e2e/data/e2e_eks_conf.yaml @@ -133,7 +133,6 @@ variables: CONFORMANCE_CI_ARTIFACTS_KUBERNETES_VERSION: "v1.31.5" IP_FAMILY: "IPv4" CAPA_LOGLEVEL: "4" - EXP_EXTERNAL_RESOURCE_GC: "true" GC_WORKLOAD: "../../data/gcworkload.yaml" intervals: diff --git a/test/e2e/suites/gc_managed/gc_managed_suite_test.go b/test/e2e/suites/gc_managed/gc_managed_suite_test.go deleted file mode 100644 index e5bb7c3de4..0000000000 --- a/test/e2e/suites/gc_managed/gc_managed_suite_test.go +++ /dev/null @@ -1,77 +0,0 @@ -//go:build e2e -// +build e2e - -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gc_managed //nolint:stylecheck - -import ( - "testing" - - "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/klog/v2" - ctrl "sigs.k8s.io/controller-runtime" - - ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2" - expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/v1beta2" - "sigs.k8s.io/cluster-api-provider-aws/v2/test/e2e/shared" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - expclusterv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" -) - -var ( - e2eCtx *shared.E2EContext -) - -func init() { - e2eCtx = shared.NewE2EContext(shared.WithManaged(), shared.WithSchemeInit(initScheme)) - - shared.CreateDefaultFlags(e2eCtx) -} - -func TestE2E(t *testing.T) { - ctrl.SetLogger(klog.Background()) - RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "capa-eks-gc-e2e") -} - -var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { - return shared.Node1BeforeSuite(e2eCtx) -}, func(data []byte) { - shared.AllNodesBeforeSuite(e2eCtx, data) -}) - -var _ = ginkgo.SynchronizedAfterSuite( - func() { - shared.AllNodesAfterSuite(e2eCtx) - }, - func() { - shared.Node1AfterSuite(e2eCtx) - }, -) - -func initScheme() *runtime.Scheme { - sc := shared.DefaultScheme() - _ = expinfrav1.AddToScheme(sc) - _ = clusterv1.AddToScheme(sc) - _ = ekscontrolplanev1.AddToScheme(sc) - _ = expclusterv1.AddToScheme(sc) - - return sc -} diff --git a/test/e2e/suites/gc_unmanaged/gc_unmanaged_suite_test.go b/test/e2e/suites/gc_unmanaged/gc_unmanaged_suite_test.go deleted file mode 100644 index 1beeb1ac6b..0000000000 --- a/test/e2e/suites/gc_unmanaged/gc_unmanaged_suite_test.go +++ /dev/null @@ -1,64 +0,0 @@ -//go:build e2e -// +build e2e - -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gc_unmanaged //nolint:stylecheck - -import ( - "testing" - "time" - - "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "k8s.io/klog/v2" - ctrl "sigs.k8s.io/controller-runtime" - - "sigs.k8s.io/cluster-api-provider-aws/v2/test/e2e/shared" -) - -var ( - e2eCtx *shared.E2EContext -) - -func init() { - e2eCtx = shared.NewE2EContext() - shared.CreateDefaultFlags(e2eCtx) - SetDefaultEventuallyTimeout(20 * time.Minute) - SetDefaultEventuallyPollingInterval(10 * time.Second) -} - -func TestE2E(t *testing.T) { - ctrl.SetLogger(klog.Background()) - RegisterFailHandler(ginkgo.Fail) - ginkgo.RunSpecs(t, "capa-e2e") -} - -var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { - return shared.Node1BeforeSuite(e2eCtx) -}, func(data []byte) { - shared.AllNodesBeforeSuite(e2eCtx, data) -}) - -var _ = ginkgo.SynchronizedAfterSuite( - func() { - shared.AllNodesAfterSuite(e2eCtx) - }, - func() { - shared.Node1AfterSuite(e2eCtx) - }, -) diff --git a/test/e2e/suites/gc_managed/gc_managed_test.go b/test/e2e/suites/managed/gc_test.go similarity index 87% rename from test/e2e/suites/gc_managed/gc_managed_test.go rename to test/e2e/suites/managed/gc_test.go index a39d4a018c..74b4029564 100644 --- a/test/e2e/suites/gc_managed/gc_managed_test.go +++ b/test/e2e/suites/managed/gc_test.go @@ -17,25 +17,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gc_managed //nolint:stylecheck +package managed import ( "context" "fmt" "os" - "path/filepath" "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2" "sigs.k8s.io/cluster-api-provider-aws/v2/test/e2e/shared" - ms "sigs.k8s.io/cluster-api-provider-aws/v2/test/e2e/suites/managed" "sigs.k8s.io/cluster-api/test/framework" - "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" ) @@ -59,18 +55,18 @@ var _ = ginkgo.Describe("[managed] [gc] EKS Cluster external resource GC tests", clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) ginkgo.By("default iam role should exist") - ms.VerifyRoleExistsAndOwned(ekscontrolplanev1.DefaultEKSControlPlaneRole, clusterName, false, e2eCtx.BootstrapUserAWSSession) + VerifyRoleExistsAndOwned(ekscontrolplanev1.DefaultEKSControlPlaneRole, clusterName, false, e2eCtx.BootstrapUserAWSSession) ginkgo.By("should create an EKS control plane") - ms.ManagedClusterSpec(ctx, func() ms.ManagedClusterSpecInput { - return ms.ManagedClusterSpecInput{ + ManagedClusterSpec(ctx, func() ManagedClusterSpecInput { + return ManagedClusterSpecInput{ E2EConfig: e2eCtx.E2EConfig, ConfigClusterFn: defaultConfigCluster, BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, AWSSession: e2eCtx.BootstrapUserAWSSession, Namespace: namespace, ClusterName: clusterName, - Flavour: ms.EKSManagedPoolFlavor, + Flavour: EKSManagedPoolFlavor, ControlPlaneMachineCount: 1, // NOTE: this cannot be zero as clusterctl returns an error WorkerMachineCount: 1, } @@ -85,7 +81,7 @@ var _ = ginkgo.Describe("[managed] [gc] EKS Cluster external resource GC tests", Expect(cluster).NotTo(BeNil(), "couldn't find cluster") ginkgo.By("getting AWSManagedControlPlane") - cp := ms.GetControlPlaneByName(ctx, ms.GetControlPlaneByNameInput{ + cp := GetControlPlaneByName(ctx, GetControlPlaneByNameInput{ Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), Namespace: cluster.Spec.InfrastructureRef.Namespace, Name: cluster.Spec.ControlPlaneRef.Name, @@ -180,18 +176,18 @@ var _ = ginkgo.Describe("[managed] [gc] EKS Cluster external resource GC tests", clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) ginkgo.By("default iam role should exist") - ms.VerifyRoleExistsAndOwned(ekscontrolplanev1.DefaultEKSControlPlaneRole, clusterName, false, e2eCtx.BootstrapUserAWSSession) + VerifyRoleExistsAndOwned(ekscontrolplanev1.DefaultEKSControlPlaneRole, clusterName, false, e2eCtx.BootstrapUserAWSSession) ginkgo.By("should create an EKS control plane") - ms.ManagedClusterSpec(ctx, func() ms.ManagedClusterSpecInput { - return ms.ManagedClusterSpecInput{ + ManagedClusterSpec(ctx, func() ManagedClusterSpecInput { + return ManagedClusterSpecInput{ E2EConfig: e2eCtx.E2EConfig, ConfigClusterFn: defaultConfigCluster, BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, AWSSession: e2eCtx.BootstrapUserAWSSession, Namespace: namespace, ClusterName: clusterName, - Flavour: ms.EKSManagedPoolFlavor, + Flavour: EKSManagedPoolFlavor, ControlPlaneMachineCount: 1, // NOTE: this cannot be zero as clusterctl returns an error WorkerMachineCount: 1, } @@ -206,7 +202,7 @@ var _ = ginkgo.Describe("[managed] [gc] EKS Cluster external resource GC tests", Expect(cluster).NotTo(BeNil(), "couldn't find cluster") ginkgo.By("getting AWSManagedControlPlane") - cp := ms.GetControlPlaneByName(ctx, ms.GetControlPlaneByNameInput{ + cp := GetControlPlaneByName(ctx, GetControlPlaneByNameInput{ Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), Namespace: cluster.Spec.InfrastructureRef.Namespace, Name: cluster.Spec.ControlPlaneRef.Name, @@ -289,19 +285,3 @@ var _ = ginkgo.Describe("[managed] [gc] EKS Cluster external resource GC tests", }, shared.DisableAlternativeGCStrategy, shared.ValidateAlternativeGCStrategyDisabled) }) }) - -// TODO (richardcase): remove this when we merge these tests with the main eks e2e tests. -func defaultConfigCluster(clusterName, namespace string) clusterctl.ConfigClusterInput { - return clusterctl.ConfigClusterInput{ - LogFolder: filepath.Join(e2eCtx.Settings.ArtifactFolder, "clusters", e2eCtx.Environment.BootstrapClusterProxy.GetName()), - ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath, - KubeconfigPath: e2eCtx.Environment.BootstrapClusterProxy.GetKubeconfigPath(), - InfrastructureProvider: "aws", - Flavor: ms.EKSManagedPoolFlavor, - Namespace: namespace, - ClusterName: clusterName, - KubernetesVersion: e2eCtx.E2EConfig.GetVariable(shared.KubernetesVersion), - ControlPlaneMachineCount: ptr.To[int64](1), - WorkerMachineCount: ptr.To[int64](0), - } -} diff --git a/test/e2e/suites/gc_unmanaged/gc_unmanaged_test.go b/test/e2e/suites/unmanaged/gc_test.go similarity index 85% rename from test/e2e/suites/gc_unmanaged/gc_unmanaged_test.go rename to test/e2e/suites/unmanaged/gc_test.go index 2bcc1d0cc8..c5b000c818 100644 --- a/test/e2e/suites/gc_unmanaged/gc_unmanaged_test.go +++ b/test/e2e/suites/unmanaged/gc_test.go @@ -17,13 +17,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gc_unmanaged //nolint:stylecheck +package unmanaged import ( "context" "fmt" "os" - "path/filepath" "github.com/gofrs/flock" "github.com/onsi/ginkgo/v2" @@ -32,8 +31,6 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" "sigs.k8s.io/cluster-api-provider-aws/v2/test/e2e/shared" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" @@ -251,32 +248,3 @@ var _ = ginkgo.Context("[unmanaged] [gc]", func() { }, shared.DisableAlternativeGCStrategy, shared.ValidateAlternativeGCStrategyDisabled) }) }) - -// TODO (richardcase): remove this when we merge these tests with the main eks e2e tests. -func defaultConfigCluster(clusterName, namespace string) clusterctl.ConfigClusterInput { - return clusterctl.ConfigClusterInput{ - LogFolder: filepath.Join(e2eCtx.Settings.ArtifactFolder, "clusters", e2eCtx.Environment.BootstrapClusterProxy.GetName()), - ClusterctlConfigPath: e2eCtx.Environment.ClusterctlConfigPath, - KubeconfigPath: e2eCtx.Environment.BootstrapClusterProxy.GetKubeconfigPath(), - InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, - Flavor: clusterctl.DefaultFlavor, - Namespace: namespace, - ClusterName: clusterName, - KubernetesVersion: e2eCtx.E2EConfig.GetVariable(shared.KubernetesVersion), - ControlPlaneMachineCount: ptr.To[int64](1), - WorkerMachineCount: ptr.To[int64](0), - } -} - -// TODO (richardcase): remove this when we merge these tests with the main eks e2e tests. -func createCluster(ctx context.Context, configCluster clusterctl.ConfigClusterInput, result *clusterctl.ApplyClusterTemplateAndWaitResult) (*clusterv1.Cluster, []*clusterv1.MachineDeployment, *controlplanev1.KubeadmControlPlane) { - clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ - ClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, - ConfigCluster: configCluster, - WaitForClusterIntervals: e2eCtx.E2EConfig.GetIntervals("", "wait-cluster"), - WaitForControlPlaneIntervals: e2eCtx.E2EConfig.GetIntervals("", "wait-control-plane"), - WaitForMachineDeployments: e2eCtx.E2EConfig.GetIntervals("", "wait-worker-nodes"), - }, result) - - return result.Cluster, result.MachineDeployments, result.ControlPlane -} From 6f1e7af54281cf293273d24bd49c1b32ee3e1469 Mon Sep 17 00:00:00 2001 From: Richard Case Date: Mon, 17 Mar 2025 14:36:26 +0000 Subject: [PATCH 2/2] chore: fixing e2e tests Changes to fix the e2e tests for the garbage collection feature after graduating the feature. Signed-off-by: Richard Case --- devbox.lock | 3 +++ docs/book/src/topics/external-resource-gc.md | 2 +- test/e2e/data/gcworkload.yaml | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/devbox.lock b/devbox.lock index b59be55935..a87bff9394 100644 --- a/devbox.lock +++ b/devbox.lock @@ -13,6 +13,9 @@ "resolved": "github:NixOS/nixpkgs/75a52265bda7fd25e06e3a67dee3f0354e73243c#git", "source": "nixpkg" }, + "github:NixOS/nixpkgs/nixpkgs-unstable": { + "resolved": "github:NixOS/nixpkgs/573c650e8a14b2faa0041645ab18aed7e60f0c9a?lastModified=1741865919&narHash=sha256-4thdbnP6dlbdq%2BqZWTsm4ffAwoS8Tiq1YResB%2BRP6WE%3D" + }, "go@1.22": { "last_modified": "2024-12-23T21:10:33Z", "resolved": "github:NixOS/nixpkgs/de1864217bfa9b5845f465e771e0ecb48b30e02d#go_1_22", diff --git a/docs/book/src/topics/external-resource-gc.md b/docs/book/src/topics/external-resource-gc.md index abe8f3ee50..4ebd0379e0 100644 --- a/docs/book/src/topics/external-resource-gc.md +++ b/docs/book/src/topics/external-resource-gc.md @@ -30,7 +30,7 @@ export EXTERNAL_RESOURCE_GC=false clusterctl init --infrastructure aws ``` -> Note: if you disablw this feature **ALL** clusters will be marked as not requiring garbage collection. +> Note: if you disable this feature **ALL** clusters will be marked as not requiring garbage collection. ## Operations diff --git a/test/e2e/data/gcworkload.yaml b/test/e2e/data/gcworkload.yaml index 9c5af54139..ab62414cfa 100644 --- a/test/e2e/data/gcworkload.yaml +++ b/test/e2e/data/gcworkload.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: podinfo + namespace: default spec: minReadySeconds: 3 revisionHistoryLimit: 5 @@ -77,6 +78,7 @@ apiVersion: v1 kind: Service metadata: name: podinfo-elb + namespace: default spec: type: LoadBalancer selector: @@ -95,6 +97,7 @@ apiVersion: v1 kind: Service metadata: name: podinfo-nlb + namespace: default annotations: service.beta.kubernetes.io/aws-load-balancer-type: "nlb" spec: