ci: minimal CI changes for faster tests #13031
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| FROM cgr.dev/chainguard/static | ||
|
|
||
| ARG GOARCH=amd64 | ||
|
|
||
| COPY kgateway-linux-$GOARCH /usr/local/bin/kgateway | ||
|
|
||
| USER 10101 | ||
|
|
||
| ENTRYPOINT ["/usr/local/bin/kgateway"] |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -56,36 +56,43 @@ function create_kind_cluster_or_skip() { | |
| fi | ||
| } | ||
|
|
||
| function create_and_setup() { | ||
| create_kind_cluster_or_skip | ||
|
|
||
| # 5. Apply the Kubernetes Gateway API CRDs | ||
| kubectl apply --server-side -f "https://github.com/kubernetes-sigs/gateway-api/releases/download/$CONFORMANCE_VERSION/$CONFORMANCE_CHANNEL-install.yaml" | ||
|
|
||
| # 6. Apply the Kubernetes Gateway API Inference Extension CRDs | ||
| kubectl apply --kustomize "https://github.com/kubernetes-sigs/gateway-api-inference-extension/config/crd?ref=$GIE_CRD_VERSION" | ||
|
Comment on lines
+62
to
+66
Contributor
There was a problem hiding this comment. nit: remove the 5. / 6. numbering |
||
|
|
||
| . $SCRIPT_DIR/setup-metalllb-on-kind.sh | ||
|
Contributor
There was a problem hiding this comment. should this be only if CONFORMANCE=true, like before?
Contributor
Author
There was a problem hiding this comment. metallb is used for all tests, which all set
Contributor
There was a problem hiding this comment. in that case it would be good to clean up the CONFORMANCE var if it's not used anymore (can be a follow-up) |
||
| } | ||
|
|
||
| # 1. Create a kind cluster (or skip creation if a cluster with name=CLUSTER_NAME already exists) | ||
| # This config is roughly based on: https://kind.sigs.k8s.io/docs/user/ingress/ | ||
| create_and_setup & | ||
| KIND_PID=$! | ||
|
|
||
| if [[ $SKIP_DOCKER == 'true' ]]; then | ||
| # TODO(tim): refactor the Makefile & CI scripts so we're loading local | ||
| # charts to real helm repos, and then we can remove this block. | ||
| echo "SKIP_DOCKER=true, not building images or chart" | ||
| else | ||
| # 2. Make all the docker images and load them to the kind cluster | ||
| if [[ $AGENTGATEWAY == 'true' ]]; then | ||
| # Skip expensive envoy build | ||
| VERSION=$VERSION CLUSTER_NAME=$CLUSTER_NAME make kind-build-and-load-kgateway-agentgateway | ||
| else | ||
| VERSION=$VERSION CLUSTER_NAME=$CLUSTER_NAME make kind-build-and-load | ||
| fi | ||
|
|
||
| # 3. Build the test helm chart, ensuring we have a chart in the `_test` folder | ||
| VERSION=$VERSION make package-kgateway-charts | ||
| fi | ||
|
|
||
| wait "$KIND_PID" | ||
|
Comment on lines
+73
to
+93
There was a problem hiding this comment. The |
||
|
|
||
| # 7. Setup localstack | ||
| if [[ $LOCALSTACK == "true" ]]; then | ||
| echo "Setting up localstack" | ||
| . $SCRIPT_DIR/setup-localstack.sh | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,9 +9,12 @@ import ( | |
| "io/fs" | ||
| "os" | ||
| "path/filepath" | ||
| "testing" | ||
|
|
||
| corev1 "k8s.io/api/core/v1" | ||
| "k8s.io/apimachinery/pkg/labels" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
|
|
||
| "github.com/kgateway-dev/kgateway/v2/pkg/utils/helmutils" | ||
| "github.com/kgateway-dev/kgateway/v2/test/e2e/testutils/actions" | ||
| "github.com/kgateway-dev/kgateway/v2/test/e2e/testutils/assertions" | ||
|
|
@@ -78,7 +81,9 @@ func CreateTestInstallationForCluster( | |
| // between TestInstallation outputs per CI run | ||
| GeneratedFiles: MustGeneratedFiles(installContext.InstallNamespace, clusterContext.Name), | ||
| } | ||
| testutils.Cleanup(t, func() { | ||
| installation.finalize() | ||
| }) | ||
| return installation | ||
| } | ||
|
|
||
|
|
@@ -159,7 +164,7 @@ func (i *TestInstallation) InstallKgatewayCRDsFromLocalChart(ctx context.Context | |
|
|
||
| // Check if we should skip installation if the release already exists (PERSIST_INSTALL or FAIL_FAST_AND_PERSIST mode) | ||
| if testutils.ShouldPersistInstall() || testutils.ShouldFailFastAndPersist() { | ||
| if i.releaseExists(ctx, helmutils.CRDChartName, i.Metadata.InstallNamespace) { | ||
| return | ||
| } | ||
| } | ||
|
|
@@ -185,7 +190,7 @@ func (i *TestInstallation) InstallKgatewayCoreFromLocalChart(ctx context.Context | |
|
|
||
| // Check if we should skip installation if the release already exists (PERSIST_INSTALL or FAIL_FAST_AND_PERSIST mode) | ||
| if testutils.ShouldPersistInstall() || testutils.ShouldFailFastAndPersist() { | ||
| if i.releaseExists(ctx, helmutils.ChartName, i.Metadata.InstallNamespace) { | ||
|
Contributor
There was a problem hiding this comment. can we just update
Contributor
Author
There was a problem hiding this comment. The reason I didn't do this is the
Contributor
There was a problem hiding this comment. hmm, I think we can handle that in a follow-up if needed |
||
| return | ||
| } | ||
| } | ||
|
|
@@ -309,3 +314,16 @@ func MustGeneratedFiles(tmpDirId, clusterId string) GeneratedFiles { | |
| FailureDir: failureDir, | ||
| } | ||
| } | ||
| func (i *TestInstallation) releaseExists(ctx context.Context, releaseName, namespace string) bool { | ||
| l := &corev1.SecretList{} | ||
| if err := i.ClusterContext.Client.List(ctx, l, &client.ListOptions{ | ||
| Namespace: namespace, | ||
| LabelSelector: labels.SelectorFromSet(map[string]string{ | ||
| "owner": "helm", | ||
| "name": releaseName, | ||
| }), | ||
| }); err != nil { | ||
| return false | ||
| } | ||
| return len(l.Items) > 0 | ||
| } | ||
|
Comment on lines
+317
to
+329
There was a problem hiding this comment. The new
Consider adding error logging at minimum, or evaluate if the performance gain justifies replacing the official Helm API. |
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -443,7 +443,7 @@ func (s *BaseTestingSuite) ApplyManifests(testCase *TestCase) { | |||||
| s.TestInstallation.Assertions.EventuallyPodsRunning(s.Ctx, ns, metav1.ListOptions{ | ||||||
| LabelSelector: fmt.Sprintf("%s=%s", defaults.WellKnownAppLabel, name), | ||||||
| // Provide a longer timeout as the pod needs to be pulled and pass HCs | ||||||
| }, time.Second*60, time.Second) | ||||||
|
There was a problem hiding this comment. [nitpick] The polling interval has been reduced from 2 seconds to 1 second. While this may speed up tests that pass quickly, it doubles the polling frequency which could:
Consider whether the performance gain (potentially saving ~1 second per assertion) justifies the increased API load and potential for flakiness.
Suggested change
|
||||||
| } | ||||||
| } | ||||||
|
|
||||||
|
|
||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
"only needs agentgateway" meaning envoy is disabled?