e2e: add separate e2e target for infext (#1054)

Author: mathetake

.github/codecov.yml

@@ -17,6 +17,7 @@ ignore:
   - "**/*.pb.go"
   - "**/zz_generated.*.go"
   - "tests/internal/envtest.go"
+  - "tests/internal/e2elib/**/*"
   - "tests/internal/testenvironment/**/*"
   - "tests/internal/testopeninference/openai_proxy.go"
   - "internal/testing/**/*"

.github/workflows/build_and_test.yaml

@@ -222,6 +222,38 @@ jobs:
           TEST_GEMINI_API_KEY: ${{ secrets.ENVOY_AI_GATEWAY_GEMINI_API_KEY }}
         run: make test-e2e
 
+  test_e2e_inference_extension:
+    name: E2E Test for Inference Extensions (Envoy Gateway ${{ matrix.name }})
+    # TODO: make it possible to run this job on macOS as well, which is a bit tricky due to the nested
+    # virtualization is not supported on macOS runners.
+    # E.g. Use https://github.com/douglascamata/setup-docker-macos-action  per the comment in
+    # https://github.com/actions/runner-images/issues/17#issuecomment-1971073406
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: latest
+            envoy_gateway_version: v0.0.0-latest
+          - name: v1.5.0
+            envoy_gateway_version: v1.5.0
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          cache: false
+          go-version-file: go.mod
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/.cache/golangci-lint
+            ~/go/pkg/mod
+            ~/go/bin
+          key: e2e-test-${{ hashFiles('**/go.mod', '**/go.sum', '**/Makefile') }}
+      - uses: docker/setup-buildx-action@v3
+      - run: make test-e2e-inference-extension
+
   docker_push:
     # Docker builds are verified in test_e2e job, so we only need to push the images when the event is a push event.
     if: github.event_name == 'push'

.testcoverage.yml

@@ -18,6 +18,7 @@ exclude:
     - zz_generated.deepcopy.go
     # They are test-only libraries.
     - tests/internal/envtest.go
+    - tests/internal/e2elib/
     - tests/internal/testenvironment/
     - tests/internal/testopeninference/openai_proxy.go
     - internal/testing/

Makefile

@@ -172,6 +172,12 @@ test-e2e: build-e2e ## Run the end-to-end tests with a local kind cluster.
 	@echo "Run E2E tests"
 	@go test -v ./tests/e2e/... $(GO_TEST_ARGS) $(GO_TEST_E2E_ARGS)
 
+# This runs the end-to-end tests for the controller and extproc with a local kind cluster.
+.PHONY: test-e2e-inference-extension
+test-e2e-inference-extension: build-e2e ## Run the end-to-end tests with a local kind cluster for Gateway API Inference Extension.
+	@echo "Run E2E tests for inference extension"
+	@go test -v ./tests/e2e-inference-extension/... $(GO_TEST_ARGS) $(GO_TEST_E2E_ARGS)
+
 ##@ Common
 
 # This builds a binary for the given command under the internal/cmd directory.

tests/e2e/conformance_test.go renamed to tests/e2e-inference-extension/conformance_test.go

@@ -13,11 +13,13 @@ import (
 	gie "sigs.k8s.io/gateway-api-inference-extension/conformance"
 	v1 "sigs.k8s.io/gateway-api/conformance/apis/v1"
 	"sigs.k8s.io/gateway-api/conformance/utils/config"
+
+	"github.com/envoyproxy/ai-gateway/tests/internal/e2elib"
 )
 
 func TestGatewayAPIInferenceExtension(t *testing.T) {
 	const manifest = "testdata/inference-extension-conformance.yaml"
-	require.NoError(t, kubectlApplyManifest(t.Context(), manifest))
+	require.NoError(t, e2elib.KubectlApplyManifest(t.Context(), manifest))
 
 	options := gie.DefaultOptions(t)
 	options.ReportOutputPath = "./inference-extension-conformance-test-report.yaml"

tests/e2e-inference-extension/e2e_inference_extension_test.go

@@ -0,0 +1,16 @@
+// Copyright Envoy AI Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package e2e
+
+import (
+	"testing"
+
+	"github.com/envoyproxy/ai-gateway/tests/internal/e2elib"
+)
+
+func TestMain(m *testing.M) {
+	e2elib.TestMain(m, nil, true)
+}

tests/e2e/inference_pool_test.go renamed to tests/e2e-inference-extension/inference_pool_test.go

@@ -19,20 +19,22 @@ import (
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	gwaiev1a2 "sigs.k8s.io/gateway-api-inference-extension/api/v1alpha2"
+
+	"github.com/envoyproxy/ai-gateway/tests/internal/e2elib"
 )
 
 // TestInferencePoolIntegration tests the InferencePool integration with AI Gateway.
 func TestInferencePoolIntegration(t *testing.T) {
 	// Apply the base test manifest.
 	const baseManifest = "../../examples/inference-pool/base.yaml"
-	require.NoError(t, kubectlApplyManifest(t.Context(), baseManifest))
+	require.NoError(t, e2elib.KubectlApplyManifest(t.Context(), baseManifest))
 
 	// Test inferencePool with AIGatewayRoute.
 	const aiGWRouteManifest = "../../examples/inference-pool/aigwroute.yaml"
-	require.NoError(t, kubectlApplyManifest(t.Context(), aiGWRouteManifest))
+	require.NoError(t, e2elib.KubectlApplyManifest(t.Context(), aiGWRouteManifest))
 
 	egSelector := "gateway.envoyproxy.io/owning-gateway-name=inference-pool-with-aigwroute"
-	requireWaitForGatewayPodReady(t, egSelector)
+	e2elib.RequireWaitForGatewayPodReady(t, egSelector)
 
 	// Verify InferencePool status is correctly set for the Gateway.
 	t.Run("verify_inference_pool_status", func(t *testing.T) {
@@ -93,15 +95,15 @@ func TestInferencePoolIntegration(t *testing.T) {
 	})
 
 	t.Cleanup(func() {
-		_ = kubectlDeleteManifest(context.Background(), aiGWRouteManifest)
+		_ = e2elib.KubectlDeleteManifest(context.Background(), aiGWRouteManifest)
 	})
 
 	// Test inferencePool with HTTPRoute.
 	const httpRouteManifest = "../../examples/inference-pool/httproute.yaml"
-	require.NoError(t, kubectlApplyManifest(t.Context(), httpRouteManifest))
+	require.NoError(t, e2elib.KubectlApplyManifest(t.Context(), httpRouteManifest))
 
 	egSelector = "gateway.envoyproxy.io/owning-gateway-name=inference-pool-with-httproute"
-	requireWaitForPodReady(t, egSelector)
+	e2elib.RequireWaitForPodReady(t, egSelector)
 
 	// Verify InferencePool status is correctly set for the HTTPRoute Gateway.
 	t.Run("verify_inference_pool_status_httproute", func(t *testing.T) {
@@ -116,7 +118,7 @@ func TestInferencePoolIntegration(t *testing.T) {
 	})
 
 	t.Cleanup(func() {
-		_ = kubectlDeleteManifest(context.Background(), httpRouteManifest)
+		_ = e2elib.KubectlDeleteManifest(context.Background(), httpRouteManifest)
 	})
 }
 
@@ -130,16 +132,16 @@ func testInferenceGatewayConnectivityByModel(t *testing.T, egSelector, model str
 // testInferenceGatewayConnectivity tests that the InferenceGateway is working as expected and returns a expected status code.
 func testInferenceGatewayConnectivity(t *testing.T, egSelector, body string, additionalHeaders map[string]string, expectedStatusCode int) {
 	require.Eventually(t, func() bool {
-		fwd := requireNewHTTPPortForwarder(t, egNamespace, egSelector, egDefaultServicePort)
-		defer fwd.kill()
+		fwd := e2elib.RequireNewHTTPPortForwarder(t, e2elib.EnvoyGatewayNamespace, egSelector, e2elib.EnvoyGatewayDefaultServicePort)
+		defer fwd.Kill()
 
 		// Set timeout context.
 		ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
 		defer cancel()
 		// Create a request to the InferencePool backend with the correct model header.
 		requestBody := body
 		t.Logf("Request body: %s", requestBody)
-		req, err := http.NewRequestWithContext(ctx, http.MethodPost, fwd.address()+"/v1/chat/completions", strings.NewReader(requestBody))
+		req, err := http.NewRequestWithContext(ctx, http.MethodPost, fwd.Address()+"/v1/chat/completions", strings.NewReader(requestBody))
 		require.NoError(t, err)
 		// Set required headers for InferencePool routing.
 		req.Header.Set("Content-Type", "application/json")

tests/e2e/testdata/inference-extension-conformance.yaml renamed to tests/e2e-inference-extension/testdata/inference-extension-conformance.yaml

@@ -17,19 +17,20 @@ import (
 	"github.com/stretchr/testify/require"
 
 	internaltesting "github.com/envoyproxy/ai-gateway/internal/testing"
+	"github.com/envoyproxy/ai-gateway/tests/internal/e2elib"
 )
 
 // TestExamplesBasic tests the basic example in examples/basic directory.
 func Test_Examples_Basic(t *testing.T) {
 	const manifestDir = "../../examples/basic"
 	const manifest = manifestDir + "/basic.yaml"
-	require.NoError(t, kubectlApplyManifest(t.Context(), manifest))
+	require.NoError(t, e2elib.KubectlApplyManifest(t.Context(), manifest))
 
 	const egSelector = "gateway.envoyproxy.io/owning-gateway-name=envoy-ai-gateway-basic"
-	requireWaitForGatewayPodReady(t, egSelector)
+	e2elib.RequireWaitForGatewayPodReady(t, egSelector)
 
 	testUpstreamCase := examplesBasicTestCase{name: "testupsream", modelName: "some-cool-self-hosted-model"}
-	testUpstreamCase.run(t, egNamespace, egSelector)
+	testUpstreamCase.run(t, egSelector)
 
 	// This requires the following environment variables to be set:
 	//   - TEST_AWS_ACCESS_KEY_ID
@@ -43,20 +44,20 @@ func Test_Examples_Basic(t *testing.T) {
 		// Replace the placeholders with the actual credentials and apply the manifests.
 		openAIManifest, err := os.ReadFile(manifestDir + "/openai.yaml")
 		require.NoError(t, err)
-		require.NoError(t, kubectlApplyManifestStdin(t.Context(), strings.ReplaceAll(string(openAIManifest), "OPENAI_API_KEY", cc.OpenAIAPIKey)))
+		require.NoError(t, e2elib.KubectlApplyManifestStdin(t.Context(), strings.ReplaceAll(string(openAIManifest), "OPENAI_API_KEY", cc.OpenAIAPIKey)))
 		awsManifest, err := os.ReadFile(manifestDir + "/aws.yaml")
 		require.NoError(t, err)
 		awsManifestReplaced := strings.ReplaceAll(string(awsManifest), "AWS_ACCESS_KEY_ID", cc.AWSAccessKeyID)
 		awsManifestReplaced = strings.ReplaceAll(awsManifestReplaced, "AWS_SECRET_ACCESS_KEY", cc.AWSSecretAccessKey)
-		require.NoError(t, kubectlApplyManifestStdin(t.Context(), awsManifestReplaced))
+		require.NoError(t, e2elib.KubectlApplyManifestStdin(t.Context(), awsManifestReplaced))
 
 		time.Sleep(5 * time.Second) // At least 5 seconds for the updated secret to be propagated.
 
 		for _, tc := range []examplesBasicTestCase{
 			{name: "openai", modelName: "gpt-4o-mini", skip: !cc.OpenAIValid},
 			{name: "aws", modelName: "us.meta.llama3-2-1b-instruct-v1:0", skip: !cc.AWSValid},
 		} {
-			tc.run(t, egNamespace, egSelector)
+			tc.run(t, egSelector)
 		}
 	})
 }
@@ -67,19 +68,19 @@ type examplesBasicTestCase struct {
 	skip      bool
 }
 
-func (tc examplesBasicTestCase) run(t *testing.T, egNamespace, egSelector string) {
+func (tc examplesBasicTestCase) run(t *testing.T, egSelector string) {
 	t.Run(tc.name, func(t *testing.T) {
 		if tc.skip {
 			t.Skip("skipped due to missing credentials")
 		}
 		require.Eventually(t, func() bool {
-			fwd := requireNewHTTPPortForwarder(t, egNamespace, egSelector, egDefaultServicePort)
-			defer fwd.kill()
+			fwd := e2elib.RequireNewHTTPPortForwarder(t, e2elib.EnvoyGatewayNamespace, egSelector, e2elib.EnvoyGatewayDefaultServicePort)
+			defer fwd.Kill()
 
 			ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
 			defer cancel()
 
-			client := openai.NewClient(option.WithBaseURL(fwd.address() + "/v1/"))
+			client := openai.NewClient(option.WithBaseURL(fwd.Address() + "/v1/"))
 
 			chatCompletion, err := client.Chat.Completions.New(ctx, openai.ChatCompletionNewParams{
 				Messages: []openai.ChatCompletionMessageParamUnion{