diff --git a/tests/e2e-inference-extension/conformance_test.go b/tests/e2e-inference-extension/conformance_test.go
index a67e87982..b6babbbb5 100644
--- a/tests/e2e-inference-extension/conformance_test.go
+++ b/tests/e2e-inference-extension/conformance_test.go
@@ -13,8 +13,8 @@ import (
 
 	"github.com/stretchr/testify/require"
 	gie "sigs.k8s.io/gateway-api-inference-extension/conformance"
+	gieconfig "sigs.k8s.io/gateway-api-inference-extension/conformance/utils/config"
 	v1 "sigs.k8s.io/gateway-api/conformance/apis/v1"
-	"sigs.k8s.io/gateway-api/conformance/utils/config"
 
 	"github.com/envoyproxy/ai-gateway/tests/internal/e2elib"
 )
@@ -35,12 +35,18 @@ func TestGatewayAPIInferenceExtension(t *testing.T) {
 		Version:      "latest",
 	}
 	options.ConformanceProfiles.Insert(gie.GatewayLayerProfileName)
-	defaultTimeoutConfig := config.DefaultTimeoutConfig()
-	defaultTimeoutConfig.HTTPRouteMustHaveCondition = 10 * time.Second
-	defaultTimeoutConfig.HTTPRouteMustNotHaveParents = 10 * time.Second
-	defaultTimeoutConfig.GatewayMustHaveCondition = 10 * time.Second
-	config.SetupTimeoutConfig(&defaultTimeoutConfig)
-	options.TimeoutConfig = defaultTimeoutConfig
+	// Use the inference extension's default timeout config which has appropriate
+	// timeout values for InferencePool operations (e.g., 300s for conditions).
+	inferenceTimeoutConfig := gieconfig.DefaultInferenceExtensionTimeoutConfig()
+	// Add test isolation delay to mitigate accumulated state across test runs.
+	// Root cause: Resources/state accumulate in the kind cluster across consecutive tests,
+	// causing API server slowness and "context deadline exceeded" errors. Delays between
+	// tests help but don't fully solve it - proper fix requires cluster recreation or
+	// improved cleanup between test runs.
+	inferenceTimeoutConfig.TestIsolation = 5 * time.Second
+	// Increase polling interval to reduce API call frequency and pressure on API server
+	inferenceTimeoutConfig.InferencePoolMustHaveConditionInterval = 15 * time.Second
+	options.TimeoutConfig = inferenceTimeoutConfig.TimeoutConfig
 	options.GatewayClassName = "inference-pool"
 	options.SkipTests = []string{}