Skip to content

Commit 951dfcb

Browse files
danehansdanehans
authored
Refactors TCPRoute e2e tests (#10418)
Signed-off-by: Daneyon Hansen <[email protected]>
1 parent 7a55554 commit 951dfcb

12 files changed

+367
-147
lines changed

‎changelog/v1.18.0-rc3/issue_10414.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
changelog:
2+
- type: FIX
3+
issueLink: https://github.com/solo-io/gloo/issues/10414
4+
resolvesIssue: true
5+
description: >-
6+
Refactors TCPRoute e2e tests to run in parallel, reduce flakes, and
7+
parameterize tests for reusability.

‎test/kubernetes/e2e/features/services/tcproute/suite.go

+189-88
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,24 @@ package tcproute
22

33
import (
44
"context"
5+
"fmt"
6+
"os"
57

68
"github.com/stretchr/testify/suite"
9+
appsv1 "k8s.io/api/apps/v1"
10+
corev1 "k8s.io/api/core/v1"
711
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
812
v1 "sigs.k8s.io/gateway-api/apis/v1"
913

1014
"github.com/solo-io/gloo/pkg/utils/kubeutils"
15+
"github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl"
1116
"github.com/solo-io/gloo/pkg/utils/requestutils/curl"
17+
"github.com/solo-io/gloo/test/gomega/matchers"
1218
"github.com/solo-io/gloo/test/kubernetes/e2e"
1319
"github.com/solo-io/gloo/test/kubernetes/e2e/defaults"
1420
)
1521

16-
// testingSuite is the entire Suite of tests for testing K8s Service-specific features/fixes
22+
// testingSuite is the entire suite of tests for testing K8s Service-specific features/fixes
1723
type testingSuite struct {
1824
suite.Suite
1925

@@ -31,95 +37,190 @@ func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.
3137
}
3238
}
3339

34-
func (s *testingSuite) TestConfigureTCPRouteBackingDestinationsWithSingleService() {
35-
s.T().Cleanup(func() {
36-
err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, singleTcpRouteManifest)
37-
s.NoError(err, "can delete manifest")
38-
err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, multiBackendServiceManifest)
39-
s.NoError(err, "can delete manifest")
40-
err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, singleListenerGatewayAndClientManifest)
41-
s.NoError(err, "can delete manifest")
42-
s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, proxyService, proxyDeployment)
43-
})
44-
45-
err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, singleListenerGatewayAndClientManifest)
46-
s.Assert().NoError(err, "can apply gateway and client manifest")
47-
s.testInstallation.Assertions.EventuallyGatewayCondition(s.ctx, "tcp-gateway", "default", v1.GatewayConditionAccepted, metav1.ConditionTrue, timeout)
48-
49-
err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, multiBackendServiceManifest)
50-
s.Assert().NoError(err, "can apply backend service manifest")
51-
s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxyService, proxyDeployment)
52-
53-
err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, singleTcpRouteManifest)
54-
s.Assert().NoError(err, "can apply tcproute manifest")
55-
s.testInstallation.Assertions.EventuallyTCPRouteCondition(s.ctx, "tcp-app-1", "default", v1.RouteConditionAccepted, metav1.ConditionTrue, timeout)
56-
s.testInstallation.Assertions.EventuallyTCPRouteCondition(s.ctx, "tcp-app-1", "default", v1.RouteConditionResolvedRefs, metav1.ConditionTrue, timeout)
57-
s.testInstallation.Assertions.EventuallyGatewayCondition(s.ctx, "tcp-gateway", "default", v1.GatewayConditionProgrammed, metav1.ConditionTrue, timeout)
58-
s.testInstallation.Assertions.EventuallyGatewayListenerAttachedRoutes(s.ctx, "tcp-gateway", "default", v1.SectionName("foo"), int32(1), timeout)
59-
60-
s.testInstallation.Assertions.AssertEventualCurlResponse(
61-
s.ctx,
62-
defaults.CurlPodExecOpt,
63-
[]curl.Option{
64-
curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)),
65-
curl.WithPort(8088),
66-
},
67-
expectedTcpFooSvcResp)
68-
s.testInstallation.Assertions.AssertEventualCurlResponse(
69-
s.ctx,
70-
defaults.CurlPodExecOpt,
71-
[]curl.Option{
72-
curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)),
73-
curl.WithPort(8088),
74-
},
75-
expectedTcpBarSvcResp)
40+
func (s *testingSuite) SetupSuite() {
41+
var cancel context.CancelFunc
42+
s.ctx, cancel = context.WithTimeout(context.Background(), ctxTimeout)
43+
s.T().Cleanup(cancel)
44+
45+
manifests := []string{
46+
singleSvcNsManifest,
47+
singleSvcGatewayAndClientManifest,
48+
singleSvcBackendManifest,
49+
singleSvcTcpRouteManifest,
50+
multiSvcNsManifest,
51+
multiSvcGatewayAndClientManifest,
52+
multiSvcBackendManifest,
53+
multiSvcTcpRouteManifest,
54+
}
55+
for _, file := range manifests {
56+
s.Require().NoError(validateManifestFile(file), "Invalid manifest file: %s", file)
57+
}
58+
}
59+
60+
type tcpRouteTestCase struct {
61+
name string
62+
nsManifest string
63+
gtwName string
64+
gtwNs string
65+
gtwManifest string
66+
svcManifest string
67+
tcpRouteManifest string
68+
proxyService *corev1.Service
69+
proxyDeployment *appsv1.Deployment
70+
expectedResponses []*matchers.HttpResponse
71+
ports []int
72+
listenerNames []v1.SectionName
73+
expectedRouteCounts []int32
74+
tcpRouteNames []string
7675
}
7776

78-
func (s *testingSuite) TestConfigureTCPRouteBackingDestinationsWithMultiServices() {
79-
s.T().Skip("skipping test until we resolve the reason it is flaky")
80-
81-
s.T().Cleanup(func() {
82-
err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, multiTcpRouteManifest)
83-
s.NoError(err, "can delete manifest")
84-
err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, multiBackendServiceManifest)
85-
s.NoError(err, "can delete manifest")
86-
err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, multiListenerGatewayAndClientManifest)
87-
s.NoError(err, "can delete manifest")
88-
s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, proxyService, proxyDeployment)
89-
})
90-
91-
err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, multiListenerGatewayAndClientManifest)
92-
s.Assert().NoError(err, "can apply gateway and client manifest")
93-
s.testInstallation.Assertions.EventuallyGatewayCondition(s.ctx, "tcp-gateway", "default", v1.GatewayConditionAccepted, metav1.ConditionTrue, timeout)
94-
95-
err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, multiBackendServiceManifest)
96-
s.Assert().NoError(err, "can apply backend service manifest")
97-
s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxyService, proxyDeployment)
98-
99-
err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, multiTcpRouteManifest)
100-
s.Assert().NoError(err, "can apply tcproute manifest")
101-
s.testInstallation.Assertions.EventuallyTCPRouteCondition(s.ctx, "tcp-app-1", "default", v1.RouteConditionAccepted, metav1.ConditionTrue, timeout)
102-
s.testInstallation.Assertions.EventuallyTCPRouteCondition(s.ctx, "tcp-app-1", "default", v1.RouteConditionResolvedRefs, metav1.ConditionTrue, timeout)
103-
s.testInstallation.Assertions.EventuallyTCPRouteCondition(s.ctx, "tcp-app-2", "default", v1.RouteConditionAccepted, metav1.ConditionTrue, timeout)
104-
s.testInstallation.Assertions.EventuallyTCPRouteCondition(s.ctx, "tcp-app-2", "default", v1.RouteConditionResolvedRefs, metav1.ConditionTrue, timeout)
105-
s.testInstallation.Assertions.EventuallyGatewayCondition(s.ctx, "tcp-gateway", "default", v1.GatewayConditionProgrammed, metav1.ConditionTrue, timeout)
106-
s.testInstallation.Assertions.EventuallyGatewayListenerAttachedRoutes(s.ctx, "tcp-gateway", "default", v1.SectionName("foo"), int32(1), timeout)
107-
s.testInstallation.Assertions.EventuallyGatewayListenerAttachedRoutes(s.ctx, "tcp-gateway", "default", v1.SectionName("bar"), int32(1), timeout)
108-
109-
s.testInstallation.Assertions.AssertEventualCurlResponse(
110-
s.ctx,
111-
defaults.CurlPodExecOpt,
112-
[]curl.Option{
113-
curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)),
114-
curl.WithPort(8088),
77+
func (s *testingSuite) TestConfigureTCPRouteBackingDestinations() {
78+
testCases := []tcpRouteTestCase{
79+
{
80+
name: "SingleServiceTCPRoute",
81+
nsManifest: singleSvcNsManifest,
82+
gtwName: singleSvcGatewayName,
83+
gtwNs: singleSvcNsName,
84+
gtwManifest: singleSvcGatewayAndClientManifest,
85+
svcManifest: singleSvcBackendManifest,
86+
tcpRouteManifest: singleSvcTcpRouteManifest,
87+
proxyService: singleSvcProxyService,
88+
proxyDeployment: singleSvcProxyDeployment,
89+
expectedResponses: []*matchers.HttpResponse{
90+
expectedSingleSvcResp,
91+
},
92+
ports: []int{8087},
93+
listenerNames: []v1.SectionName{
94+
v1.SectionName(singleSvcListenerName8087),
95+
},
96+
expectedRouteCounts: []int32{1},
97+
tcpRouteNames: []string{singleSvcTCPRouteName},
11598
},
116-
expectedTcpFooSvcResp)
117-
s.testInstallation.Assertions.AssertEventualCurlResponse(
118-
s.ctx,
119-
defaults.CurlPodExecOpt,
120-
[]curl.Option{
121-
curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)),
122-
curl.WithPort(8089),
99+
{
100+
name: "MultiServicesTCPRoute",
101+
nsManifest: multiSvcNsManifest,
102+
gtwName: multiSvcGatewayName,
103+
gtwNs: multiSvcNsName,
104+
gtwManifest: multiSvcGatewayAndClientManifest,
105+
svcManifest: multiSvcBackendManifest,
106+
tcpRouteManifest: multiSvcTcpRouteManifest,
107+
proxyService: multiProxyService,
108+
proxyDeployment: multiProxyDeployment,
109+
expectedResponses: []*matchers.HttpResponse{
110+
expectedMultiSvc1Resp,
111+
expectedMultiSvc2Resp,
112+
},
113+
ports: []int{8088, 8089},
114+
listenerNames: []v1.SectionName{
115+
v1.SectionName(multiSvcListenerName8088),
116+
v1.SectionName(multiSvcListenerName8089),
117+
},
118+
expectedRouteCounts: []int32{1, 1},
119+
tcpRouteNames: []string{multiSvcTCPRouteName1, multiSvcTCPRouteName2},
123120
},
124-
expectedTcpBarSvcResp)
121+
}
122+
123+
for _, tc := range testCases {
124+
tc := tc // capture range variable
125+
s.Run(tc.name, func() {
126+
// Cleanup function
127+
s.T().Cleanup(func() {
128+
err := s.deleteManifests(tc.nsManifest)
129+
s.Require().NoError(err, fmt.Sprintf("Failed to delete manifest %s", tc.nsManifest))
130+
131+
s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: tc.gtwNs}})
132+
})
133+
134+
// Setup environment
135+
s.setupTestEnvironment(
136+
tc.nsManifest,
137+
tc.gtwName,
138+
tc.gtwNs,
139+
tc.gtwManifest,
140+
tc.svcManifest,
141+
tc.proxyService,
142+
tc.proxyDeployment,
143+
)
144+
145+
// Apply TCPRoute manifest
146+
err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, tc.tcpRouteManifest)
147+
s.Require().NoError(err, fmt.Sprintf("Failed to apply manifest %s", tc.tcpRouteManifest))
148+
149+
// Assert gateway conditions
150+
s.testInstallation.Assertions.EventuallyGatewayCondition(s.ctx, tc.gtwName, tc.gtwNs, v1.GatewayConditionAccepted, metav1.ConditionTrue, timeout)
151+
152+
// Assert TCPRoute conditions
153+
for _, tcpRouteName := range tc.tcpRouteNames {
154+
s.testInstallation.Assertions.EventuallyTCPRouteCondition(s.ctx, tcpRouteName, tc.gtwNs, v1.RouteConditionAccepted, metav1.ConditionTrue, timeout)
155+
s.testInstallation.Assertions.EventuallyTCPRouteCondition(s.ctx, tcpRouteName, tc.gtwNs, v1.RouteConditionResolvedRefs, metav1.ConditionTrue, timeout)
156+
}
157+
158+
// Assert gateway programmed condition
159+
s.testInstallation.Assertions.EventuallyGatewayCondition(s.ctx, tc.gtwName, tc.gtwNs, v1.GatewayConditionProgrammed, metav1.ConditionTrue, timeout)
160+
161+
// Assert listener attached routes
162+
for i, listenerName := range tc.listenerNames {
163+
expectedRouteCount := tc.expectedRouteCounts[i]
164+
s.testInstallation.Assertions.EventuallyGatewayListenerAttachedRoutes(s.ctx, tc.gtwName, tc.gtwNs, listenerName, expectedRouteCount, timeout)
165+
}
166+
167+
// Assert expected responses
168+
for i, port := range tc.ports {
169+
expectedResponse := tc.expectedResponses[i]
170+
s.testInstallation.Assertions.AssertEventualCurlResponse(
171+
s.ctx,
172+
s.execOpts(tc.gtwNs),
173+
[]curl.Option{
174+
curl.WithHost(kubeutils.ServiceFQDN(tc.proxyService.ObjectMeta)),
175+
curl.WithPort(port),
176+
},
177+
expectedResponse)
178+
}
179+
})
180+
}
181+
}
182+
183+
func validateManifestFile(path string) error {
184+
if _, err := os.Stat(path); os.IsNotExist(err) {
185+
return fmt.Errorf("Manifest file not found: %s", path)
186+
}
187+
return nil
188+
}
189+
190+
func (s *testingSuite) setupTestEnvironment(nsManifest, gtwName, gtwNs, gtwManifest, svcManifest string, proxySvc *corev1.Service, proxyDeploy *appsv1.Deployment) {
191+
s.applyManifests(nsManifest)
192+
193+
s.applyManifests(gtwManifest)
194+
s.testInstallation.Assertions.EventuallyGatewayCondition(s.ctx, gtwName, gtwNs, v1.GatewayConditionAccepted, metav1.ConditionTrue, timeout)
195+
196+
s.applyManifests(svcManifest)
197+
s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxySvc, proxyDeploy)
198+
}
199+
200+
func (s *testingSuite) applyManifests(manifests ...string) {
201+
for _, manifest := range manifests {
202+
s.Eventually(func() bool {
203+
err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, manifest)
204+
if err != nil {
205+
s.T().Logf("Retrying apply manifest: %s, error: %v", manifest, err)
206+
return false
207+
}
208+
return true
209+
}, waitTime, tickTime, fmt.Sprintf("Can apply manifest %s", manifest))
210+
}
211+
}
212+
213+
func (s *testingSuite) deleteManifests(manifests ...string) error {
214+
for _, manifest := range manifests {
215+
if err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, manifest); err != nil {
216+
return err
217+
}
218+
}
219+
return nil
220+
}
221+
222+
func (s *testingSuite) execOpts(ns string) kubectl.PodExecOptions {
223+
opts := defaults.CurlPodExecOpt
224+
opts.Namespace = ns
225+
return opts
125226
}

‎test/kubernetes/e2e/features/services/tcproute/testdata/multi-backend-service.yaml

+10-6
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
apiVersion: v1
22
kind: Service
33
metadata:
4-
name: foo
4+
name: multi-svc-1
5+
namespace: multi-tcp-route
56
labels:
6-
app: backend-1
7+
app: multi-svc
78
spec:
89
ports:
910
- name: http
@@ -15,9 +16,10 @@ spec:
1516
apiVersion: v1
1617
kind: Service
1718
metadata:
18-
name: bar
19+
name: multi-svc-2
20+
namespace: multi-tcp-route
1921
labels:
20-
app: backend-2
22+
app: multi-svc
2123
spec:
2224
ports:
2325
- name: http
@@ -30,6 +32,7 @@ apiVersion: apps/v1
3032
kind: Deployment
3133
metadata:
3234
name: backend-1
35+
namespace: multi-tcp-route
3336
spec:
3437
replicas: 1
3538
selector:
@@ -58,12 +61,13 @@ spec:
5861
fieldRef:
5962
fieldPath: metadata.namespace
6063
- name: SERVICE_NAME
61-
value: foo
64+
value: multi-svc-1
6265
---
6366
apiVersion: apps/v1
6467
kind: Deployment
6568
metadata:
6669
name: backend-2
70+
namespace: multi-tcp-route
6771
spec:
6872
replicas: 1
6973
selector:
@@ -92,4 +96,4 @@ spec:
9296
fieldRef:
9397
fieldPath: metadata.namespace
9498
- name: SERVICE_NAME
95-
value: bar
99+
value: multi-svc-2

0 commit comments

Comments
 (0)