Skip to content

Commit 1a6a2d9

Browse files
arianaw66ryanrolds
arianaw66
and
ryanrolds
authored
[Backport v1.18.x] Fix glooctl check and TestOneWayServerTlsFailsWithoutOneWayTls test flakes (#10622)
Co-authored-by: Ryan Old <[email protected]>
1 parent 3b1b1b2 commit 1a6a2d9

File tree

6 files changed

+47
-22
lines changed

6 files changed

+47
-22
lines changed

‎changelog/v1.18.8/fix_glooctl-check_test-flake.yaml

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
changelog:
2+
- type: NON_USER_FACING
3+
issueLink: https://github.com/solo-io/solo-projects/issues/6619
4+
description: fixes flaky enterprise glooctl check tests
5+
resolvesIssue: false
6+
- type: NON_USER_FACING
7+
description: |
8+
Adjusted timeout in an effort to reduce failures of these tests.
9+
issueLink: https://github.com/solo-io/solo-projects/issues/7685
10+
resolvesIssue: false

‎projects/gloo/cli/pkg/cmd/check/gloo_stats.go

+9-2
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ var (
3434
customGlooDeploymentName = helpers.GlooDeploymentName
3535
)
3636

37-
func ResourcesSyncedOverXds(stats, deploymentName string) bool {
37+
func ResourcesSyncedOverXds(printer printers.P, stats, deploymentName string) bool {
3838
var outOfSyncResources []string
3939
metrics := parseMetrics(stats, []string{glooeTotalEntites, glooeInSyncEntities}, deploymentName)
4040
for metric, val := range metrics {
@@ -51,6 +51,12 @@ func ResourcesSyncedOverXds(stats, deploymentName string) bool {
5151
fmt.Println(resourcesOutOfSyncMessage(outOfSyncResources))
5252
return false
5353
}
54+
55+
if len(metrics) == 0 {
56+
printer.AppendStatus("xds metrics", "No xds metrics to check")
57+
} else {
58+
printer.AppendStatus("xds metrics", "OK")
59+
}
5460
return true
5561
}
5662

@@ -69,6 +75,7 @@ func RateLimitIsConnected(stats string) bool {
6975
}
7076

7177
func checkXdsMetrics(ctx context.Context, printer printers.P, opts *options.Options, deployments *appsv1.DeploymentList) error {
78+
printer.AppendCheck("Checking xds metrics... ")
7279
errMessage := "Problem while checking for gloo xds errors"
7380
if deployments == nil {
7481
fmt.Println("Skipping due to an error in checking deployments")
@@ -103,7 +110,7 @@ func checkXdsMetrics(ctx context.Context, printer printers.P, opts *options.Opti
103110
return fmt.Errorf(err)
104111
}
105112

106-
if !ResourcesSyncedOverXds(stats, customGlooDeploymentName) {
113+
if !ResourcesSyncedOverXds(printer, stats, customGlooDeploymentName) {
107114
fmt.Println(errMessage)
108115
return fmt.Errorf(errMessage)
109116
}

‎projects/gloo/cli/pkg/cmd/check/gloo_stats_test.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
. "github.com/onsi/ginkgo/v2"
55
. "github.com/onsi/gomega"
66
"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/check"
7+
"github.com/solo-io/gloo/projects/gloo/cli/pkg/printers"
78
)
89

910
var _ = Describe("Gloo Stats", func() {
@@ -14,7 +15,7 @@ var _ = Describe("Gloo Stats", func() {
1415

1516
Context("check resources in sync", func() {
1617
It("returns true when there are no stats", func() {
17-
result := check.ResourcesSyncedOverXds("", deploymentName)
18+
result := check.ResourcesSyncedOverXds(printers.P{}, "", deploymentName)
1819
Expect(result).To(BeTrue())
1920
})
2021

@@ -38,7 +39,7 @@ glooe_solo_io_xds_total_entities{resource="type.googleapis.com/envoy.api.v2.List
3839
glooe_solo_io_xds_total_entities{resource="type.googleapis.com/envoy.api.v2.RouteConfiguration"} 1
3940
glooe_solo_io_xds_total_entities{resource="type.googleapis.com/glooe.solo.io.RateLimitConfig"} 1
4041
`
41-
result := check.ResourcesSyncedOverXds(stats, deploymentName)
42+
result := check.ResourcesSyncedOverXds(printers.P{}, stats, deploymentName)
4243
Expect(result).To(BeFalse())
4344
})
4445

@@ -59,7 +60,7 @@ glooe_solo_io_xds_total_entities{resource="type.googleapis.com/envoy.api.v2.List
5960
glooe_solo_io_xds_total_entities{resource="type.googleapis.com/envoy.api.v2.RouteConfiguration"} 1
6061
glooe_solo_io_xds_total_entities{resource="type.googleapis.com/glooe.solo.io.RateLimitConfig"} 1
6162
`
62-
result := check.ResourcesSyncedOverXds(stats, deploymentName)
63+
result := check.ResourcesSyncedOverXds(printers.P{}, stats, deploymentName)
6364
Expect(result).To(BeTrue())
6465
})
6566
})

‎projects/gloo/cli/pkg/cmd/check/root.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -952,7 +952,7 @@ func checkSecrets(ctx context.Context, printer printers.P, _ *options.Options, n
952952
client, err := helpers.GetSecretClient(ctx, namespaces)
953953
if err != nil {
954954
multiErr = multierror.Append(multiErr, err)
955-
printer.AppendStatus("secrets", fmt.Sprintf("%v Errors!", multiErr.Len()))
955+
printer.AppendStatus("Secrets", fmt.Sprintf("%v Errors!", multiErr.Len()))
956956
return multiErr
957957
}
958958

‎test/kubernetes/e2e/features/glooctl/check_suite.go

+21-15
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ import (
1212
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1313
)
1414

15+
// TODO(nfuden): Fix clusterloadassignment issues that forced xds-metrics to be excluded.
16+
// Consider https://github.com/envoyproxy/envoy/issues/7529#issuecomment-1227724217
17+
1518
var _ e2e.NewSuiteFunc = NewCheckSuite
1619

1720
// checkSuite contains the set of tests to validate the behavior of `glooctl check`
@@ -23,9 +26,7 @@ type checkSuite struct {
2326
testInstallation *e2e.TestInstallation
2427
}
2528

26-
// NewChecksuite for glooctl check validation
27-
// TODO(nfuden): Fix clusterloadassignment issues that forced xds-metrics to be excluded.
28-
// Consider https://github.com/envoyproxy/envoy/issues/7529#issuecomment-1227724217
29+
// NewCheckSuite for glooctl check validation
2930
func NewCheckSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite {
3031
return &checkSuite{
3132
ctx: ctx,
@@ -34,6 +35,7 @@ func NewCheckSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.Te
3435
}
3536

3637
func (s *checkSuite) TestCheck() {
38+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
3739
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
3840
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics")
3941
s.NoError(err)
@@ -50,6 +52,7 @@ func (s *checkSuite) TestCheck() {
5052
}
5153

5254
func (s *checkSuite) TestCheckExclude() {
55+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
5356
for excludeKey, expectedOutput := range checkCommonGlooGatewayOutputByKey {
5457
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
5558
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", fmt.Sprintf("xds-metrics,%s", excludeKey))
@@ -68,6 +71,7 @@ func (s *checkSuite) TestCheckExclude() {
6871
}
6972

7073
func (s *checkSuite) TestCheckReadOnly() {
74+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
7175
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
7276
"-n", s.testInstallation.Metadata.InstallNamespace, "--read-only", "-x", "xds-metrics")
7377
s.NoError(err)
@@ -92,11 +96,12 @@ func (s *checkSuite) TestCheckReadOnly() {
9296
func (s *checkSuite) TestCheckKubeContext() {
9397
// When passing an invalid kube-context, `glooctl check` should succeed
9498
_, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
95-
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", "invalid-context")
99+
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", "invalid-context", "-x", "xds-metrics")
96100
s.Error(err)
97101
s.Contains(err.Error(), "Could not get kubernetes client: Error retrieving Kubernetes configuration: context \"invalid-context\" does not exist")
98102

99103
// When passing the kube-context of the running cluster, `glooctl check` should succeed
104+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
100105
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
101106
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
102107
s.NoError(err)
@@ -110,7 +115,7 @@ func (s *checkSuite) TestCheckTimeout() {
110115
s.NoError(err)
111116

112117
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
113-
"-n", s.testInstallation.Metadata.InstallNamespace,
118+
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics",
114119
"-c", shortTimeoutValues.Name())
115120
s.Error(err)
116121
s.Contains(err.Error(), "context deadline exceeded")
@@ -121,38 +126,36 @@ func (s *checkSuite) TestCheckTimeout() {
121126
_, err = normalTimeoutValues.Write([]byte(`checkTimeoutSeconds: 300s`))
122127
s.NoError(err)
123128

124-
// Note: This test does not use `"-x", "xds-metrics"`, so it will require the xds-metrics check to pass with no errors
125-
// In gloo-ee the prometheus metrics must be enabled for the check to pass
129+
// exclude xds-metrics due to Envoy bug: see TODO at top of file
126130
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
127-
"-n", s.testInstallation.Metadata.InstallNamespace,
131+
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics",
128132
"-c", normalTimeoutValues.Name())
129133
s.NoError(err)
130134
}
131135

132136
func (s *checkSuite) TestCheckNamespace() {
133137
// namespace does not exist
134-
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "not-gloo-system")
138+
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "not-gloo-system", "-x", "xds-metrics")
135139
s.Error(err)
136140
s.Contains(output, "Could not communicate with kubernetes cluster: namespaces \"not-gloo-system\" not found")
137141

138142
// gloo not in namespace
139-
output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "default")
143+
output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "default", "-x", "xds-metrics")
140144
s.Error(err)
141145
s.Contains(output, "Warning: The provided label selector (gloo) applies to no pods")
142146

143147
// pod does not exist
144-
// Note: This test does not use `"-x", "xds-metrics"`, so it will require the xds-metrics check to pass with no errors
145-
// In gloo-ee the prometheus metrics must be enabled for the check to pass
148+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
146149
output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
147-
"-n", s.testInstallation.Metadata.InstallNamespace,
150+
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics",
148151
"-p", "not-gloo")
149152
s.NoError(err)
150153
s.Contains(output, "Warning: The provided label selector (not-gloo) applies to no pods")
151154
s.Contains(output, "No problems detected.")
152155

153156
// resource namespace does not exist
154157
output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
155-
"-n", s.testInstallation.Metadata.InstallNamespace,
158+
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics",
156159
"-r", "not-gloo-system")
157160
s.Error(err)
158161
s.Contains(output, fmt.Sprintf("No namespaces specified are currently being watched (defaulting to '%s' namespace)", s.testInstallation.Metadata.InstallNamespace))
@@ -192,6 +195,7 @@ func (s *checkSuite) TestNoGateways() {
192195
Namespace: s.testInstallation.Metadata.InstallNamespace,
193196
}, gomega.Equal(0))
194197

198+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
195199
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
196200
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
197201
s.Error(err)
@@ -219,6 +223,7 @@ func (s *checkSuite) TestEdgeGatewayScaled() {
219223
Namespace: s.testInstallation.Metadata.InstallNamespace,
220224
}, gomega.Equal(0))
221225

226+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
222227
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
223228
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
224229
s.NoError(err)
@@ -252,9 +257,10 @@ func (s *checkSuite) TestEdgeResourceError() {
252257
s.NoError(err)
253258

254259
// Run check. This needs to run in eventually to get all errors reported
260+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
255261
gomega.Eventually(func() error {
256262
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
257-
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext)
263+
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
258264
return err
259265
}, time.Minute*2, time.Second*10).Should(gomega.SatisfyAll(
260266
gomega.MatchError(gomega.ContainSubstring(fmt.Sprintf("* Found rejected virtual service by '%s': default reject-me-too (Reason: 2 errors occurred:", s.testInstallation.Metadata.InstallNamespace))),

‎test/kubernetes/e2e/features/server_tls/k8s_suite.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -131,5 +131,6 @@ func (s *k8sServerTlsTestingSuite) assertEventualError(hostHeaderValue string, c
131131
Container: "curl",
132132
},
133133
append(curlOptions("gloo-proxy-gw", s.ns, hostHeaderValue), curl.WithPath("/status/200")),
134-
code)
134+
code,
135+
time.Minute*2)
135136
}

0 commit comments

Comments
 (0)