Skip to content

Commit c0b3907

Browse files
arianaw66arianaw66
authored
[Backport v1.17.x] Fix glooctl check test flake (#10626)
1 parent f9ae8d3 commit c0b3907

File tree

5 files changed

+42
-20
lines changed

5 files changed

+42
-20
lines changed

‎changelog/v1.17.24/fix_glooctl-check_test-flake.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
changelog:
2+
- type: NON_USER_FACING
3+
issueLink: https://github.com/solo-io/solo-projects/issues/6619
4+
description: fixes flaky enterprise glooctl check tests
5+
resolvesIssue: false

‎projects/gloo/cli/pkg/cmd/check/gloo_stats.go

+9-2
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ var (
3434
customGlooDeploymentName = helpers.GlooDeploymentName
3535
)
3636

37-
func ResourcesSyncedOverXds(stats, deploymentName string) bool {
37+
func ResourcesSyncedOverXds(printer printers.P, stats, deploymentName string) bool {
3838
var outOfSyncResources []string
3939
metrics := parseMetrics(stats, []string{glooeTotalEntites, glooeInSyncEntities}, deploymentName)
4040
for metric, val := range metrics {
@@ -51,6 +51,12 @@ func ResourcesSyncedOverXds(stats, deploymentName string) bool {
5151
fmt.Println(resourcesOutOfSyncMessage(outOfSyncResources))
5252
return false
5353
}
54+
55+
if len(metrics) == 0 {
56+
printer.AppendStatus("xds metrics", "No xds metrics to check")
57+
} else {
58+
printer.AppendStatus("xds metrics", "OK")
59+
}
5460
return true
5561
}
5662

@@ -69,6 +75,7 @@ func RateLimitIsConnected(stats string) bool {
6975
}
7076

7177
func checkXdsMetrics(ctx context.Context, printer printers.P, opts *options.Options, deployments *appsv1.DeploymentList) error {
78+
printer.AppendCheck("Checking xds metrics... ")
7279
errMessage := "Problem while checking for gloo xds errors"
7380
if deployments == nil {
7481
fmt.Println("Skipping due to an error in checking deployments")
@@ -103,7 +110,7 @@ func checkXdsMetrics(ctx context.Context, printer printers.P, opts *options.Opti
103110
return fmt.Errorf(err)
104111
}
105112

106-
if !ResourcesSyncedOverXds(stats, customGlooDeploymentName) {
113+
if !ResourcesSyncedOverXds(printer, stats, customGlooDeploymentName) {
107114
fmt.Println(errMessage)
108115
return fmt.Errorf(errMessage)
109116
}

‎projects/gloo/cli/pkg/cmd/check/gloo_stats_test.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
. "github.com/onsi/ginkgo/v2"
55
. "github.com/onsi/gomega"
66
"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/check"
7+
"github.com/solo-io/gloo/projects/gloo/cli/pkg/printers"
78
)
89

910
var _ = Describe("Gloo Stats", func() {
@@ -14,7 +15,7 @@ var _ = Describe("Gloo Stats", func() {
1415

1516
Context("check resources in sync", func() {
1617
It("returns true when there are no stats", func() {
17-
result := check.ResourcesSyncedOverXds("", deploymentName)
18+
result := check.ResourcesSyncedOverXds(printers.P{}, "", deploymentName)
1819
Expect(result).To(BeTrue())
1920
})
2021

@@ -38,7 +39,7 @@ glooe_solo_io_xds_total_entities{resource="type.googleapis.com/envoy.api.v2.List
3839
glooe_solo_io_xds_total_entities{resource="type.googleapis.com/envoy.api.v2.RouteConfiguration"} 1
3940
glooe_solo_io_xds_total_entities{resource="type.googleapis.com/glooe.solo.io.RateLimitConfig"} 1
4041
`
41-
result := check.ResourcesSyncedOverXds(stats, deploymentName)
42+
result := check.ResourcesSyncedOverXds(printers.P{}, stats, deploymentName)
4243
Expect(result).To(BeFalse())
4344
})
4445

@@ -59,7 +60,7 @@ glooe_solo_io_xds_total_entities{resource="type.googleapis.com/envoy.api.v2.List
5960
glooe_solo_io_xds_total_entities{resource="type.googleapis.com/envoy.api.v2.RouteConfiguration"} 1
6061
glooe_solo_io_xds_total_entities{resource="type.googleapis.com/glooe.solo.io.RateLimitConfig"} 1
6162
`
62-
result := check.ResourcesSyncedOverXds(stats, deploymentName)
63+
result := check.ResourcesSyncedOverXds(printers.P{}, stats, deploymentName)
6364
Expect(result).To(BeTrue())
6465
})
6566
})

‎projects/gloo/cli/pkg/cmd/check/root.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -952,7 +952,7 @@ func checkSecrets(ctx context.Context, printer printers.P, _ *options.Options, n
952952
client, err := helpers.GetSecretClient(ctx, namespaces)
953953
if err != nil {
954954
multiErr = multierror.Append(multiErr, err)
955-
printer.AppendStatus("secrets", fmt.Sprintf("%v Errors!", multiErr.Len()))
955+
printer.AppendStatus("Secrets", fmt.Sprintf("%v Errors!", multiErr.Len()))
956956
return multiErr
957957
}
958958

‎test/kubernetes/e2e/features/glooctl/check_suite.go

+23-14
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ import (
1212
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1313
)
1414

15+
// TODO(nfuden): Fix clusterloadassignment issues that forced xds-metrics to be excluded.
16+
// Consider https://github.com/envoyproxy/envoy/issues/7529#issuecomment-1227724217
17+
1518
var _ e2e.NewSuiteFunc = NewCheckSuite
1619

1720
// checkSuite contains the set of tests to validate the behavior of `glooctl check`
@@ -23,6 +26,7 @@ type checkSuite struct {
2326
testInstallation *e2e.TestInstallation
2427
}
2528

29+
// NewCheckSuite for glooctl check validation
2630
func NewCheckSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite {
2731
return &checkSuite{
2832
ctx: ctx,
@@ -31,6 +35,7 @@ func NewCheckSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.Te
3135
}
3236

3337
func (s *checkSuite) TestCheck() {
38+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
3439
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
3540
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics")
3641
s.NoError(err)
@@ -47,6 +52,7 @@ func (s *checkSuite) TestCheck() {
4752
}
4853

4954
func (s *checkSuite) TestCheckExclude() {
55+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
5056
for excludeKey, expectedOutput := range checkCommonGlooGatewayOutputByKey {
5157
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
5258
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", fmt.Sprintf("xds-metrics,%s", excludeKey))
@@ -65,8 +71,9 @@ func (s *checkSuite) TestCheckExclude() {
6571
}
6672

6773
func (s *checkSuite) TestCheckReadOnly() {
74+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
6875
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
69-
"-n", s.testInstallation.Metadata.InstallNamespace, "--read-only")
76+
"-n", s.testInstallation.Metadata.InstallNamespace, "--read-only", "-x", "xds-metrics")
7077
s.NoError(err)
7178

7279
for _, expectedOutput := range checkCommonGlooGatewayOutputByKey {
@@ -89,13 +96,14 @@ func (s *checkSuite) TestCheckReadOnly() {
8996
func (s *checkSuite) TestCheckKubeContext() {
9097
// When passing an invalid kube-context, `glooctl check` should succeed
9198
_, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
92-
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", "invalid-context")
99+
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", "invalid-context", "-x", "xds-metrics")
93100
s.Error(err)
94101
s.Contains(err.Error(), "Could not get kubernetes client: Error retrieving Kubernetes configuration: context \"invalid-context\" does not exist")
95102

96103
// When passing the kube-context of the running cluster, `glooctl check` should succeed
104+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
97105
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
98-
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext)
106+
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
99107
s.NoError(err)
100108
}
101109

@@ -107,7 +115,7 @@ func (s *checkSuite) TestCheckTimeout() {
107115
s.NoError(err)
108116

109117
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
110-
"-n", s.testInstallation.Metadata.InstallNamespace,
118+
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics",
111119
"-c", shortTimeoutValues.Name())
112120
s.Error(err)
113121
s.Contains(err.Error(), "context deadline exceeded")
@@ -118,38 +126,36 @@ func (s *checkSuite) TestCheckTimeout() {
118126
_, err = normalTimeoutValues.Write([]byte(`checkTimeoutSeconds: 300s`))
119127
s.NoError(err)
120128

121-
// Note: This test does not use `"-x", "xds-metrics"`, so it will require the xds-metrics check to pass with no errors
122-
// In gloo-ee the prometheus metrics must be enabled for the check to pass
129+
// exclude xds-metrics due to Envoy bug: see TODO at top of file
123130
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
124-
"-n", s.testInstallation.Metadata.InstallNamespace,
131+
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics",
125132
"-c", normalTimeoutValues.Name())
126133
s.NoError(err)
127134
}
128135

129136
func (s *checkSuite) TestCheckNamespace() {
130137
// namespace does not exist
131-
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "not-gloo-system")
138+
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "not-gloo-system", "-x", "xds-metrics")
132139
s.Error(err)
133140
s.Contains(output, "Could not communicate with kubernetes cluster: namespaces \"not-gloo-system\" not found")
134141

135142
// gloo not in namespace
136-
output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "default")
143+
output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx, "-n", "default", "-x", "xds-metrics")
137144
s.Error(err)
138145
s.Contains(output, "Warning: The provided label selector (gloo) applies to no pods")
139146

140147
// pod does not exist
141-
// Note: This test does not use `"-x", "xds-metrics"`, so it will require the xds-metrics check to pass with no errors
142-
// In gloo-ee the prometheus metrics must be enabled for the check to pass
148+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
143149
output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
144-
"-n", s.testInstallation.Metadata.InstallNamespace,
150+
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics",
145151
"-p", "not-gloo")
146152
s.NoError(err)
147153
s.Contains(output, "Warning: The provided label selector (not-gloo) applies to no pods")
148154
s.Contains(output, "No problems detected.")
149155

150156
// resource namespace does not exist
151157
output, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
152-
"-n", s.testInstallation.Metadata.InstallNamespace,
158+
"-n", s.testInstallation.Metadata.InstallNamespace, "-x", "xds-metrics",
153159
"-r", "not-gloo-system")
154160
s.Error(err)
155161
s.Contains(output, fmt.Sprintf("No namespaces specified are currently being watched (defaulting to '%s' namespace)", s.testInstallation.Metadata.InstallNamespace))
@@ -189,6 +195,7 @@ func (s *checkSuite) TestNoGateways() {
189195
Namespace: s.testInstallation.Metadata.InstallNamespace,
190196
}, gomega.Equal(0))
191197

198+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
192199
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
193200
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
194201
s.Error(err)
@@ -216,6 +223,7 @@ func (s *checkSuite) TestEdgeGatewayScaled() {
216223
Namespace: s.testInstallation.Metadata.InstallNamespace,
217224
}, gomega.Equal(0))
218225

226+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
219227
output, err := s.testInstallation.Actions.Glooctl().Check(s.ctx,
220228
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
221229
s.NoError(err)
@@ -249,9 +257,10 @@ func (s *checkSuite) TestEdgeResourceError() {
249257
s.NoError(err)
250258

251259
// Run check. This needs to run in eventually to get all errors reported
260+
// exclude xds-metrics check due to Envoy bug: see TODO at top of file
252261
gomega.Eventually(func() error {
253262
_, err = s.testInstallation.Actions.Glooctl().Check(s.ctx,
254-
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext)
263+
"-n", s.testInstallation.Metadata.InstallNamespace, "--kube-context", s.testInstallation.ClusterContext.KubeContext, "-x", "xds-metrics")
255264
return err
256265
}, time.Minute*2, time.Second*10).Should(gomega.SatisfyAll(
257266
gomega.MatchError(gomega.ContainSubstring(fmt.Sprintf("* Found rejected virtual service by '%s': default reject-me-too (Reason: 2 errors occurred:", s.testInstallation.Metadata.InstallNamespace))),

0 commit comments

Comments
 (0)