feat: Add Match Conditions to the validation webhook (#10554)

Co-authored-by: Nathan Fudenberg <[email protected]>

Author: davidjumani

changelog/v1.19.0-beta3/match-conditions.yaml

@@ -0,0 +1,9 @@
+changelog:
+- type: HELM
+  issueLink: https://github.com/k8sgateway/k8sgateway/issues/9828
+  resolvesIssue: false
+  description: >-
+    Adds support for match conditions (defined via Common Expression Language (CEL)) to the validating webhook to allow fine grained request filtering. They can be set via two new helm values :
+    - `gateway.validation.matchConditions` on the Gloo webhook
+    - `gateway.validation.kubeCoreMatchConditions` on the Kube webhook
+    Note that match labels are supported from Kubernetes v1.30+ but need to be enabled in Kubernetes v1.27 to v1.30 via the AdmissionWebhookMatchConditions feature gate.

docs/content/reference/values.txt

@@ -628,6 +628,8 @@
 |gateway.validation.validationServerGrpcMaxSizeBytes|int|104857600|gRPC max message size in bytes for the gloo validation server|
 |gateway.validation.livenessProbeEnabled|bool||Set to true to enable a liveness probe for the gateway (default is false). You must also set the 'Probes' value to true.|
 |gateway.validation.fullEnvoyValidation|bool|false|enable feature which validates all final translated config against envoy Validate mode|
+|gateway.validation.matchConditions[].NAME|interface||Match conditions defined via Common Expression Language (CEL) that should evaluate to true for the Gloo resources webhook to be called. Used for fine-grained request filtering|
+|gateway.validation.kubeCoreMatchConditions[].NAME|interface||Match conditions defined via Common Expression Language (CEL) that should evaluate to true for the Kubernetes core resources webhook to be called. Used for fine-grained request filtering|
 |gateway.certGenJob.image.tag|string|<release_version, ex: 1.2.3>|The image tag for the container.|
 |gateway.certGenJob.image.repository|string|certgen|The image repository (name) for the container.|
 |gateway.certGenJob.image.digest|string||The container image's hash digest (e.g. 'sha256:12345...'), consumed when variant=standard.|

docs/content/static/content/osa_provided.md

@@ -56,6 +56,7 @@ Name|Version|License
 [go.uber.org/zap](https://go.uber.org/zap)|v1.27.0|MIT License
 [x/crypto](https://golang.org/x/crypto)|v0.31.0|BSD 3-clause "New" or "Revised" License
 [x/exp](https://golang.org/x/exp)|v0.0.0-20240719175910-8a7402abbf56|BSD 3-clause "New" or "Revised" License
+[x/mod](https://golang.org/x/mod)|v0.21.0|BSD 3-clause "New" or "Revised" License
 [x/sync](https://golang.org/x/sync)|v0.10.0|BSD 3-clause "New" or "Revised" License
 [x/tools](https://golang.org/x/tools)|v0.24.0|BSD 3-clause "New" or "Revised" License
 [googleapis/api](https://google.golang.org/genproto/googleapis/api)|v0.0.0-20241021214115-324edc3d5d38|Apache License 2.0

go.mod

@@ -102,6 +102,7 @@ require (
 	github.com/stoewer/go-strcase v1.3.0
 	github.com/stretchr/testify v1.9.0
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
+	golang.org/x/mod v0.21.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20241021214115-324edc3d5d38
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38
 	istio.io/api v1.24.0-alpha.0.0.20241106042855-9e26cdd3450a
@@ -319,7 +320,6 @@ require (
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.starlark.net v0.0.0-20231121155337-90ade8b19d09 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
-	golang.org/x/mod v0.21.0 // indirect
 	golang.org/x/net v0.33.0 // indirect
 	golang.org/x/oauth2 v0.23.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect

install/helm/gloo/generate/values.go

@@ -482,20 +482,22 @@ type ServiceAccount struct {
 }
 
 type GatewayValidation struct {
-	Enabled                          *bool    `json:"enabled,omitempty" desc:"enable Gloo Edge API Gateway validation hook (default true)"`
-	AlwaysAcceptResources            *bool    `json:"alwaysAcceptResources,omitempty" desc:"unless this is set this to false in order to ensure validation webhook rejects invalid resources. by default, validation webhook will only log and report metrics for invalid resource admission without rejecting them outright."`
-	AllowWarnings                    *bool    `json:"allowWarnings,omitempty" desc:"set this to false in order to ensure validation webhook rejects resources that would have warning status or rejected status, rather than just rejected."`
-	WarnMissingTlsSecret             *bool    `json:"warnMissingTlsSecret,omitempty" desc:"set this to false in order to treat missing tls secret references as errors, causing validation to fail."`
-	ServerEnabled                    *bool    `json:"serverEnabled,omitempty" desc:"By providing the validation field (parent of this object) the user is implicitly opting into validation. This field allows the user to opt out of the validation server, while still configuring pre-existing fields such as warn_route_short_circuiting and disable_transformation_validation."`
-	DisableTransformationValidation  *bool    `json:"disableTransformationValidation,omitempty" desc:"set this to true to disable transformation validation. This may bring significant performance benefits if using many transformations, at the cost of possibly incorrect transformations being sent to Envoy. When using this value make sure to pre-validate transformations."`
-	WarnRouteShortCircuiting         *bool    `json:"warnRouteShortCircuiting,omitempty" desc:"Write a warning to route resources if validation produced a route ordering warning (defaults to false). By setting to true, this means that Gloo Edge will start assigning warnings to resources that would result in route short-circuiting within a virtual host."`
-	SecretName                       *string  `json:"secretName,omitempty" desc:"Name of the Kubernetes Secret containing TLS certificates used by the validation webhook server. This secret will be created by the certGen Job if the certGen Job is enabled."`
-	FailurePolicy                    *string  `json:"failurePolicy,omitempty" desc:"Specify how to handle unrecognized errors for Gloo resources that are returned from the Gateway validation endpoint. Supported values are 'Ignore' or 'Fail'"`
-	KubeCoreFailurePolicy            *string  `json:"kubeCoreFailurePolicy,omitempty" desc:"Specify how to handle unrecognized errors for Kubernetes core resources that are returned by the Gateway validation endpoint. Currently the [validation webhook](https://github.com/solo-io/gloo/blob/main/install/helm/gloo/templates/5-gateway-validation-webhook-configuration.yaml) is configured to handle errors for Kubernetes secrets and namespaces. Supported values are 'Ignore' or 'Fail'. If you set this value to 'Fail', you cannot modify these core resources if the 'gloo' service is unavailable."`
-	Webhook                          *Webhook `json:"webhook,omitempty" desc:"webhook specific configuration"`
-	ValidationServerGrpcMaxSizeBytes *int     `json:"validationServerGrpcMaxSizeBytes,omitempty" desc:"gRPC max message size in bytes for the gloo validation server"`
-	LivenessProbeEnabled             *bool    `json:"livenessProbeEnabled,omitempty" desc:"Set to true to enable a liveness probe for the gateway (default is false). You must also set the 'Probes' value to true."`
-	FullEnvoyValidation              *bool    `json:"fullEnvoyValidation,omitempty" desc:"enable feature which validates all final translated config against envoy Validate mode"`
+	Enabled                          *bool                    `json:"enabled,omitempty" desc:"enable Gloo Edge API Gateway validation hook (default true)"`
+	AlwaysAcceptResources            *bool                    `json:"alwaysAcceptResources,omitempty" desc:"unless this is set this to false in order to ensure validation webhook rejects invalid resources. by default, validation webhook will only log and report metrics for invalid resource admission without rejecting them outright."`
+	AllowWarnings                    *bool                    `json:"allowWarnings,omitempty" desc:"set this to false in order to ensure validation webhook rejects resources that would have warning status or rejected status, rather than just rejected."`
+	WarnMissingTlsSecret             *bool                    `json:"warnMissingTlsSecret,omitempty" desc:"set this to false in order to treat missing tls secret references as errors, causing validation to fail."`
+	ServerEnabled                    *bool                    `json:"serverEnabled,omitempty" desc:"By providing the validation field (parent of this object) the user is implicitly opting into validation. This field allows the user to opt out of the validation server, while still configuring pre-existing fields such as warn_route_short_circuiting and disable_transformation_validation."`
+	DisableTransformationValidation  *bool                    `json:"disableTransformationValidation,omitempty" desc:"set this to true to disable transformation validation. This may bring significant performance benefits if using many transformations, at the cost of possibly incorrect transformations being sent to Envoy. When using this value make sure to pre-validate transformations."`
+	WarnRouteShortCircuiting         *bool                    `json:"warnRouteShortCircuiting,omitempty" desc:"Write a warning to route resources if validation produced a route ordering warning (defaults to false). By setting to true, this means that Gloo Edge will start assigning warnings to resources that would result in route short-circuiting within a virtual host."`
+	SecretName                       *string                  `json:"secretName,omitempty" desc:"Name of the Kubernetes Secret containing TLS certificates used by the validation webhook server. This secret will be created by the certGen Job if the certGen Job is enabled."`
+	FailurePolicy                    *string                  `json:"failurePolicy,omitempty" desc:"Specify how to handle unrecognized errors for Gloo resources that are returned from the Gateway validation endpoint. Supported values are 'Ignore' or 'Fail'"`
+	KubeCoreFailurePolicy            *string                  `json:"kubeCoreFailurePolicy,omitempty" desc:"Specify how to handle unrecognized errors for Kubernetes core resources that are returned by the Gateway validation endpoint. Currently the [validation webhook](https://github.com/solo-io/gloo/blob/main/install/helm/gloo/templates/5-gateway-validation-webhook-configuration.yaml) is configured to handle errors for Kubernetes secrets and namespaces. Supported values are 'Ignore' or 'Fail'. If you set this value to 'Fail', you cannot modify these core resources if the 'gloo' service is unavailable."`
+	Webhook                          *Webhook                 `json:"webhook,omitempty" desc:"webhook specific configuration"`
+	ValidationServerGrpcMaxSizeBytes *int                     `json:"validationServerGrpcMaxSizeBytes,omitempty" desc:"gRPC max message size in bytes for the gloo validation server"`
+	LivenessProbeEnabled             *bool                    `json:"livenessProbeEnabled,omitempty" desc:"Set to true to enable a liveness probe for the gateway (default is false). You must also set the 'Probes' value to true."`
+	FullEnvoyValidation              *bool                    `json:"fullEnvoyValidation,omitempty" desc:"enable feature which validates all final translated config against envoy Validate mode"`
+	MatchConditions                  []map[string]interface{} `json:"matchConditions,omitempty" desc:"Match conditions defined via Common Expression Language (CEL) that should evaluate to true for the Gloo resources webhook to be called. Used for fine-grained request filtering"`
+	KubeCoreMatchConditions          []map[string]interface{} `json:"kubeCoreMatchConditions,omitempty" desc:"Match conditions defined via Common Expression Language (CEL) that should evaluate to true for the Kubernetes core resources webhook to be called. Used for fine-grained request filtering"`
 }
 
 type Webhook struct {

install/helm/gloo/templates/5-gateway-validation-webhook-configuration.yaml

@@ -60,6 +60,10 @@ specific resources, we will manage the resources that the webhook receives via t
   matchPolicy: Exact
 {{- if .Values.gateway.validation.webhook.timeoutSeconds }}
   timeoutSeconds: {{ .Values.gateway.validation.webhook.timeoutSeconds }}
+{{- end }}
+{{- with .Values.gateway.validation.matchConditions }}
+  matchConditions:
+    {{- toYaml . | nindent 4 }}
 {{- end }}
   admissionReviewVersions:
     - v1beta1 # v1beta1 still live in 1.22 https://github.com/kubernetes/api/blob/release-1.22/admission/v1beta1/types.go#L33
@@ -68,7 +72,7 @@ specific resources, we will manage the resources that the webhook receives via t
 {{- end }} {{- /* if .Values.gateway.validation.failurePolicy */}}
 
 {{/* Webhook for core resources - only render if we need to */}}
-{{- if and 
+{{- if and
   (not (has "*" .Values.gateway.validation.webhook.skipDeleteValidationResources))
   (or (not (has "secrets" .Values.gateway.validation.webhook.skipDeleteValidationResources))
        (not (has "namespaces" .Values.gateway.validation.webhook.skipDeleteValidationResources)))
@@ -99,6 +103,10 @@ specific resources, we will manage the resources that the webhook receives via t
   matchPolicy: Exact
 {{- if .Values.gateway.validation.webhook.timeoutSeconds }}
   timeoutSeconds: {{ .Values.gateway.validation.webhook.timeoutSeconds }}
+{{- end }}
+{{- with .Values.gateway.validation.kubeCoreMatchConditions }}
+  matchConditions:
+    {{- toYaml . | nindent 4 }}
 {{- end }}
   admissionReviewVersions:
     - v1beta1 # v1beta1 still live in 1.22 https://github.com/kubernetes/api/blob/release-1.22/admission/v1beta1/types.go#L33

pkg/utils/kubeutils/kubectl/cli.go

@@ -3,6 +3,7 @@ package kubectl
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -11,13 +12,14 @@ import (
 	"time"
 
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubectl/pkg/cmd/version"
 
 	"github.com/solo-io/gloo/pkg/utils/cmdutils"
+	"github.com/solo-io/gloo/pkg/utils/kubeutils/portforward"
 	"github.com/solo-io/gloo/pkg/utils/requestutils/curl"
 	"github.com/solo-io/k8s-utils/testutils/kube"
 
 	"github.com/avast/retry-go/v4"
-	"github.com/solo-io/gloo/pkg/utils/kubeutils/portforward"
 )
 
 // Cli is a utility for executing `kubectl` commands
@@ -373,3 +375,14 @@ func (c *Cli) GetPodsInNsWithLabel(ctx context.Context, namespace string, label
 	glooPodNames := strings.Fields(glooPodNamesString)
 	return glooPodNames, nil
 }
+
+// Version returns the unmarshalled output of `kubectl version -o json`
+func (c *Cli) Version(ctx context.Context) (version.Version, error) {
+	ver := version.Version{}
+	out, _, err := c.Execute(ctx, "version", "-o", "json")
+	if err != nil {
+		return ver, err
+	}
+	err = json.Unmarshal([]byte(out), &ver)
+	return ver, err
+}

test/kubernetes/e2e/features/validation/split_webhook/suite.go

@@ -13,6 +13,7 @@ import (
 	"github.com/solo-io/gloo/test/kubernetes/testutils/helper"
 	"github.com/solo-io/solo-kit/pkg/api/v1/clients"
 	"github.com/solo-io/solo-kit/pkg/api/v1/resources"
+	"golang.org/x/mod/semver"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/stretchr/testify/suite"
@@ -62,6 +63,8 @@ func (s *testingSuite) SetupSuite() {
 }
 
 func (s *testingSuite) BeforeTest(suiteName, testName string) {
+	s.skipUnsupportedTests(testName)
+
 	// Apply the upgrade values file
 	var err error
 	s.rollback, err = s.testHelper.UpgradeGloo(s.ctx, 600*time.Second, helper.WithExtraArgs([]string{
@@ -99,6 +102,8 @@ func (s *testingSuite) BeforeTest(suiteName, testName string) {
 }
 
 func (s *testingSuite) AfterTest(suiteName, testName string) {
+	s.skipUnsupportedTests(testName)
+
 	// Scale gloo deployment back to original replica count
 	err := s.testInstallation.Actions.Kubectl().Scale(s.ctx, s.testInstallation.Metadata.InstallNamespace, "deployment/gloo", uint(s.glooReplicas))
 	s.Assert().NoError(err, "can scale gloo deployment back to %d", s.glooReplicas)
@@ -161,6 +166,14 @@ func (s *testingSuite) TestKubeFailurePolicyIgnore() {
 	s.testDeleteResource(validation.Secret, true)
 }
 
+func (s *testingSuite) TestGlooFailurePolicyMatchConditions() {
+	s.testDeleteResource(validation.BasicUpstream, true)
+}
+
+func (s *testingSuite) TestKubeFailurePolicyMatchConditions() {
+	s.testDeleteResource(validation.Secret, true)
+}
+
 func (s *testingSuite) testDeleteResource(fileName string, shouldDelete bool) {
 	output, err := s.testInstallation.Actions.Kubectl().DeleteFileWithOutput(s.ctx, fileName, "-n", s.testInstallation.Metadata.InstallNamespace)
 
@@ -172,14 +185,29 @@ func (s *testingSuite) testDeleteResource(fileName string, shouldDelete bool) {
 		s.Assert().Error(err)
 		s.Assert().Contains(output, "Internal error occurred: failed calling webhook")
 	}
+}
 
+func (s *testingSuite) skipUnsupportedTests(testName string) {
+	// Skip the MatchCondition tests as they are supported only in k8s v1.30+
+	if strings.Contains(testName, "MatchConditions") {
+		ver, _ := s.testInstallation.Actions.Kubectl().Version(s.ctx)
+		serverVersion := ver.ServerVersion.GitVersion
+		// This handles scenarios where the server version is invalid or the prior command returns an error
+		// semver.Compare("v1.30.0", "") = 1
+		// semver.Compare("v1.30.0", "v1.28.8") = 1
+		if semver.Compare("v1.30.0", serverVersion) == 1 {
+			s.T().Skip(fmt.Sprintf("Skipping %s as the k8s version %s is below the required version (v1.30.0+)", testName, serverVersion))
+		}
+	}
 }
 
 var upgradeValues = map[string]string{
-	"TestGlooFailurePolicyFail":   validation.GlooFailurePolicyFailValues,
-	"TestKubeFailurePolicyFail":   validation.KubeFailurePolicyFailValues,
-	"TestGlooFailurePolicyIgnore": validation.GlooFailurePolicyIgnoreValues,
-	"TestKubeFailurePolicyIgnore": validation.KubeFailurePolicyIgnoreValues,
+	"TestGlooFailurePolicyFail":            validation.GlooFailurePolicyFailValues,
+	"TestKubeFailurePolicyFail":            validation.KubeFailurePolicyFailValues,
+	"TestGlooFailurePolicyIgnore":          validation.GlooFailurePolicyIgnoreValues,
+	"TestKubeFailurePolicyIgnore":          validation.KubeFailurePolicyIgnoreValues,
+	"TestGlooFailurePolicyMatchConditions": validation.GlooFailurePolicyMatchConditions,
+	"TestKubeFailurePolicyMatchConditions": validation.KubeFailurePolicyMatchConditions,
 }
 
 // These tests create one resource and try to delete it, so don't need lists of resources
@@ -223,9 +251,11 @@ var (
 	}
 
 	manifests = map[string]*testManifest{
-		"TestGlooFailurePolicyFail":   upstreamManifest,
-		"TestGlooFailurePolicyIgnore": upstreamManifest,
-		"TestKubeFailurePolicyFail":   secretManifest,
-		"TestKubeFailurePolicyIgnore": secretManifest,
+		"TestGlooFailurePolicyFail":            upstreamManifest,
+		"TestGlooFailurePolicyIgnore":          upstreamManifest,
+		"TestGlooFailurePolicyMatchConditions": upstreamManifest,
+		"TestKubeFailurePolicyFail":            secretManifest,
+		"TestKubeFailurePolicyIgnore":          secretManifest,
+		"TestKubeFailurePolicyMatchConditions": secretManifest,
 	}
 )

test/kubernetes/e2e/features/validation/testdata/split-webhook/gloo-webhook-failure-policy-match-conditions.yaml

@@ -0,0 +1,8 @@
+gateway:
+  validation:
+    failurePolicy: Fail # For "strict" validation mode, fail the validation if webhook server is not available
+    matchConditions:
+      - name: skip-upstreams
+        expression: '!(request.resource.group == "gloo.solo.io" && request.resource.resource == "upstreams")' # Match non-upstream resources.
+    webhook:
+      skipDeleteValidationResources: []

test/kubernetes/e2e/features/validation/testdata/split-webhook/kube-webhook-failure-policy-match-conditions.yaml

@@ -0,0 +1,8 @@
+gateway:
+  validation:
+    kubeCoreFailurePolicy: Fail # For "strict" validation mode, fail the validation if webhook server is not available
+    kubeCoreMatchConditions:
+      - name: skip-secrets
+        expression: '!(request.resource.group == "" && request.resource.resource == "secrets")' # Match non-secret resources.
+    webhook:
+      skipDeleteValidationResources: []