feat: support http2 and h2c app protocol (#10687)

Author: davidjumani

Makefile

@@ -1243,7 +1243,7 @@ $(TEST_ASSET_DIR)/conformance/conformance_test.go:
 	cat $(shell go list -json -m sigs.k8s.io/gateway-api | jq -r '.Dir')/conformance/conformance_test.go >> $@
 	go fmt $@
 
-CONFORMANCE_SUPPORTED_FEATURES ?= -supported-features=Gateway,ReferenceGrant,HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRouteResponseHeaderModification,HTTPRoutePortRedirect,HTTPRouteHostRewrite,HTTPRouteSchemeRedirect,HTTPRoutePathRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,HTTPRouteRequestMirror,TLSRoute
+CONFORMANCE_SUPPORTED_FEATURES ?= -supported-features=Gateway,ReferenceGrant,HTTPRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRouteResponseHeaderModification,HTTPRoutePortRedirect,HTTPRouteHostRewrite,HTTPRouteSchemeRedirect,HTTPRoutePathRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,HTTPRouteRequestMirror,TLSRoute,HTTPRouteBackendProtocolH2C
 CONFORMANCE_SUPPORTED_PROFILES ?= -conformance-profiles=GATEWAY-HTTP
 CONFORMANCE_REPORT_ARGS ?= -report-output=$(TEST_ASSET_DIR)/conformance/$(VERSION)-report.yaml -organization=solo.io -project=gloo-gateway -version=$(VERSION) -url=github.com/solo-io/gloo -contact=github.com/solo-io/gloo/issues/new/choose
 CONFORMANCE_ARGS := -gateway-class=gloo-gateway $(CONFORMANCE_SUPPORTED_FEATURES) $(CONFORMANCE_SUPPORTED_PROFILES) $(CONFORMANCE_REPORT_ARGS)

changelog/v1.19.0-beta13/support-approto.yaml

@@ -0,0 +1,6 @@
+changelog:
+- type: NEW_FEATURE
+  issueLink: https://github.com/solo-io/solo-projects/issues/7824
+  resolvesIssue: false
+  description: Adds support for http2 via the service port appProtocol spec
+

projects/gloo/pkg/plugins/kubernetes/serviceconverter/use_http2_annotation_converter.go

@@ -19,6 +19,15 @@ var http2PortNames = []string{
 	"http2",
 }
 
+var http2AppProtocolNames = map[string]bool{
+	// Defined by istio : https://istio.io/latest/docs/ops/configuration/traffic-management/protocol-selection/
+	"http2":    true,
+	"grpc":     true,
+	"grpc-web": true,
+	// Defined by GEP-1911 : https://gateway-api.sigs.k8s.io/geps/gep-1911/#api-semantics
+	"kubernetes.io/h2c": true,
+}
+
 // UseHttp2Converter sets UseHttp2 on the upstream if:
 // (1) the service has the "h2_service" annotation; or
 // (2) the "h2_service" annotation defined in Settings.UpstreamOptions; or
@@ -46,6 +55,10 @@ func useHttp2(ctx context.Context, svc *corev1.Service, port corev1.ServicePort)
 		}
 	}
 
+	if port.AppProtocol != nil && http2AppProtocolNames[*port.AppProtocol] {
+		return &wrappers.BoolValue{Value: true}
+	}
+
 	for _, http2Name := range http2PortNames {
 		if strings.HasPrefix(port.Name, http2Name) {
 			return &wrappers.BoolValue{Value: true}

projects/gloo/pkg/plugins/kubernetes/uds_convert_test.go

@@ -101,6 +101,29 @@ var _ = Describe("UdsConvert", func() {
 			Entry("exactly http2", "http2"),
 		)
 
+		DescribeTable("should create upstream with use_http2=true when port appProtocol is a supported type", func(appProtocol string, useHttp2 bool) {
+			svc := &corev1.Service{
+				Spec: corev1.ServiceSpec{},
+			}
+			svc.Name = "test"
+			svc.Namespace = "test-ns"
+
+			port := corev1.ServicePort{
+				Port:        123,
+				AppProtocol: &appProtocol,
+			}
+			up := uc.CreateUpstream(context.TODO(), svc, port)
+			Expect(up.GetUseHttp2().GetValue()).To(Equal(useHttp2))
+		},
+			Entry("http2", "http2", true),
+			Entry("grpc", "grpc", true),
+			Entry("grpc-web", "grpc-web", true),
+			Entry("kubernetes.io/h2c", "kubernetes.io/h2c", true),
+			Entry("http2-suffix", "http2-suffix", false),
+			Entry("grpc-suffix", "grpc-suffix", false),
+			Entry("tcp", "tcp", false),
+		)
+
 		Describe("Upstream Config when Annotations Exist", func() {
 
 			It("Should create upstream with use_http2=true when annotation exists", testSetUseHttp2Converter)

test/gomega/matchers/have_http_response.go

@@ -74,6 +74,9 @@ type HttpResponse struct {
 	// Each header can be of type: {string, GomegaMatcher}
 	// Optional: If not provided, does not perform header validation
 	Headers map[string]interface{}
+	// Protocol is the expected protocol of an http.Response
+	// Optional: If not provided, does not perform additional validation
+	Protocol string
 	// Custom is a generic matcher that can be applied to validate any other properties of an http.Response
 	// Optional: If not provided, does not perform additional validation
 	Custom types.GomegaMatcher
@@ -90,8 +93,8 @@ func (r *HttpResponse) String() string {
 		bodyString = fmt.Sprintf("%#v", bodyMatcher)
 	}
 
-	return fmt.Sprintf("HttpResponse{StatusCode: %d, Body: %s, Headers: %v, Custom: %v}",
-		r.StatusCode, bodyString, r.Headers, r.Custom)
+	return fmt.Sprintf("HttpResponse{StatusCode: %d, Body: %s, Headers: %v, Protocol: %s, Custom: %v}",
+		r.StatusCode, bodyString, r.Headers, r.Protocol, r.Custom)
 
 }
 
@@ -116,6 +119,9 @@ func HaveHttpResponse(expected *HttpResponse) types.GomegaMatcher {
 			Expected: expected.Body,
 		})
 	}
+	if expected.Protocol != "" {
+		partialResponseMatchers = append(partialResponseMatchers, HaveProtocol(expected.Protocol))
+	}
 	for headerName, headerMatch := range expected.Headers {
 		partialResponseMatchers = append(partialResponseMatchers, &matchers.HaveHTTPHeaderWithValueMatcher{
 			Header: headerName,

test/gomega/matchers/have_protocol.go

@@ -0,0 +1,33 @@
+package matchers
+
+import (
+	"github.com/onsi/gomega"
+	"github.com/onsi/gomega/gstruct"
+	"github.com/onsi/gomega/types"
+	"github.com/solo-io/gloo/test/gomega/transforms"
+)
+
+const (
+	HTTP1Protocol = "HTTP/1.1"
+	HTTP2Protocol = "HTTP/2"
+)
+
+// HaveProtocol expects an http response with the given protocol
+func HaveProtocol(protocol string) types.GomegaMatcher {
+	if protocol == "" {
+		// If protocol is not defined, we create a matcher that always succeeds
+		return gstruct.Ignore()
+	}
+	//nolint:bodyclose // The caller of this matcher constructor should be responsible for ensuring the body close
+	return gomega.WithTransform(transforms.WithProtocol(), gomega.Equal(protocol))
+}
+
+// HaveHTTP1Protocol expects an http response with the HTTP/1.1 protocol
+func HaveHTTP1Protocol() types.GomegaMatcher {
+	return HaveProtocol(HTTP1Protocol)
+}
+
+// HaveHTTP2Protocol expects an http response with the HTTP/2 protocol
+func HaveHTTP2Protocol() types.GomegaMatcher {
+	return HaveProtocol(HTTP2Protocol)
+}

test/gomega/matchers/have_protocol_test.go

@@ -0,0 +1,28 @@
+package matchers_test
+
+import (
+	"net/http"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/solo-io/gloo/test/gomega/matchers"
+)
+
+var _ = Describe("HaveProtocol", func() {
+
+	It("succeeds if there is no protocol specified", func() {
+		httpResponse := &http.Response{
+			Proto: matchers.HTTP1Protocol,
+		}
+		Expect(httpResponse).To(matchers.HaveProtocol(""))
+	})
+
+	It("matches the specified protocol", func() {
+		httpResponse := &http.Response{
+			Proto: matchers.HTTP1Protocol,
+		}
+		Expect(httpResponse).To(matchers.HaveProtocol(matchers.HTTP1Protocol))
+		Expect(httpResponse).To(matchers.HaveHTTP1Protocol())
+		Expect(httpResponse).ToNot(matchers.HaveHTTP2Protocol())
+	})
+})

test/gomega/transforms/curl.go

@@ -22,12 +22,13 @@ const (
 // WithCurlHttpResponse is a Gomega Transform that converts the string returned by an exec.Curl
 // and transforms it into an http.Response. This is useful to be used in tandem with matchers.HaveHttpResponse
 // NOTE: This is not feature complete, as we do not convert the entire response.
-// For now, we handle HTTP/1.1 response headers, status, and body.
+// For now, we handle HTTP/1.1 && HTTP/2 response headers, status, protocol and body.
 // The curl must be executed with verbose=true to include both the response headers/status
 // and response body.
 func WithCurlHttpResponse(curlResponse string) *http.Response {
 	headers := make(http.Header)
 	statusCode := 0
+	protocol := ""
 	var bodyBuf bytes.Buffer
 
 	for _, line := range strings.Split(curlResponse, "\n") {
@@ -37,9 +38,10 @@ func WithCurlHttpResponse(curlResponse string) *http.Response {
 			continue
 		}
 
-		code := processResponseCode(line)
+		proto, code := processResponseCodeAndProtocol(line)
 		if code != 0 {
 			statusCode = code
+			protocol = proto
 			continue
 		}
 
@@ -52,6 +54,7 @@ func WithCurlHttpResponse(curlResponse string) *http.Response {
 	}
 
 	return &http.Response{
+		Proto:      protocol,
 		StatusCode: statusCode,
 		Header:     headers,
 		Body:       bytesBody(bodyBuf.Bytes()),
@@ -61,6 +64,7 @@ func WithCurlHttpResponse(curlResponse string) *http.Response {
 func WithCurlResponse(curlResponse *kubectl.CurlResponse) *http.Response {
 	headers := make(http.Header)
 	statusCode := 0
+	protocol := ""
 	var bodyBuf bytes.Buffer
 
 	// Curl writes the body to stdout and the headers/status to stderr
@@ -72,16 +76,18 @@ func WithCurlResponse(curlResponse *kubectl.CurlResponse) *http.Response {
 			continue
 		}
 
-		code := processResponseCode(line)
+		proto, code := processResponseCodeAndProtocol(line)
 		if code != 0 {
 			statusCode = code
+			protocol = proto
 		}
 	}
 
 	// Body
 	bodyBuf.WriteString(curlResponse.StdOut)
 
 	return &http.Response{
+		Proto:      protocol,
 		StatusCode: statusCode,
 		Header:     headers,
 		Body:       bytesBody(bodyBuf.Bytes()),
@@ -102,21 +108,21 @@ func processResponseHeader(line string) (string, string) {
 	return "", ""
 }
 
-// processResponseCode processes the current line if it's a response status code.
-// Returns the status code if the line was processed, otherwise returns 0.
-func processResponseCode(line string) int {
+// processResponseCodeAndProtocol processes the current line if it's a response status code with the protocol.
+// Returns the status code and protocol if the line was processed, otherwise returns 0 and an empty string.
+func processResponseCodeAndProtocol(line string) (string, int) {
 	// check for response status. the line with the response code will be in the format
-	// `< HTTP/1.1 <code> <message>` or `< HTTP/2 <code> <message>`
+	// `< HTTP/1.1 <code> <message>` or `< HTTP/2 <code>`
 	if strings.HasPrefix(line, responseStatusPrefix1dot1) || strings.HasPrefix(line, responseStatusPrefix2) {
 		statusParts := strings.Split(line, " ")
-		if len(statusParts) >= 4 {
+		if len(statusParts) >= 3 {
 			statusCode, err := strconv.Atoi(statusParts[2])
 			if err == nil {
-				return statusCode
+				return statusParts[1], statusCode
 			}
 		}
 	}
-	return 0
+	return "", 0
 }
 
 // isResponseBody returns true if the current line is part of the response body, false otherwise.

test/gomega/transforms/transforms.go

@@ -67,3 +67,10 @@ func BytesToInt(b []byte) int {
 	Expect(err).NotTo(HaveOccurred())
 	return i
 }
+
+// WithProtocol returns a Gomega Transform that extracts the protocol of a request
+func WithProtocol() func(response *http.Response) string {
+	return func(response *http.Response) string {
+		return response.Proto
+	}
+}

test/kubernetes/e2e/features/services/httproute/suite.go

@@ -10,45 +10,23 @@ import (
 	"github.com/solo-io/gloo/projects/gateway2/wellknown"
 	"github.com/solo-io/gloo/test/kubernetes/e2e"
 	"github.com/solo-io/gloo/test/kubernetes/e2e/defaults"
+	"github.com/solo-io/gloo/test/kubernetes/e2e/tests/base"
 )
 
 // testingSuite is the entire Suite of tests for testing K8s Service-specific features/fixes
 type testingSuite struct {
-	suite.Suite
-
-	ctx context.Context
-
-	// testInstallation contains all the metadata/utilities necessary to execute a series of tests
-	// against an installation of Gloo Gateway
-	testInstallation *e2e.TestInstallation
+	*base.BaseTestingSuite
 }
 
 func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite {
 	return &testingSuite{
-		ctx:              ctx,
-		testInstallation: testInst,
+		base.NewBaseTestingSuite(ctx, testInst, base.SimpleTestCase{}, testCases),
 	}
 }
 
 func (s *testingSuite) TestConfigureHTTPRouteBackingDestinationsWithService() {
-	s.T().Cleanup(func() {
-		err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, routeWithServiceManifest)
-		s.NoError(err, "can delete manifest")
-		err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, serviceManifest)
-		s.NoError(err, "can delete manifest")
-		s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, proxyService, proxyDeployment)
-	})
-
-	err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, routeWithServiceManifest)
-	s.Assert().NoError(err, "can apply gloo.solo.io Route manifest")
-
-	// apply the service manifest separately, after the route table is applied, to ensure it can be applied after the route table
-	err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, serviceManifest)
-	s.Assert().NoError(err, "can apply gloo.solo.io Service manifest")
-
-	s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxyService, proxyDeployment)
-	s.testInstallation.Assertions.AssertEventualCurlResponse(
-		s.ctx,
+	s.TestInstallation.Assertions.AssertEventualCurlResponse(
+		s.Ctx,
 		defaults.CurlPodExecOpt,
 		[]curl.Option{
 			curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)),
@@ -59,34 +37,33 @@ func (s *testingSuite) TestConfigureHTTPRouteBackingDestinationsWithService() {
 
 func (s *testingSuite) TestConfigureHTTPRouteBackingDestinationsWithServiceAndWithoutTCPRoute() {
 	s.T().Cleanup(func() {
-		err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, routeWithServiceManifest)
-		s.NoError(err, "can delete manifest")
-		err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, serviceManifest)
-		s.NoError(err, "can delete manifest")
-		s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, proxyService, proxyDeployment)
-		err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, tcpRouteCrdManifest)
+		err := s.TestInstallation.Actions.Kubectl().ApplyFile(s.Ctx, tcpRouteCrdManifest)
 		s.NoError(err, "can apply manifest")
-		s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, &wellknown.TCPRouteCRD)
+		s.TestInstallation.Assertions.EventuallyObjectsExist(s.Ctx, &wellknown.TCPRouteCRD)
 	})
 
 	// Remove the TCPRoute CRD to assert HTTPRoute services still work.
-	err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, tcpRouteCrdManifest)
+	err := s.TestInstallation.Actions.Kubectl().DeleteFile(s.Ctx, tcpRouteCrdManifest)
 	s.NoError(err, "can delete manifest")
 
-	err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, routeWithServiceManifest)
-	s.Assert().NoError(err, "can apply gloo.solo.io Route manifest")
-
-	// apply the service manifest separately, after the route table is applied, to ensure it can be applied after the route table
-	err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, serviceManifest)
-	s.Assert().NoError(err, "can apply gloo.solo.io Service manifest")
-
-	s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxyService, proxyDeployment)
-	s.testInstallation.Assertions.AssertEventualCurlResponse(
-		s.ctx,
+	s.TestInstallation.Assertions.AssertEventualCurlResponse(
+		s.Ctx,
 		defaults.CurlPodExecOpt,
 		[]curl.Option{
 			curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)),
 			curl.WithHostHeader("example.com"),
 		},
 		expectedSvcResp)
 }
+
+func (s *testingSuite) TestHTTP2AppProtocol() {
+	s.TestInstallation.Assertions.AssertEventualCurlResponse(
+		s.Ctx,
+		defaults.CurlPodExecOpt,
+		[]curl.Option{
+			curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)),
+			curl.WithHostHeader("example.com"),
+			curl.WithArgs([]string{"--http2-prior-knowledge"}),
+		},
+		expectedHTTP2SvcResp)
+}