Allow server to cancel when deadline expires (#999)

This makes some assertions for timeout cases a little more lenient. The
tests previously asserted that a server would return a "deadline
exceeded" error when the client's deadline expires. However, not all
gRPC servers behave this way. The gRPC Go reference implementation was
recently updated to sometimes just cancel the stream instead of sending
back an error response. This doesn't seem unreasonable, so instead of
updating the "known failing" list of cases for the grpc-go server, this
PR makes this behavior allowed (which means it will be allowed in any
other implementation-under-test, too).

The test case structure lets us define other acceptable error codes for
test cases, however we don't actually want to add "canceled" as
generally acceptable for these tests. It's really only acceptable for
the server (clients should always report "deadline exceeded" for these
cases). So we need to update the test assertions logic to get that level
of nuance.

Author: jhump

internal/app/connectconformance/connectconformance.go

@@ -312,7 +312,7 @@ func run( //nolint:gocyclo
 		}
 	}
 
-	results := newResults(filteredTestCount, knownFailing, knownFlaky, trace)
+	results := newResults(mode, filteredTestCount, knownFailing, knownFlaky, trace)
 
 	for _, clientInfo := range clients {
 		clientProcess, err := runClient(ctx, clientInfo.start)

internal/app/connectconformance/results.go

@@ -53,6 +53,7 @@ const timeoutCheckGracePeriodMillis = 500
 // log writer. It can also incorporate data provided out-of-band by a reference
 // server, when testing a client implementation.
 type testResults struct {
+	mode           conformancev1.TestSuite_TestMode
 	totalTestCount int
 	knownFailing   *testTrie
 	knownFlaky     *testTrie
@@ -66,8 +67,9 @@ type testResults struct {
 	serverSideband map[string]string
 }
 
-func newResults(totalTestCount int, knownFailing, knownFlaky *testTrie, tracer *tracer.Tracer) *testResults {
+func newResults(mode conformancev1.TestSuite_TestMode, totalTestCount int, knownFailing, knownFlaky *testTrie, tracer *tracer.Tracer) *testResults {
 	return &testResults{
+		mode:           mode,
 		totalTestCount: totalTestCount,
 		knownFailing:   knownFailing,
 		knownFlaky:     knownFlaky,
@@ -168,7 +170,19 @@ func (r *testResults) assert(
 	expected := definition.ExpectedResponse
 	var errs multiErrors
 
-	errs = append(errs, checkError(expected.Error, actual.Error, definition.OtherAllowedErrorCodes)...)
+	otherErrors := definition.OtherAllowedErrorCodes
+	if definition.Request.TimeoutMs != nil && r.mode == conformancev1.TestSuite_TEST_MODE_SERVER &&
+		expected.Error != nil && expected.Error.Code == conformancev1.Code_CODE_DEADLINE_EXCEEDED &&
+		definition.Request.HttpVersion > conformancev1.HTTPVersion_HTTP_VERSION_1 && len(otherErrors) == 0 {
+		// When testing a server, the reference client does not set an actual timeout, in order to
+		// see if the server enforces the timeout. Typically, a server will return a "deadline exceeded"
+		// error when it notices that time is up. But it is also possible that the server chooses to
+		// simply cancel the stream (only possible with HTTP/2 or HTTP/3) -- why send back an error
+		// response if the client has already "hung up"? So we allow for the reference client to observe
+		// a CANCELED error in these cases, as an alternative to DEADLINE_EXCEEDED.
+		otherErrors = []conformancev1.Code{conformancev1.Code_CODE_CANCELED}
+	}
+	errs = append(errs, checkError(expected.Error, actual.Error, otherErrors)...)
 	errs = append(errs, checkPayloads(expected.Payloads, actual.Payloads)...)
 
 	if len(expected.Payloads) == 0 &&

internal/app/connectconformance/results_test.go

@@ -30,7 +30,7 @@ import (
 
 func TestResults_SetOutcome(t *testing.T) {
 	t.Parallel()
-	results := newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results := newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	results.setOutcome("foo/bar/1", false, nil)
 	results.setOutcome("foo/bar/2", true, errors.New("fail"))
 	results.setOutcome("foo/bar/3", false, errors.New("fail"))
@@ -60,7 +60,7 @@ func TestResults_SetOutcome(t *testing.T) {
 
 func TestResults_FailedToStart(t *testing.T) {
 	t.Parallel()
-	results := newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results := newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	results.failedToStart([]*conformancev1.TestCase{
 		{Request: &conformancev1.ClientCompatRequest{TestName: "foo/bar/1"}},
 		{Request: &conformancev1.ClientCompatRequest{TestName: "known-to-fail/1"}},
@@ -80,7 +80,7 @@ func TestResults_FailedToStart(t *testing.T) {
 
 func TestResults_FailRemaining(t *testing.T) {
 	t.Parallel()
-	results := newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results := newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	results.setOutcome("foo/bar/1", false, nil)
 	results.setOutcome("known-to-fail/1", false, errors.New("fail"))
 	results.failRemaining([]*conformancev1.TestCase{
@@ -107,7 +107,7 @@ func TestResults_FailRemaining(t *testing.T) {
 
 func TestResults_Failed(t *testing.T) {
 	t.Parallel()
-	results := newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results := newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	results.failed("foo/bar/1", &conformancev1.ClientErrorResult{Message: "fail"})
 	results.failed("known-to-fail/1", &conformancev1.ClientErrorResult{Message: "fail"})
 
@@ -124,7 +124,7 @@ func TestResults_Failed(t *testing.T) {
 
 func TestResults_Assert(t *testing.T) {
 	t.Parallel()
-	results := newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results := newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	payload1 := &conformancev1.ClientResponseResult{
 		Payloads: []*conformancev1.ConformancePayload{
 			{Data: []byte{0, 1, 2, 3, 4}},
@@ -685,7 +685,7 @@ func TestResults_Assert_ReportsAllErrors(t *testing.T) {
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
-			results := newResults(0, &testTrie{}, &testTrie{}, nil)
+			results := newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, &testTrie{}, &testTrie{}, nil)
 
 			expected := &conformancev1.TestCase{
 				Request:          &conformancev1.ClientCompatRequest{StreamType: conformancev1.StreamType_STREAM_TYPE_UNARY},
@@ -722,7 +722,7 @@ func TestResults_Assert_ReportsAllErrors(t *testing.T) {
 
 func TestResults_ServerSideband(t *testing.T) {
 	t.Parallel()
-	results := newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results := newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	results.setOutcome("foo/bar/1", false, nil)
 	results.setOutcome("foo/bar/2", false, errors.New("fail"))
 	results.setOutcome("foo/bar/3", false, nil)
@@ -747,50 +747,50 @@ func TestResults_ServerSideband(t *testing.T) {
 
 func TestResults_Report(t *testing.T) {
 	t.Parallel()
-	results := newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results := newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	logger := &internal.SimplePrinter{}
 
 	// No test cases? Report success.
 	success := results.report(logger)
 	require.True(t, success)
 
 	// Only successful outcomes? Report success.
-	results = newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results = newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	results.setOutcome("foo/bar/1", false, nil)
 	success = results.report(logger)
 	require.True(t, success)
 
 	// Unexpected failure? Report failure.
-	results = newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results = newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	results.setOutcome("foo/bar/1", false, errors.New("ruh roh"))
 	success = results.report(logger)
 	require.False(t, success)
 
 	// Unexpected failure during setup? Report failure.
-	results = newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results = newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	results.setOutcome("foo/bar/1", true, errors.New("ruh roh"))
 	success = results.report(logger)
 	require.False(t, success)
 
 	// Expected failure? Report success.
-	results = newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results = newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	results.setOutcome("known-to-fail/1", false, errors.New("ruh roh"))
 	success = results.report(logger)
 	require.True(t, success)
 
 	// Setup error from expected failure? Report failure (setup errors never acceptable).
-	results = newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results = newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	results.setOutcome("known-to-fail/1", true, errors.New("ruh roh"))
 	success = results.report(logger)
 	require.False(t, success)
 
 	// Flaky? Report success whether it passes or fails
-	results = newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results = newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	results.setOutcome("known-to-flake/1", false, nil) // succeeds
 	success = results.report(logger)
 	require.True(t, success)
 
-	results = newResults(0, makeKnownFailing(), makeKnownFlaky(), nil)
+	results = newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, 0, makeKnownFailing(), makeKnownFlaky(), nil)
 	results.setOutcome("known-to-flake/1", false, errors.New("ruh roh"))
 	success = results.report(logger)
 	require.True(t, success)

internal/app/connectconformance/server_runner_test.go

@@ -195,7 +195,7 @@ func TestRunTestCasesForServer(t *testing.T) {
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
-			results := newResults(len(requests), &testTrie{}, &testTrie{}, nil)
+			results := newResults(conformancev1.TestSuite_TEST_MODE_UNSPECIFIED, len(requests), &testTrie{}, &testTrie{}, nil)
 
 			var procAddr atomic.Pointer[process] // populated when server process created
 			var actualSvrRequest bytes.Buffer