ING-1399: Accurately test gracefuil shutdown behaviour

Author: Westwooo

.github/test.yml

@@ -0,0 +1,79 @@
+name: Run Integration Tests
+permissions:
+  contents: read
+  packages: read
+
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - master
+  pull_request:
+jobs:
+  test:
+    name: Test
+    strategy:
+      matrix:
+        server:
+          - 8.1.0-1262
+          - 8.0.0
+          - 7.6.8
+          - 7.2.8
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Install cbdinocluster
+        uses: ./.github/actions/install-cbdinocluster
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Start couchbase cluster
+        id: start-cluster
+        uses: ./.github/actions/start-couchbase-cluster
+      - name: Install Stellar Gateway
+        uses: actions/checkout@v4
+        with:
+          repository: couchbase/stellar-gateway
+          submodules: recursive
+      - uses: actions/setup-go@v5
+        with:
+          go-version: 1.24
+      - uses: arduino/setup-protoc@v3
+        with:
+          version: 31.1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Tools
+        run: |
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.5
+          go install github.com/matryer/moq@v0.5
+      - name: Install Dependencies
+        run: go get ./...
+      - name: Generate Files
+        run: |
+          go generate ./...
+      
+      
+      
+      - name: Run Tests
+        timeout-minutes: 10
+        env:
+          SGTEST_CBCONNSTR:  ${{ steps.start-cluster.outputs.node-ip }}
+        run: go test -v $(go list ./... | grep -v /contrib/)
+
+      - name: Collect couchbase logs
+        timeout-minutes: 10
+        if: failure()
+        run: |
+          mkdir -p ./logs
+          cbdinocluster -v collect-logs ${{ steps.start-cluster.outputs.dino-id }} ./logs
+      - name: Upload couchbase logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cbcollect-logs-${{ matrix.server }}
+          path: ./logs/*
+          retention-days: 5

.github/workflows/test.yml

@@ -56,6 +56,12 @@ jobs:
           SGTEST_CBCONNSTR: ${{ steps.start-cluster.outputs.node-ip }}
         run: go test -v $(go list ./... | grep -v /contrib/)
 
+      - name: Run Graceful Shutdown Test X 100
+        timeout-minutes: 30
+        env:
+          SGTEST_CBCONNSTR: ${{ steps.start-cluster.outputs.node-ip }}
+        run: go test ./gateway/test -run TestGatewayOps -testify.m TestGracefulShutdown -count 100 -timeout 30m
+
       - name: Collect couchbase logs
         timeout-minutes: 10
         if: failure()

gateway/test/dapi_graceful_shutdown_test.go

@@ -3,8 +3,11 @@ package test
 import (
 	"context"
 	"crypto/tls"
+	"errors"
 	"fmt"
+	"io"
 	"net/http"
+	"net/http/httptrace"
 	"sync"
 	"syscall"
 	"time"
@@ -69,19 +72,51 @@ func (s *GatewayOpsTestSuite) TestGracefulShutdown() {
 		},
 	}
 
+	var requestsWritten int
+	var responsesReceived int
+	trace := &httptrace.ClientTrace{
+		WroteRequest: func(info httptrace.WroteRequestInfo) {
+			requestsWritten++
+		},
+		GotFirstResponseByte: func() {
+			responsesReceived++
+		},
+	}
+
+	ctx := httptrace.WithClientTrace(context.Background(), trace)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("https://%s/v1/callerIdentity", dapiAddr), nil)
+	assert.NoError(s.T(), err)
+
 	respCloseChan := make(chan (bool), 10000)
 	var wg sync.WaitGroup
 
+	var eofs int
+	var unexpectedErr error
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
 
 		for {
-			resp, err := dapiCli.Get(fmt.Sprintf("https://%s/v1/callerIdentity", dapiAddr))
+			resp, err := dapiCli.Do(req)
 			if err != nil {
-				// A non-nil error should be caused by sending requests to the gateway
-				// after it has already shutdown.
-				assert.ErrorIs(s.T(), err, syscall.ECONNREFUSED)
+				switch {
+				case errors.Is(err, io.EOF):
+					// There is a small window between the flushing the request bytes to the socket and the http handler receiving
+					// the bytes where the server sees the connection as idle and will close it. We record these errors and
+					// include them when checking all requests recieved responses.
+					eofs++
+				case errors.Is(err, syscall.ECONNREFUSED):
+					// This is what we expect to see once the listeners have closed
+				case errors.Is(err, syscall.ECONNRESET), errors.Is(err, syscall.EPIPE):
+					// Connection reset and broken pipe errors occur before the request is completely written, therefore the
+					// wroteRequest hook has not triggered. Such racy errors are unavoidable when running directly against an
+					// http server.
+				default:
+					// Any errors not mentinoned above are not expected and should cause a failure to be investigated.
+					unexpectedErr = err
+					return
+				}
+
 				return
 			}
 
@@ -97,6 +132,9 @@ func (s *GatewayOpsTestSuite) TestGracefulShutdown() {
 
 	wg.Wait()
 
+	assert.NoError(s.T(), unexpectedErr)
+	assert.Equal(s.T(), requestsWritten, responsesReceived+eofs)
+
 	isFirstResponse := true
 	keepAlivesDisabled := false
 	for len(respCloseChan) > 0 {