ING-1329: Add support for testing client cert auth

Author: Westwooo

.github/actions/install-cbdinocluster/action.yml

@@ -11,7 +11,7 @@ runs:
       shell: bash
       run: |
         mkdir -p "$HOME/bin"
-        wget -nv -O $HOME/bin/cbdinocluster https://github.com/couchbaselabs/cbdinocluster/releases/download/v0.0.86/cbdinocluster-linux-amd64
+        wget -nv -O $HOME/bin/cbdinocluster https://github.com/couchbaselabs/cbdinocluster/releases/download/v0.0.95/cbdinocluster-linux-amd64
         chmod +x $HOME/bin/cbdinocluster
         echo "$HOME/bin" >> $GITHUB_PATH
     - name: Initialize cbdinocluster

.github/actions/start-couchbase-cluster/action.yml

@@ -24,6 +24,7 @@ runs:
             kv-memory: 2048
             index-memory: 1024
             fts-memory: 1024
+            use-dino-certs: true
       run: |
         CBDC_ID=$(cbdinocluster -v alloc --def="${CLUSTERCONFIG}")
         cbdinocluster -v buckets add ${CBDC_ID} default --ram-quota-mb=100 --flush-enabled=true --num-replicas=2

.github/workflows/client_cert_auth_test.yml

@@ -0,0 +1,71 @@
+name: Run Client Cert Auth Tests
+permissions:
+  contents: read
+  packages: read
+
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - master
+  pull_request:
+jobs:
+  test:
+    name: Test
+    strategy:
+      matrix:
+        server:
+          - 8.1.0-1203
+          - 8.0.0
+          - 7.6.8
+          - 7.2.8
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Install cbdinocluster
+        uses: ./.github/actions/install-cbdinocluster
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Start couchbase cluster
+        id: start-cluster
+        uses: ./.github/actions/start-couchbase-cluster
+      - uses: actions/setup-go@v5
+        with:
+          go-version: 1.24
+      - uses: arduino/setup-protoc@v3
+        with:
+          version: 31.1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install Tools
+        run: |
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.5
+          go install github.com/matryer/moq@v0.5
+      - name: Install Dependencies
+        run: go get ./...
+      - name: Generate Files
+        run: |
+          go generate ./...
+      - name: Run Test
+        env:
+          SGTEST_CBCONNSTR: ${{ steps.start-cluster.outputs.node-ip }}
+          SGTEST_DINOID: ${{ steps.start-cluster.outputs.dino-id }}
+        run: go test ./gateway/test -run TestGatewayOps -v -testify.m TestClientCertAuth
+
+      - name: Collect couchbase logs
+        timeout-minutes: 10
+        if: failure()
+        run: |
+          mkdir -p ./client-cert-auth-logs
+          cbdinocluster -v collect-logs ${{ steps.start-cluster.outputs.dino-id }} ./client-cert-auth-logs
+      - name: Upload couchbase logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cbcollect-logs-${{ matrix.server }}
+          path: ./client-cert-auth-logs/*
+          retention-days: 5

gateway/test/mtls_test.go

@@ -0,0 +1,115 @@
+package test
+
+import (
+	"context"
+	"crypto/tls"
+	"time"
+
+	"github.com/couchbase/goprotostellar/genproto/kv_v1"
+	"github.com/couchbase/stellar-gateway/testutils"
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+)
+
+func (s *GatewayOpsTestSuite) TestClientCertAuth() {
+	testutils.SkipIfNoDinoCluster(s.T())
+
+	s.Run("KvService", s.KvService)
+}
+
+func (s *GatewayOpsTestSuite) KvService() {
+	dino := testutils.StartDinoTesting(s.T(), false)
+	username := "kvUser"
+	conn := s.newClientCertConn(dino, username)
+	kvClient := kv_v1.NewKvServiceClient(conn)
+	getFn := func() (*kv_v1.GetResponse, error) {
+		return kvClient.Get(context.Background(), &kv_v1.GetRequest{
+			BucketName:     s.bucketName,
+			ScopeName:      s.scopeName,
+			CollectionName: s.collectionName,
+			Key:            s.testDocId(),
+		})
+	}
+
+	s.Run("UserMissing", func() {
+		_, err := getFn()
+		assertRpcStatus(s.T(), err, codes.PermissionDenied)
+		assert.Contains(s.T(), err.Error(), "Your certificate is invalid")
+	})
+
+	dino.AddUnprivilegedUser(username)
+	time.Sleep(time.Second * 5)
+	s.T().Cleanup(func() {
+		dino.RemoveUser(username)
+	})
+
+	s.Run("NoUserPermissions", func() {
+		_, err := getFn()
+		assertRpcStatus(s.T(), err, codes.PermissionDenied)
+		assert.Contains(s.T(), err.Error(), "No permissions to read documents")
+	})
+
+	dino.AddReadOnlyUser(username)
+	time.Sleep(time.Second * 5)
+
+	s.Run("ReadSuccess", func() {
+		resp, err := getFn()
+		requireRpcSuccess(s.T(), resp, err)
+	})
+
+	s.Run("NoWritePermission", func() {
+		docId := s.randomDocId()
+		_, err := kvClient.Upsert(context.Background(), &kv_v1.UpsertRequest{
+			BucketName:     s.bucketName,
+			ScopeName:      s.scopeName,
+			CollectionName: s.collectionName,
+			Key:            docId,
+			Content: &kv_v1.UpsertRequest_ContentUncompressed{
+				ContentUncompressed: TEST_CONTENT,
+			},
+			ContentFlags: TEST_CONTENT_FLAGS,
+		})
+		assertRpcStatus(s.T(), err, codes.PermissionDenied)
+		assert.Contains(s.T(), err.Error(), "No permissions to write documents")
+	})
+
+	dino.AddWriteUser(username)
+	time.Sleep(time.Second * 5)
+
+	s.Run("WriteSuccess", func() {
+		docId := s.randomDocId()
+		resp, err := kvClient.Upsert(context.Background(), &kv_v1.UpsertRequest{
+			BucketName:     s.bucketName,
+			ScopeName:      s.scopeName,
+			CollectionName: s.collectionName,
+			Key:            docId,
+			Content: &kv_v1.UpsertRequest_ContentUncompressed{
+				ContentUncompressed: TEST_CONTENT,
+			},
+			ContentFlags: TEST_CONTENT_FLAGS,
+		})
+		requireRpcSuccess(s.T(), resp, err)
+		assertValidCas(s.T(), resp.Cas)
+		assertValidMutationToken(s.T(), resp.MutationToken, s.bucketName)
+	})
+}
+
+func (s *GatewayOpsTestSuite) newClientCertConn(dino *testutils.DinoController, username string) *grpc.ClientConn {
+	res := dino.GetClientCert(username)
+
+	cert, err := tls.X509KeyPair([]byte(res), []byte(res))
+	assert.NoError(s.T(), err)
+
+	conn, err := grpc.NewClient(s.gwConnAddr,
+		grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{
+			RootCAs:      s.clientCaCertPool,
+			Certificates: []tls.Certificate{cert},
+		})))
+	if err != nil {
+		s.T().Fatalf("failed to connect to test gateway: %s", err)
+	}
+
+	return conn
+}

gateway/test/suite_test.go

@@ -3,8 +3,10 @@ package test
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/base64"
 	"encoding/json"
+	"encoding/pem"
 	"fmt"
 	"math/rand"
 	"net/http"
@@ -41,6 +43,7 @@ type GatewayOpsTestSuite struct {
 	testClusterInfo  *testutils.CanonicalTestCluster
 	gatewayCloseFunc func()
 	gatewayConn      *grpc.ClientConn
+	gwConnAddr       string
 	gatewayClosedCh  chan struct{}
 	dapiCli          *http.Client
 	dapiAddr         string
@@ -56,6 +59,8 @@ type GatewayOpsTestSuite struct {
 	basicRestCreds string
 	readRestCreds  string
 
+	clientCaCertPool *x509.CertPool
+
 	clusterVersion *NodeVersion
 	features       []TestFeature
 
@@ -269,6 +274,15 @@ func (s *GatewayOpsTestSuite) SetupSuite() {
 			s.T().Fatalf("failed to create testing certificate: %s", err)
 		}
 
+		s.clientCaCertPool = x509.NewCertPool()
+		if testConfig.DinoId != "" {
+			caCert := s.getDinoCaCert()
+
+			s.clientCaCertPool.AddCert(caCert)
+
+			gwCert = s.getServerCert()
+		}
+
 		gwStartInfoCh := make(chan *gateway.StartupInfo, 1)
 		gwCtx, gwCtxCancel := context.WithCancel(context.Background())
 		gw, err := gateway.NewGateway(&gateway.Config{
@@ -280,6 +294,7 @@ func (s *GatewayOpsTestSuite) SetupSuite() {
 			BindDapiPort:    0,
 			GrpcCertificate: *gwCert,
 			DapiCertificate: *gwCert,
+			ClientCaCert:    s.clientCaCertPool,
 			AlphaEndpoints:  true,
 			NumInstances:    1,
 			ProxyServices:   []string{"query", "analytics", "mgmt", "search"},
@@ -324,6 +339,7 @@ func (s *GatewayOpsTestSuite) SetupSuite() {
 		}
 
 		s.gatewayConn = conn
+		s.gwConnAddr = connAddr
 		s.gatewayCloseFunc = gwCtxCancel
 		s.gatewayClosedCh = gwClosedCh
 		s.dapiCli = dapiCli
@@ -346,6 +362,7 @@ func (s *GatewayOpsTestSuite) SetupSuite() {
 		}
 
 		s.gatewayConn = conn
+		s.gwConnAddr = connAddr
 		s.dapiCli = dapiCli
 		s.dapiAddr = dapiAddr
 	}
@@ -388,6 +405,39 @@ func (s *GatewayOpsTestSuite) ParseSupportedFeatures(featsStr string) {
 	}
 }
 
+func (s *GatewayOpsTestSuite) getDinoCaCert() *x509.Certificate {
+	res, err := testutils.GetDinoCACert()
+	if err != nil {
+		s.T().Fatalf("failed to get dino ca cert: %s", err)
+	}
+
+	block, _ := pem.Decode([]byte(res))
+	if block == nil {
+		s.T().Fatalf("failed to decode dino ca cert: %s", err)
+	}
+
+	cert, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		s.T().Fatalf("failed to parse dino ca cert: %s", err)
+	}
+
+	return cert
+}
+
+func (s *GatewayOpsTestSuite) getServerCert() *tls.Certificate {
+	res, err := testutils.GetServerCert("127.0.0.1", "")
+	if err != nil {
+		s.T().Fatalf("failed to get server cert: %s", err)
+	}
+
+	cert, err := tls.X509KeyPair([]byte(res), []byte(res))
+	if err != nil {
+		s.T().Fatalf("failed to parse server cert: %s", err)
+	}
+
+	return &cert
+}
+
 var TEST_CONTENT = []byte(`{"foo": "bar","obj":{"num":14,"arr":[2,5,8],"str":"zz"},"num":11,"arr":[3,6,9,12]}`)
 var TEST_CONTENT_FLAGS = uint32(0x01000000)
 

testutils/dinocluster.go

@@ -3,6 +3,7 @@ package testutils
 import (
 	"bufio"
 	"context"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
@@ -93,6 +94,22 @@ func runDinoRemoveNode(node string) error {
 	return runNoResDinoCmd([]string{"nodes", "rm", globalTestConfig.DinoId, node})
 }
 
+func runDinoAddUser(username string, canRead, canWrite bool) error {
+	return runNoResDinoCmd([]string{
+		"users",
+		"add",
+		globalTestConfig.DinoId,
+		username,
+		"--password=password",
+		fmt.Sprintf("--can-read=%v", canRead),
+		fmt.Sprintf("--can-write=%v", canWrite),
+	})
+}
+
+func runDinoRemoveUser(username string) error {
+	return runNoResDinoCmd([]string{"users", "remove", globalTestConfig.DinoId, username})
+}
+
 type DinoController struct {
 	t             *testing.T
 	oldFoSettings *cbmgmtx.GetAutoFailoverSettingsResponse
@@ -182,3 +199,37 @@ func (c *DinoController) RemoveNode(node string) {
 	err := runDinoRemoveNode(node)
 	require.NoError(c.t, err)
 }
+
+func (c *DinoController) AddUnprivilegedUser(username string) {
+	err := runDinoAddUser(username, false, false)
+	require.NoError(c.t, err)
+}
+
+func (c *DinoController) AddReadOnlyUser(username string) {
+	err := runDinoAddUser(username, true, false)
+	require.NoError(c.t, err)
+}
+
+func (c *DinoController) AddWriteUser(username string) {
+	err := runDinoAddUser(username, true, true)
+	require.NoError(c.t, err)
+}
+
+func (c *DinoController) RemoveUser(username string) {
+	err := runDinoRemoveUser(username)
+	require.NoError(c.t, err)
+}
+
+func (c *DinoController) GetClientCert(username string) string {
+	res, err := runDinoCmd([]string{"certificates", "get-client-cert", username})
+	require.NoError(c.t, err)
+	return res
+}
+
+func GetDinoCACert() (string, error) {
+	return runDinoCmd([]string{"certificates", "get-dino-ca"})
+}
+
+func GetServerCert(ip string, dns string) (string, error) {
+	return runDinoCmd([]string{"certificates", "get-server-cert", "--ip", ip, "--dns", dns})
+}