Supporting Custom CA without upstream changes - draft PR

.github/scality-storage-deployment/cloudserver-config-tls.json

@@ -0,0 +1,74 @@
+{
+    "port": 8443,
+    "listenOn": [],
+    "certFilePaths": {
+        "key": "/certs/server.key",
+        "cert": "/certs/server.crt",
+        "ca": "/certs/ca.crt"
+    },
+    "restEndpoints": {
+        "localhost": "us-east-1",
+        "127.0.0.1": "us-east-1",
+        "cloudserver-front": "us-east-1",
+        "s3.docker.test": "us-east-1",
+        "127.0.0.2": "us-east-1",
+        "s3.amazonaws.com": "us-east-1",
+        "s3.scality.com": "us-east-1"
+    },
+    "websiteEndpoints": [
+        "s3-website-us-east-1.amazonaws.com",
+        "s3-website.us-east-2.amazonaws.com",
+        "s3-website-us-west-1.amazonaws.com",
+        "s3-website-us-west-2.amazonaws.com",
+        "s3-website.ap-south-1.amazonaws.com",
+        "s3-website.ap-northeast-2.amazonaws.com",
+        "s3-website-ap-southeast-1.amazonaws.com",
+        "s3-website-ap-southeast-2.amazonaws.com",
+        "s3-website-ap-northeast-1.amazonaws.com",
+        "s3-website.eu-central-1.amazonaws.com",
+        "s3-website-eu-west-1.amazonaws.com",
+        "s3-website-sa-east-1.amazonaws.com",
+        "s3-website.localhost",
+        "s3-website.scality.test"
+    ],
+    "vaultd": {
+        "host": "localhost",
+        "port": 8500
+    },
+    "clusters": 1,
+    "log": {
+        "logLevel": "trace",
+        "dumpLevel": "error"
+    },
+    "healthChecks": {
+        "allowFrom": ["127.0.0.1/8", "::1"]
+    },
+    "recordLog": {
+        "enabled": false,
+        "recordLogName": "s3-recordlog"
+    },
+    "requests": {
+        "viaProxy": false,
+        "trustedProxyCIDRs": [],
+        "extractClientIPFromHeader": ""
+    },
+    "metadataClient": {
+        "host": "127.0.0.1",
+        "port": 9990
+    },
+    "dataClient": {
+        "host": "127.0.0.1",
+        "port": 9991
+    },
+    "metadataDaemon": {
+        "bindAddress": "127.0.0.1",
+        "port": 9990
+    },
+    "dataDaemon": {
+        "bindAddress": "127.0.0.1",
+        "port": 9991
+    },
+    "bucketd": {
+        "bootstrap": ["localhost:9000"]
+    }
+}

.github/scality-storage-deployment/cloudserver-config.json

@@ -1,6 +1,11 @@
 {
     "port": 8000,
     "listenOn": [],
+    "certFilePaths": {
+        "key": "/certs/server.key",
+        "cert": "/certs/server.crt",
+        "ca": "/certs/ca.crt"
+    },
     "restEndpoints": {
         "localhost": "us-east-1",
         "127.0.0.1": "us-east-1",

.github/scality-storage-deployment/docker-compose.yml

@@ -10,3 +10,36 @@ services:
     volumes:
       - ./cloudserver-config.json:/conf/config.json:ro
       - ./logs/s3:/logs/s3
+
+  # S3 service with TLS enabled for testing custom CA certificate support
+  # Prerequisites:
+  #   1. Generate certificates using: ../../tests/e2e/scripts/generate-test-certs.sh
+  #   2. Create certs directory with ca.crt, server.crt, server.key
+  # Usage:
+  #   ENDPOINT=${ENDPOINT:-s3.scality.local} docker compose --profile s3-tls up -d
+  # Note: openssl is installed at runtime as it's required by cloudserver's SSL entrypoint
+  s3-tls:
+    profiles: ['s3-tls']
+    image: ghcr.io/scality/cloudserver:${CLOUDSERVER_TAG:-7.70.65}
+    network_mode: host
+    environment:
+      S3_CONFIG_FILE: /conf/config-tls.json
+      S3VAULT: mem
+      # Enable SSL with externally generated certificates
+      SSL: "true"
+      SSL_KEY: /certs/server.key
+      SSL_CERT: /certs/server.crt
+      SSL_CA: /certs/ca.crt
+      ENDPOINT: "${ENDPOINT:-s3.scality.local}"
+    entrypoint: /bin/sh
+    command:
+      - -c
+      - |
+        apt-get update -qq && apt-get install -y -qq openssl > /dev/null 2>&1
+        exec /usr/src/app/docker-entrypoint.sh yarn start > /logs/s3/s3.log 2>&1
+    volumes:
+      - ./cloudserver-config-tls.json:/conf/config-tls.json:ro
+      - ./logs/s3:/logs/s3
+      # Mount externally generated certificates
+      # Generate using: ../../tests/e2e/scripts/generate-test-certs.sh
+      - ./certs:/certs:ro

.github/workflows/e2e-tests.yaml

@@ -149,3 +149,166 @@ jobs:
           file: ./tests/e2e/test-results/e2e-tests-results.xml
           flags: e2e_tests,cloudserver_${{ matrix.ring_version }}
           slug: scality/mountpoint-s3-csi-driver
+
+  # E2E tests with HTTPS and custom CA certificates
+  e2e-tests-tls:
+    name: TLS (Custom CA)
+    runs-on: ubuntu-22.04-8core
+    needs: dev-image
+    env:
+      CLOUDSERVER_TAG: ${{ vars.CLOUDSERVER_RING_9_5 }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Run Common Setup
+        uses: ./.github/actions/e2e-setup-common
+        with:
+          ref: ${{ github.sha }}
+          skip_container_pulls: "true"
+
+      - name: Pull Container Images
+        run: |
+          docker pull ghcr.io/${{ github.repository }}:${{ github.sha }} &
+          DRIVER_PID=$!
+          docker pull ghcr.io/scality/cloudserver:${CLOUDSERVER_TAG} &
+          CLOUDSERVER_PID=$!
+          wait $DRIVER_PID || { echo "❌ Driver image pull failed"; exit 1; }
+          wait $CLOUDSERVER_PID || { echo "❌ CloudServer image pull failed"; exit 1; }
+
+      - name: Load CSI Driver into KIND
+        run: |
+          kind load docker-image ghcr.io/${{ github.repository }}:${{ github.sha }} --name helm-test-cluster
+
+      - name: Get Host IP Address
+        id: get_ip
+        run: echo "host_ip=$(hostname -I | awk '{print $1}')" >> $GITHUB_OUTPUT
+
+      - name: Configure hosts file for S3 FQDN
+        run: |
+          echo "${{ steps.get_ip.outputs.host_ip }} s3.scality.com" | sudo tee -a /etc/hosts
+          cat /etc/hosts | grep s3.scality.com
+
+      - name: Configure CoreDNS for S3 FQDN
+        run: |
+          set -e -o pipefail
+          kubectl get configmap coredns -n kube-system -o json | \
+          jq --arg ip "${{ steps.get_ip.outputs.host_ip }}" \
+          '.data.Corefile |= sub("ready"; "ready\n        hosts {\n            " + $ip + " s3.scality.com\n            fallthrough\n        }")' | \
+          kubectl apply -f -
+          kubectl rollout restart deployment coredns -n kube-system
+          kubectl rollout status deployment coredns -n kube-system --timeout=60s
+          kubectl run dns-test --image=busybox:1.28 --rm -it --restart=Never -- nslookup s3.scality.com
+
+      - name: Generate TLS certificates
+        run: |
+          mkdir -p .github/scality-storage-deployment/certs
+          mkdir -p .github/scality-storage-deployment/logs/s3
+          # Generate self-signed certificates for TLS testing
+          ./tests/e2e/scripts/generate-test-certs.sh \
+            .github/scality-storage-deployment/certs \
+            s3.scality.com
+          echo "Generated certificates:"
+          ls -lh .github/scality-storage-deployment/certs/
+          echo "CA certificate contents:"
+          cat .github/scality-storage-deployment/certs/ca.crt
+
+      - name: Start Cloudserver with TLS
+        working-directory: .github/scality-storage-deployment
+        run: |
+          set -e -o pipefail
+          chown -R runner:docker logs certs
+          ENDPOINT=s3.scality.com docker compose --profile s3-tls up -d --quiet-pull
+          # Give container a moment to start
+          sleep 5
+          # Check container status and logs
+          echo "Container status:"
+          docker compose --profile s3-tls ps
+          echo "Container logs (last 50 lines):"
+          docker compose --profile s3-tls logs --tail=50 s3-tls || true
+          # Wait for cloudserver to start on HTTPS port (8443 when SSL is enabled)
+          bash ../scripts/wait_for_local_port.bash 8443 60
+
+      - name: Verify HTTPS endpoint
+        run: |
+          # Test that HTTPS endpoint is accessible with the CA certificate
+          curl --cacert .github/scality-storage-deployment/certs/ca.crt \
+            -v https://s3.scality.com:8443/ || echo "Expected 403 - endpoint is working"
+
+      - name: Start Kubernetes Event and Log Capture
+        run: |
+          mkdir -p artifacts/k8s-debug
+          ./tests/e2e/scripts/capture-events-and-logs.sh artifacts/k8s-debug start &
+          echo $! > capture.pid
+
+      - name: Apply CRDs
+        run: |
+          echo "Applying CRDs..."
+          kubectl apply -f ./charts/scality-mountpoint-s3-csi-driver/crds/
+
+      - name: Create CA Certificate Secret in kube-system
+        run: |
+          # Create the CA certificate secret in kube-system namespace
+          # The mount-s3 namespace secret will be created after Helm install
+          kubectl create secret generic s3-custom-ca-cert \
+            --from-file=ca-bundle.crt=.github/scality-storage-deployment/certs/ca.crt \
+            --namespace kube-system
+
+      - name: Run E2E Tests with TLS
+        run: |
+          mkdir -p test-results
+          source tests/e2e/scripts/load-credentials.sh
+          # Delete mount-s3 namespace so Helm can create it with proper annotations
+          kubectl delete namespace mount-s3 --ignore-not-found=true
+
+          # Install CSI driver (Helm will create mount-s3 namespace)
+          make csi-install \
+            S3_ENDPOINT_URL=https://s3.scality.com:8443 \
+            CSI_IMAGE_TAG=${{ github.sha }} \
+            CSI_IMAGE_REPOSITORY=ghcr.io/${{ github.repository }} \
+            ADDITIONAL_HELM_ARGS="--set tls.caCertSecret=s3-custom-ca-cert"
+
+          # Now create the CA cert secret in the Helm-managed namespace
+          kubectl create secret generic s3-custom-ca-cert \
+            --from-file=ca-bundle.crt=.github/scality-storage-deployment/certs/ca.crt \
+            --namespace mount-s3
+
+          # Run E2E tests
+          make e2e \
+            S3_ENDPOINT_URL=https://s3.scality.com:8443 \
+            ADDITIONAL_ARGS="--junit-report=./test-results/e2e-tests-tls-results.xml"
+
+      - name: Stop K8s Event Capture and Generate Debug Report
+        if: always()
+        run: |
+          if [ -f capture.pid ]; then
+            ./tests/e2e/scripts/capture-events-and-logs.sh artifacts/k8s-debug stop || true
+            rm -f capture.pid
+          fi
+          tar -czf artifacts/k8s-debug-capture.tar.gz -C artifacts k8s-debug/ || true
+
+      - name: Copy S3 logs and certificates to artifacts
+        if: always()
+        run: |
+          mkdir -p artifacts/logs/s3
+          mkdir -p artifacts/certs
+          cp -r .github/scality-storage-deployment/logs/s3/* artifacts/logs/s3/ 2>/dev/null || true
+          cp -r .github/scality-storage-deployment/certs/* artifacts/certs/ 2>/dev/null || true
+
+      - name: Upload Test Artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-test-artifacts-tls
+          path: artifacts
+
+      - name: Upload test results to Codecov
+        if: ${{ always() }}
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./tests/e2e/test-results/e2e-tests-tls-results.xml
+          flags: e2e_tests,tls
+          slug: scality/mountpoint-s3-csi-driver

Makefile

@@ -224,6 +224,10 @@ VALIDATE_S3 ?= false
 # Additional arguments to pass to the script (optional)
 ADDITIONAL_ARGS ?=
 
+# Additional Helm arguments for CSI driver installation (optional)
+# Example: ADDITIONAL_HELM_ARGS="--set tls.caCertSecret=my-ca-secret"
+ADDITIONAL_HELM_ARGS ?=
+
 ################################################################
 # Scality CSI driver commands
 ################################################################
@@ -305,6 +309,9 @@ csi-install:
 	if [ "$(VALIDATE_S3)" = "true" ]; then \
 		INSTALL_ARGS="$$INSTALL_ARGS --validate-s3"; \
 	fi; \
+	if [ ! -z "$(ADDITIONAL_HELM_ARGS)" ]; then \
+		INSTALL_ARGS="$$INSTALL_ARGS --additional-helm-args '$(ADDITIONAL_HELM_ARGS)'"; \
+	fi; \
 	if [ ! -z "$(ADDITIONAL_ARGS)" ]; then \
 		INSTALL_ARGS="$$INSTALL_ARGS $(ADDITIONAL_ARGS)"; \
 	fi; \
@@ -431,4 +438,7 @@ e2e-all:
 	if [ ! -z "$(ADDITIONAL_ARGS)" ]; then \
 		INSTALL_ARGS="$$INSTALL_ARGS $(ADDITIONAL_ARGS)"; \
 	fi; \
+	if [ ! -z "$(ADDITIONAL_HELM_ARGS)" ]; then \
+		export E2E_ADDITIONAL_HELM_ARGS="$(ADDITIONAL_HELM_ARGS)"; \
+	fi; \
 	./tests/e2e/scripts/run.sh all $$INSTALL_ARGS

charts/scality-mountpoint-s3-csi-driver/templates/controller.yaml

@@ -61,6 +61,11 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+            {{- if .Values.tls.caCertSecret }}
+            - name: custom-ca-cert
+              mountPath: /etc/ssl/custom-ca
+              readOnly: true
+            {{- end }}
           {{- with .Values.controller.resources }}
           resources:
             {{- toYaml . | nindent 12 }}
@@ -74,6 +79,11 @@ spec:
               value: "controller"
             - name: CSI_CONTROLLER_ONLY
               value: "true"
+            {{- if .Values.tls.caCertSecret }}
+            # AWS SDK Go v2 reads AWS_CA_BUNDLE to configure custom CA certificates
+            - name: AWS_CA_BUNDLE
+              value: /etc/ssl/custom-ca/ca-bundle.crt
+            {{- end }}
             - name: AWS_ACCESS_KEY_ID
               valueFrom:
                 secretKeyRef:
@@ -115,6 +125,21 @@ spec:
               value: {{ printf "%s:%s" .Values.mountpointPod.headroomImage.repository .Values.mountpointPod.headroomImage.tag | quote }}
             - name: MOUNTPOINT_IMAGE_PULL_POLICY
               value: {{ .Values.image.pullPolicy | quote }}
+            {{- if .Values.tls.caCertSecret }}
+            # TLS configuration for custom CA certificates in mounter pods
+            - name: TLS_CA_CERT_SECRET
+              value: {{ .Values.tls.caCertSecret | quote }}
+            - name: TLS_INIT_IMAGE
+              value: {{ printf "%s:%s" .Values.tls.initImage.repository .Values.tls.initImage.tag | quote }}
+            - name: TLS_INIT_IMAGE_PULL_POLICY
+              value: {{ .Values.tls.initImage.pullPolicy | quote }}
+            - name: TLS_INIT_RESOURCES_REQUESTS_CPU
+              value: {{ .Values.tls.initResources.requests.cpu | quote }}
+            - name: TLS_INIT_RESOURCES_REQUESTS_MEMORY
+              value: {{ .Values.tls.initResources.requests.memory | quote }}
+            - name: TLS_INIT_RESOURCES_LIMITS_MEMORY
+              value: {{ .Values.tls.initResources.limits.memory | quote }}
+            {{- end }}
         - name: csi-provisioner
           image: {{ .Values.sidecars.csiProvisioner.image.repository }}:{{ .Values.sidecars.csiProvisioner.image.tag }}
           imagePullPolicy: {{ .Values.sidecars.csiProvisioner.image.pullPolicy }}
@@ -127,3 +152,11 @@ spec:
       volumes:
         - name: socket-dir
           emptyDir: {}
+        {{- if .Values.tls.caCertSecret }}
+        - name: custom-ca-cert
+          secret:
+            secretName: {{ .Values.tls.caCertSecret }}
+            items:
+              - key: ca-bundle.crt
+                path: ca-bundle.crt
+        {{- end }}

charts/scality-mountpoint-s3-csi-driver/templates/node.yaml

@@ -103,6 +103,21 @@ spec:
                   key: {{ .sessionToken }}
                   optional: true
             {{- end }}
+            {{- if .Values.tls.caCertSecret }}
+            # TLS configuration for custom CA certificates in mounter pods
+            - name: TLS_CA_CERT_SECRET
+              value: {{ .Values.tls.caCertSecret | quote }}
+            - name: TLS_INIT_IMAGE
+              value: {{ printf "%s:%s" .Values.tls.initImage.repository .Values.tls.initImage.tag | quote }}
+            - name: TLS_INIT_IMAGE_PULL_POLICY
+              value: {{ .Values.tls.initImage.pullPolicy | quote }}
+            - name: TLS_INIT_RESOURCES_REQUESTS_CPU
+              value: {{ .Values.tls.initResources.requests.cpu | quote }}
+            - name: TLS_INIT_RESOURCES_REQUESTS_MEMORY
+              value: {{ .Values.tls.initResources.requests.memory | quote }}
+            - name: TLS_INIT_RESOURCES_LIMITS_MEMORY
+              value: {{ .Values.tls.initResources.limits.memory | quote }}
+            {{- end }}
           volumeMounts:
             - name: kubelet-dir
               mountPath: {{ .Values.node.kubeletPath }}