(WIP) S3CSI-7: enable-SystemdMounter-tests

.github/actions/e2e-setup-common/action.yaml

@@ -0,0 +1,50 @@
+name: "Setup Common"
+description: "Common setup steps for E2E tests"
+inputs:
+  ref:
+    description: "Git ref to checkout"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Check out repository
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.ref }}
+
+    - name: Login to Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io/scality
+        username: "${{ github.repository_owner }}"
+        password: "${{ github.token }}"
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version-file: "go.mod"
+
+    - name: Install Ginkgo CLI
+      shell: bash
+      run: go install github.com/onsi/ginkgo/v2/ginkgo
+
+    - name: Create Kind Cluster
+      uses: helm/[email protected]
+      with:
+        wait: 90s
+        cluster_name: helm-test-cluster
+
+    - name: Pull Dev Image into KIND Cluster
+      shell: bash
+      run: |
+        docker pull ghcr.io/scality/mountpoint-s3-csi-driver:${{ github.sha }}
+        kind load docker-image ghcr.io/scality/mountpoint-s3-csi-driver:${{ github.sha }} --name helm-test-cluster
+
+    - name: Set up Helm
+      uses: azure/[email protected]
+      with:
+        version: v3.17.2
+
+    - name: Set Host IP
+      shell: bash
+      run: echo "HOST_IP=$(hostname -I | awk '{print $1}')" >> $GITHUB_ENV

.github/s3_and_iam_deployment/cloudserver-config.json

@@ -0,0 +1,49 @@
+{
+    "port": 8000,
+    "listenOn": [],
+    "restEndpoints": {
+        "localhost": "us-east-1",
+        "127.0.0.1": "us-east-1",
+        "cloudserver-front": "us-east-1",
+        "s3.docker.test": "us-east-1",
+        "127.0.0.2": "us-east-1",
+        "s3.amazonaws.com": "us-east-1"
+    },
+    "websiteEndpoints": [
+        "s3-website-us-east-1.amazonaws.com",
+        "s3-website.us-east-2.amazonaws.com",
+        "s3-website-us-west-1.amazonaws.com",
+        "s3-website-us-west-2.amazonaws.com",
+        "s3-website.ap-south-1.amazonaws.com",
+        "s3-website.ap-northeast-2.amazonaws.com",
+        "s3-website-ap-southeast-1.amazonaws.com",
+        "s3-website-ap-southeast-2.amazonaws.com",
+        "s3-website-ap-northeast-1.amazonaws.com",
+        "s3-website.eu-central-1.amazonaws.com",
+        "s3-website-eu-west-1.amazonaws.com",
+        "s3-website-sa-east-1.amazonaws.com",
+        "s3-website.localhost",
+        "s3-website.scality.test"
+    ],
+    "vaultd": {
+        "host": "localhost",
+        "port": 8500
+    },
+    "clusters": 1,
+    "log": {
+        "logLevel": "trace",
+        "dumpLevel": "error"
+    },
+    "healthChecks": {
+        "allowFrom": ["127.0.0.1/8", "::1"]
+    },
+    "recordLog": {
+        "enabled": false,
+        "recordLogName": "s3-recordlog"
+    },
+    "requests": {
+        "viaProxy": false,
+        "trustedProxyCIDRs": [],
+        "extractClientIPFromHeader": ""
+    }
+}

.github/s3_and_iam_deployment/docker-compose.yml

@@ -0,0 +1,26 @@
+services:
+  s3:
+    profiles: ['iam_s3']
+    image: ${CLOUDSERVER_IMAGE}
+    network_mode: host
+    environment:
+      # TODO: S3CSI-1 enable IAM with VAULT for auth.
+      # S3VAULT: scality
+      S3_CONFIG_FILE: /conf/config.json
+    command: /bin/sh -c "yarn run mem_backend > /logs/s3/s3.log 2>&1"
+    volumes:
+      - ./cloudserver-config.json:/conf/config.json:ro
+      - ./logs/s3:/logs/s3
+
+  iam:
+    profiles: ['iam_s3']
+    image: ${VAULT_IMAGE}
+    network_mode: host
+    command: /bin/sh -c "chmod 400 tests/utils/keyfile && yarn start > /logs/iam/iam.log 2>&1"
+    environment:
+      VAULT_CONFIG_FILE: /conf/config.json
+      VAULT_DB_BACKEND: LEVELDB
+    volumes:
+      - ./vault-config.json:/conf/config.json:ro
+      - ./data/vaultdb:/data
+      - ./logs/iam:/logs/iam

.github/s3_and_iam_deployment/vault-config.json

@@ -0,0 +1,92 @@
+{
+    "clusters": 1,
+    "healthChecks": {
+        "allowFrom": ["127.0.0.1/8", "::1"]
+    },
+    "interfaces": {
+        "S3": {
+            "address": "0.0.0.0",
+            "port": 8500,
+            "allowFrom": ["0.0.0.0/8", "::1"]
+        },
+        "administration": {
+            "address": "0.0.0.0",
+            "port": 8600
+        },
+        "sts": {
+            "address": "127.0.0.1",
+            "port": 8800
+        },
+        "sso": {
+            "address": "127.0.0.1",
+            "port": 8700,
+            "cert": "./tests/utils/sso/defaultCert.crt",
+            "key": "./tests/utils/sso/defaultCert.key"
+        }
+    },
+    "sso": {
+        "clients": [
+            {
+                "id": "grafana",
+                "secret": "123123",
+                "redirectUri": [
+                    "http://localhost:3000/login/generic_oauth",
+                    "https://localhost:3000/login/generic_oauth"
+                ],
+                "scopes": ["dev", "prod"]
+            }
+        ],
+        "publicClients": [
+            {
+                "id": "identisee",
+                "redirectUri": "https://127.0.0.1:8700/user/info",
+                "scopes": ["admin", "superadmin"]
+            }
+        ],
+        "authCodeTTL": 480,
+        "accessTokenTTL": 7600
+    },
+    "map": ["127.0.0.1:4300", "127.0.0.2:4301", "127.0.0.3:4302", "127.0.0.4:4303", "127.0.0.5:4304"],
+    "keyFilePath": "./tests/utils/keyfile",
+    "adminCredentialsFilePath": "./tests/utils/admincredentials.json.encrypted",
+    "log": {
+        "level": "trace",
+        "dump": "error"
+    },
+    "accountSeeds": [
+        {
+            "role": {
+                "roleName": "scality-role1",
+                "trustPolicy": {
+                    "Version": "2012-10-17",
+                    "Statement": [
+                        {
+                            "Effect": "Allow",
+                            "Principal": { "AWS": "arn:aws:iam::000000000000:user/root" },
+                            "Action": "sts:AssumeRole",
+                            "Condition": {}
+                        }
+                    ]
+                }
+            },
+            "permissionPolicy": {
+                "policyName": "scality-policy1",
+                "policyDocument": {
+                    "Version": "2012-10-17",
+                    "Statement": [
+                        {
+                            "Sid": "FullAccess",
+                            "Effect": "Allow",
+                            "Action": ["s3:*"],
+                            "Resource": ["*"]
+                        }
+                    ]
+                }
+            }
+        }
+    ],
+    "utapi": {
+        "host": "127.0.0.1",
+        "port": 8100
+    }
+}

.github/scrips/wait_for_local_port.bash

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+wait_for_local_port() {
+    local port=$1
+    local timeout=$2
+    local count=0
+    local ret=1
+    echo "waiting for storage-service:$port"
+    while [[ "$ret" -eq "1" && "$count" -lt "$timeout" ]] ; do
+        nc -z -w 1 localhost $port
+        ret=$?
+        if [ ! "$ret" -eq "0" ]; then
+            echo -n .
+            sleep 1
+            count=$(($count+1))
+        fi
+    done
+
+    echo ""
+
+    if [[ "$count" -eq "$timeout" ]]; then
+        echo "Server did not start in less than $timeout seconds. Exiting..."
+        exit 1
+    fi
+
+    echo "Server got ready in ~${count} seconds. Starting test now..."
+}
+
+wait_for_local_port $1 $2

.github/scripts/wait_for_local_port.bash

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+wait_for_local_port() {
+    local port=$1
+    local timeout=$2
+    local count=0
+    local ret=1
+    echo "waiting for storage-service:$port"
+    while [[ "$ret" -eq "1" && "$count" -lt "$timeout" ]] ; do
+        nc -z -w 1 localhost $port
+        ret=$?
+        if [ ! "$ret" -eq "0" ]; then
+            echo -n .
+            sleep 1
+            count=$(($count+1))
+        fi
+    done
+
+    echo ""
+
+    if [[ "$count" -eq "$timeout" ]]; then
+        echo "Server did not start in less than $timeout seconds. Exiting..."
+        exit 1
+    fi
+
+    echo "Server got ready in ~${count} seconds. Starting test now..."
+}
+
+wait_for_local_port $1 $2

.github/workflows/ci-and-e2e-tests.yaml

@@ -0,0 +1,91 @@
+name: CI & E2E Tests
+
+on:
+  push:
+    branches:
+      - '**'
+
+env:
+  CLOUDSERVER_IMAGE: ${{ vars.CLOUDSERVER_IMAGE }}
+  VAULT_IMAGE: ${{ vars.VAULT_IMAGE }}
+  KUBECONFIG: "/home/runner/.kube/config"
+  AWS_ACCESS_KEY_ID: "accessKey1"
+  AWS_SECRET_ACCESS_KEY: "verySecretKey1"
+
+jobs:
+  dev-image:
+    name: Dev Image
+    permissions:
+      contents: read
+      packages: write
+    uses: scality/workflows/.github/workflows/docker-build.yaml@v2
+    with:
+      context: .
+      name: mountpoint-s3-csi-driver
+      namespace: ${{ github.repository_owner }}
+      tag: ${{ github.sha }}
+
+  controller-e2e-tests:
+    name: E2E Controller Tests
+    runs-on: ubuntu-22.04
+    needs: dev-image
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Run Common Setup
+        uses: ./.github/actions/e2e-setup-common
+        with:
+          ref: ${{ github.sha }}
+
+      - name: Run Controller Tests
+        run: make e2e-controller
+
+
+  systemd-mounter-e2e-tests:
+    name: E2E Systemd Mounter Tests
+    runs-on: ubuntu-22.04
+    needs: dev-image
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Run Common Setup
+        uses: ./.github/actions/e2e-setup-common
+        with:
+          ref: ${{ github.sha }}
+
+      - name: Install helm release
+        run: |
+          helm upgrade --install mountpoint-s3-csi-driver --namespace kube-system ./charts/scality-mountpoint-s3-csi-driver --values \
+            ./charts/scality-mountpoint-s3-csi-driver/values.yaml \
+            --set image.repository=ghcr.io/scality/mountpoint-s3-csi-driver \
+            --set image.tag=${{ github.sha }} \
+            --set node.serviceAccount.create=true \
+            --set node.podInfoOnMountCompat.enable=true \
+            --set experimental.podMounter=systemd
+          kubectl rollout status daemonset s3-csi-node -n kube-system --timeout=60s
+          kubectl get pods -A
+          echo "s3-csi-node-image: $(kubectl get daemonset s3-csi-node -n kube-system -o jsonpath="{$.spec.template.spec.containers[:1].image}")"
+
+      - name: Setup IAM and S3 Services
+        run: |-
+          set -e -o pipefail;
+          mkdir -p logs/s3 logs/iam logs/cosi_driver data/vaultdb
+          chown -R runner:docker logs data
+          chmod -R ugo+rwx logs data
+          docker compose --profile iam_s3 up -d --quiet-pull
+          bash ../scripts/wait_for_local_port.bash 8600 30
+          bash ../scripts/wait_for_local_port.bash 8000 30
+        working-directory: .github/s3_and_iam_deployment
+
+      - name: Run E2E Tests (SystemdMounter)
+        run: |
+          set +e
+          pushd tests/e2e-kubernetes
+          KUBECONFIG=/home/runner/.kube/config ginkgo -p -vv -timeout 60m -- --bucket-region=us-east-1 --commit-id=${{ github.sha }} --bucket-prefix=helm-test-cluster
+          EXIT_CODE=$?
+          print_cluster_info
+          popd
+          cat tests/e2e-kubernetes/csi-test-artifacts/output.json
+          exit $EXIT_CODE

.github/workflows/e2e-test-trusted.yaml

@@ -1,11 +1,11 @@
 name: "E2E Tests (Trusted)"
 
-on:
-  push:
-    branches: [ "main", "release-**", "workflow/**" ]
-  merge_group:
-    types: [ "checks_requested" ]
-  pull_request:
+# on:
+#   push:
+#     branches: [ "main", "release-**", "workflow/**" ]
+#   merge_group:
+#     types: [ "checks_requested" ]
+#   pull_request:
 
 permissions:
   id-token: write