[MISC] Migrate to microceph (#155)

Author: Batalex

.github/workflows/ci-gpu.yaml

@@ -103,6 +103,14 @@ jobs:
           snap alias microk8s.kubectl kubectl
           usermod -aG microk8s ubuntu
 
+      - name: Setup microceph
+        run: |
+          sudo snap install microceph
+          sudo microceph cluster bootstrap
+          sudo microceph disk add loop,1G,3
+          sudo microceph enable rgw
+          sudo microceph.radosgw-admin user create --uid test --display-name test --access-key=foo --secret-key=bar
+
       - name: Configure user, microk8s and juju
         run: |
           su ubuntu -c "/bin/bash ./tests/integration/setup/setup-ec2-gpu.sh"
@@ -120,6 +128,8 @@ jobs:
           mv ./** /home/ubuntu/workdir
           chown -R ubuntu /home/ubuntu
           cd /home/ubuntu/workdir
+          echo -e "S3_SERVER_URL=http://$(ip -4 -j route get 2.2.2.2 | yq '.[].prefsrc'):80/\nS3_ACCESS_KEY=foo\nS3_SECRET_KEY=bar" > .env
+          chown ubuntu:ubuntu .env
           su ubuntu -c "~/.local/bin/tox run -e integration-gpu"
 
   stop-runner:

.github/workflows/ci.yaml

@@ -88,10 +88,7 @@ jobs:
           df --human-readable
       - name: Checkout
         uses: actions/checkout@v5
-      - name: Get prefsrc
-        # One IP needed to test external access
-        run: |
-          echo "IPADDR=$(ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc')" >> $GITHUB_ENV
+
       - name: Setup operator environment
         # TODO: Replace with custom image on self-hosted runner
         uses: charmed-kubernetes/actions-operator@main
@@ -101,7 +98,18 @@ jobs:
           provider: microk8s
           channel: 1.32-strict/stable
           microk8s-group: snap_microk8s
-          microk8s-addons: "rbac hostpath-storage dns minio metallb:${{ env.IPADDR }}-${{ env.IPADDR }}"
+          microk8s-addons: "rbac hostpath-storage dns minio metallb:10.64.140.43-10.64.140.49"
+
+      - name: Setup microceph
+        id: microceph
+        run: |
+          sudo snap install microceph
+          sudo microceph cluster bootstrap
+          sudo microceph disk add loop,1G,3
+          sudo microceph enable rgw
+          sudo microceph.radosgw-admin user create --uid test --display-name test --access-key=foo --secret-key=bar
+          echo -e "S3_SERVER_URL=http://$(ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc'):80/\nS3_ACCESS_KEY=foo\nS3_SECRET_KEY=bar" > .env
+
       - name: Download packed charm(s)
         uses: actions/download-artifact@v5
         with:

poetry.lock

@@ -69,6 +69,7 @@ spark-k8s-test = "^0.0.4"
 jubilant = "^1.4.0"
 tomli = "^2.2.1"
 tomli-w = "^1.2.0"
+python-dotenv = "^1.2.1"
 
 [tool.poetry.group.build-refresh-version]
 optional = true

pyproject.toml

@@ -3,6 +3,7 @@
 # See LICENSE file for licensing details.
 
 import logging
+import os
 import subprocess
 from pathlib import Path
 from string import Template
@@ -14,9 +15,11 @@
 import pytest
 import yaml
 from botocore.client import Config
+from dotenv import load_dotenv
 
-from .types import IntegrationTestsCharms, TestCharm
+from .types import IntegrationTestsCharms, S3Info, TestCharm
 
+load_dotenv()
 logger = logging.getLogger(__name__)
 logging.getLogger("jubilant.wait").setLevel(logging.WARNING)
 
@@ -112,23 +115,29 @@ def charm_versions() -> IntegrationTestsCharms:
 
 
 @pytest.fixture(scope="module")
-def s3_bucket_and_creds(request: pytest.FixtureRequest):
+def s3_bucket_and_creds(request: pytest.FixtureRequest) -> Iterable[S3Info]:
     keep_models = bool(request.config.getoption("--keep-models"))
-    logger.info("Fetching S3 credentials from minio.....")
 
-    fetch_s3_output = (
-        subprocess.check_output(
-            "./tests/integration/setup/fetch_s3_credentials.sh | tail -n 3",
-            shell=True,
-            stderr=None,
+    if any(
+        (
+            (access_key := os.environ.get("S3_ACCESS_KEY", None)) is None,
+            (secret_key := os.environ.get("S3_SECRET_KEY", None)) is None,
+            (endpoint_url := os.environ.get("S3_SERVER_URL", None)) is None,
+        )
+    ):
+        logger.info("Fetching S3 credentials from minio.....")
+        fetch_s3_output = (
+            subprocess.check_output(
+                "./tests/integration/setup/fetch_s3_credentials.sh | tail -n 3",
+                shell=True,
+                stderr=None,
+            )
+            .decode("utf-8")
+            .strip()
         )
-        .decode("utf-8")
-        .strip()
-    )
-
-    logger.info(f"fetch_s3_credentials output:\n{fetch_s3_output}")
 
-    endpoint_url, access_key, secret_key = fetch_s3_output.strip().splitlines()
+        logger.info(f"fetch_s3_credentials output:\n{fetch_s3_output}")
+        endpoint_url, access_key, secret_key = fetch_s3_output.strip().splitlines()
 
     session = boto3.session.Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key)
     s3 = session.resource(
@@ -155,13 +164,14 @@ def s3_bucket_and_creds(request: pytest.FixtureRequest):
     # Create the test bucket
     s3.create_bucket(Bucket=TEST_BUCKET_NAME)
     logger.info(f"Created bucket: {TEST_BUCKET_NAME}")
-    test_bucket.put_object(Key=TEST_PATH_NAME)
+    test_bucket.put_object(Key=os.path.join(TEST_PATH_NAME, "touch"))
     yield {
-        "endpoint": endpoint_url,
-        "access_key": access_key,
-        "secret_key": secret_key,
+        "endpoint": str(endpoint_url),
+        "access_key": str(access_key),
+        "secret_key": str(secret_key),
         "bucket": TEST_BUCKET_NAME,
         "path": TEST_PATH_NAME,
+        "ca_bundle_path": os.environ.get("S3_CA_BUNDLE_PATH", ""),
     }
 
     if not keep_models:

tests/integration/conftest.py

@@ -8,7 +8,7 @@ sudo microk8s status --wait-ready
 mkdir ~/.kube
 mkdir ~/workdir
 sudo microk8s config > ~/.kube/config
-sudo microk8s enable hostpath-storage dns rbac nvidia minio
+sudo microk8s enable hostpath-storage dns rbac nvidia
 sudo microk8s status --wait-ready
 
 while ! sudo microk8s.kubectl logs -n gpu-operator-resources -l app=nvidia-operator-validator | grep "all validations are successful"

tests/integration/setup/setup-ec2-gpu.sh

@@ -13,6 +13,7 @@
         "secret_key": str,
         "bucket": str,
         "path": str,
+        "ca_bundle_path": str,
     },
 )