[PRA-156] Add ARM support (#164)

* feat: Add ARM build platform

* chore: Update matrix room

* tests: Include platform in juju.deploy calls

* chore: Fix tox refresh script

* tests: Remove minio setup

* ci: Bump actions

* ci: Initial CI run on ARM

* tests: Specify grafana-agent channel, remove cos arch

* fix: Bump spark job OCI resources

* ci: Add missing microceph step in refresh tests

* ci: Install libpq-dev on arm

* fix: Escape matrix link

* ci: Extract inplace upgrade tests

* ci: Rework GPU integration tests

* tests: Use proper arm revisions for charms

* ci: Disable GPU test on ARM

* ci: Migrate back to microk8s for gpu testing

* Partial revert "tests: Include platform in juju.deploy calls"

This partially reverts commit 6badd74.

* tests: Set arch in juju model constraints

* tests: Set grafana-agent-k8s channel to 1/stable

* ci: Fix broken release check due to multiple platforms

* docs: Document job matrix exclusions

* tests: Use common base for data-integrator charm

Author: Batalex

.github/workflows/ci.yaml

@@ -24,38 +24,39 @@ jobs:
     timeout-minutes: 5
     steps:
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
       - name: Install tox
         run: pipx install tox
       - name: Run linters
         run: tox run -e lint
 
-  unit-test:
+  unit-tests:
     name: Unit tests
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
       - name: Install tox
         run: pipx install tox
       - name: Run tests
         run: tox run -e unit
 
   build:
-    strategy:
-      matrix:
-        path:
-          - .
-    name: Build charm | ${{ matrix.path }}
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v35.0.2
+    name: Build charm
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v41.1.0
     with:
-      path-to-charm-directory: ${{ matrix.path }}
+      cache: false
 
   integration-test:
     strategy:
       fail-fast: false
       matrix:
+        system:
+          - os: ubuntu-22.04
+            arch: amd64
+          - os: ubuntu-22.04-arm
+            arch: arm64
         tox-environments:
           - integration-charm
           - integration-metastore
@@ -68,12 +69,34 @@ jobs:
           - integration-iceberg
           - integration-provider
           - integration-trust
-    name: ${{ matrix.tox-environments }}
+
+        exclude:
+          - tox-environments: integration-charm # TODO(zk-arm): Remove exclusion
+            system:
+              os: ubuntu-22.04-arm
+              arch: arm64
+
+          - tox-environments: integration-observability # No ARM build for obs charms
+            system:
+              os: ubuntu-22.04-arm
+              arch: arm64
+
+          - tox-environments: integration-ha # TODO(zk-arm): Remove exclusion
+            system:
+              os: ubuntu-22.04-arm
+              arch: arm64
+
+          - tox-environments: integration-external-access # TODO(zk-arm): Remove exclusion
+            system:
+              os: ubuntu-22.04-arm
+              arch: arm64
+
+    name: ${{ matrix.tox-environments }} (${{ matrix.system.arch }})
     needs:
       - lint
-      - unit-test
+      - unit-tests
       - build
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.system.os }}
     timeout-minutes: 120
     steps:
       - name: (GitHub hosted) Free up disk space
@@ -86,8 +109,17 @@ jobs:
           rm -r /opt/hostedtoolcache/
           printf '\nDisk usage after cleanup\n'
           df --human-readable
+
+      - name: Install build deps
+        if: ${{ matrix.system.os == 'ubuntu-22.04-arm' }}
+        shell: bash
+        run: |
+          sudo apt update
+          sudo apt install build-essential
+          sudo apt install libpq-dev
+
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Setup operator environment
         # TODO: Replace with custom image on self-hosted runner
@@ -98,7 +130,7 @@ jobs:
           provider: microk8s
           channel: 1.32-strict/stable
           microk8s-group: snap_microk8s
-          microk8s-addons: "rbac hostpath-storage dns minio metallb:10.64.140.43-10.64.140.49"
+          microk8s-addons: "rbac hostpath-storage dns metallb:10.64.140.43-10.64.140.49"
 
       - name: Setup microceph
         id: microceph
@@ -108,10 +140,11 @@ jobs:
           sudo microceph disk add loop,1G,3
           sudo microceph enable rgw
           sudo microceph.radosgw-admin user create --uid test --display-name test --access-key=foo --secret-key=bar
-          echo -e "S3_SERVER_URL=http://$(ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc'):80/\nS3_ACCESS_KEY=foo\nS3_SECRET_KEY=bar" > .env
+          export IP=$(ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc')
+          echo -e "S3_SERVER_URL=http://$IP:80/\nS3_ACCESS_KEY=foo\nS3_SECRET_KEY=bar" > .env
 
       - name: Download packed charm(s)
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v7
         with:
           artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}
           merge-multiple: true
@@ -131,6 +164,7 @@ jobs:
             docker rmi $IMAGE
             rm image.tar
           done
+
       - name: Select tests
         id: select-tests
         run: |
@@ -142,10 +176,12 @@ jobs:
             echo Skipping unstable tests
             echo "mark_expression=not unstable" >> $GITHUB_OUTPUT
           fi
+
       - name: Run integration tests
         run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --keep-models
         env:
           CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}
+
       - name: Collect logs if job failed
         if: ${{ failure() }}
         run: |
@@ -168,89 +204,36 @@ jobs:
           kubectl events -A
           echo "Done"
 
-  test-refresh:
-    strategy:
-      fail-fast: false
-      matrix:
-        multi: ["--refresh-multi-units", ""]
-        tls: ["--refresh-tls", ""]
-        image: ["--refresh-image", ""]
-        metastore: ["--refresh-metastore", ""]
-    name: Refresh ${{ matrix.multi }} ${{ matrix.tls }} ${{ matrix.image }} ${{ matrix.metastore }}
+  integration-gpu-amd:
+    name: GPU tests (amd64)
     needs:
       - lint
-      - unit-test
+      - unit-tests
       - build
-    runs-on: ubuntu-22.04
-    timeout-minutes: 120
-    steps:
-      - name: (GitHub hosted) Free up disk space
-        shell: bash
-        run: |
-          printf '\nDisk usage before cleanup\n'
-          df --human-readable
-          # Based on https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
-          rm -r /usr/share/dotnet
-          rm -r /opt/hostedtoolcache/
-          printf '\nDisk usage after cleanup\n'
-          df --human-readable
-      - name: Checkout
-        uses: actions/checkout@v5
-      - name: Setup operator environment
-        # TODO: Replace with custom image on self-hosted runner
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          juju-channel: 3.6/stable
-          bootstrap-options: "--agent-version 3.6.9"
-          provider: microk8s
-          channel: 1.32-strict/stable
-          microk8s-group: snap_microk8s
-          microk8s-addons: "rbac hostpath-storage dns minio"
-      - name: Download packed charm(s)
-        uses: actions/download-artifact@v5
-        with:
-          artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}
-          merge-multiple: true
-      - id: cache-images
-        name: Cache Images Locally
-        run: |
-          # Download images for avoiding time out
-          IMAGES=(
-           $(yq '.resources.kyuubi-image.upstream-source' metadata.yaml)
-           $(python3 -c "from src.constants import JOB_OCI_IMAGE;print(JOB_OCI_IMAGE, end='')")
-           "ghcr.io/canonical/charmed-spark-kyuubi@sha256:e2029c6976fc5b9ee2865eced632ca42ce554039d9832af20dfa3e63113e00f7" # rev 112 charm OCI resource
-           "ghcr.io/canonical/charmed-spark@sha256:f1f944369108c0b0112212fb0242f3c314dfad362926c234857029f13c5de2c0" # rev 112 job OCI resource
-          )
-
-          for IMAGE in "${IMAGES[@]}"; do
-            docker pull $IMAGE
-            docker save $IMAGE -o image.tar
-            sudo microk8s ctr images import --base-name $IMAGE image.tar
-            docker rmi $IMAGE
-            rm image.tar
-          done
-      - name: Run inplace upgrade tests
-        run: tox run -e integration-refresh -- ${{ matrix.multi }} ${{ matrix.tls }} ${{ matrix.image }} ${{ matrix.metastore }} --keep-models
-        env:
-          CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }}
-      - name: Collect logs if job failed
-        if: ${{ failure() }}
-        run: |
-          juju-crashdump
-
-          # get pods
-          kubectl get pods -A
-
-          # replay juju-debug logs
-          sudo snap install yq
-          for model in $(juju models --format yaml | yq '.models[] | .name');
-          do
-            echo "Model: $model"
-            if [[ ${model} != *"controller"* ]];then
-              juju debug-log --model $model --replay
-            fi
-          done
+    uses: ./.github/workflows/integration-gpu.yaml
+    with:
+      arch: amd64
+    secrets: inherit
+
+  integration-gpu-arm:
+    name: GPU tests (arm64)
+    # TODO(zk-arm): Missing arm64 support for ZK
+    # TODO(GH#165): Convert to Canonical K8s
+    if: false
+    needs:
+      - lint
+      - unit-tests
+      - build
+    uses: ./.github/workflows/integration-gpu.yaml
+    with:
+      arch: arm64
+    secrets: inherit
 
-          # events
-          kubectl events -A
-          echo "Done"
+  refresh-tests:
+    name: Inplace upgrades
+    needs:
+      - lint
+      - unit-tests
+      - build
+    uses: ./.github/workflows/refresh-tests.yaml
+    secrets: inherit