[PRA-264] Fix TIOBE workflow (3.4) (#212) #1715
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2024 Canonical Ltd. | |
| # See LICENSE file for licensing details. | |
| name: Tests | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_call: | |
| outputs: | |
| artifact-prefix: | |
| description: build_charm.yaml `artifact-prefix` output | |
| value: ${{ jobs.build.outputs.artifact-prefix }} | |
| pull_request: | |
| schedule: | |
| - cron: "53 0 * * *" # Daily at 00:53 UTC | |
| permissions: | |
| contents: read | |
| jobs: | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Install tox | |
| run: pipx install tox | |
| - name: Run linters | |
| run: tox run -e lint | |
| unit-tests: | |
| name: Unit tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Install tox | |
| run: pipx install tox | |
| - name: Run tests | |
| run: tox run -e unit | |
| build: | |
| name: Build charm | |
| uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v48.1.1 | |
| permissions: | |
| actions: read # Needed for GitHub API call to get workflow version | |
| contents: read | |
| integration-test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| system: | |
| - os: ubuntu-22.04 | |
| arch: amd64 | |
| - os: ubuntu-22.04-arm | |
| arch: arm64 | |
| tox-environments: | |
| - integration-charm | |
| - integration-metastore | |
| - integration-auth | |
| - integration-observability | |
| - integration-tls | |
| - integration-ha | |
| - integration-external-access | |
| - integration-dynamic-allocation | |
| - integration-iceberg | |
| - integration-provider | |
| - integration-trust | |
| exclude: | |
| - tox-environments: integration-charm # TODO(zk-arm): Remove exclusion | |
| system: | |
| os: ubuntu-22.04-arm | |
| arch: arm64 | |
| - tox-environments: integration-observability # No ARM build for obs charms | |
| system: | |
| os: ubuntu-22.04-arm | |
| arch: arm64 | |
| - tox-environments: integration-ha # TODO(zk-arm): Remove exclusion | |
| system: | |
| os: ubuntu-22.04-arm | |
| arch: arm64 | |
| - tox-environments: integration-external-access # TODO(zk-arm): Remove exclusion | |
| system: | |
| os: ubuntu-22.04-arm | |
| arch: arm64 | |
| name: ${{ matrix.tox-environments }} (${{ matrix.system.arch }}) | |
| needs: | |
| - lint | |
| - unit-tests | |
| - build | |
| runs-on: ${{ matrix.system.os }} | |
| timeout-minutes: 120 | |
| steps: | |
| - name: (GitHub hosted) Free up disk space | |
| shell: bash | |
| run: | | |
| printf '\nDisk usage before cleanup\n' | |
| df --human-readable | |
| # Based on https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| rm -r /usr/share/dotnet | |
| rm -r /opt/hostedtoolcache/ | |
| printf '\nDisk usage after cleanup\n' | |
| df --human-readable | |
| - name: Install build deps | |
| if: ${{ matrix.system.os == 'ubuntu-22.04-arm' }} | |
| shell: bash | |
| run: | | |
| sudo apt update | |
| sudo apt install build-essential | |
| sudo apt install libpq-dev | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Setup operator environment | |
| # TODO: Replace with custom image on self-hosted runner | |
| uses: charmed-kubernetes/actions-operator@main | |
| with: | |
| juju-channel: 3.6/stable | |
| provider: microk8s | |
| channel: 1.32-strict/stable | |
| microk8s-group: snap_microk8s | |
| microk8s-addons: "rbac hostpath-storage dns metallb:10.64.140.43-10.64.140.49" | |
| - name: Setup microceph | |
| id: microceph | |
| run: | | |
| export IP=$(ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc') | |
| # Generate self-signed CA and server certificate | |
| mkdir -p /tmp/certs | |
| openssl genrsa -out /tmp/certs/ca.key 2048 | |
| openssl req -x509 -new -nodes -key /tmp/certs/ca.key \ | |
| -days 1024 -out /tmp/certs/ca.crt -outform PEM \ | |
| -subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=$IP" | |
| openssl genrsa -out /tmp/certs/server.key 2048 | |
| openssl req -new -key /tmp/certs/server.key \ | |
| -out /tmp/certs/server.csr \ | |
| -subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=$IP" | |
| echo "subjectAltName = DNS:$IP, IP:$IP" > /tmp/certs/extfile.cnf | |
| openssl x509 -req -in /tmp/certs/server.csr \ | |
| -CA /tmp/certs/ca.crt -CAkey /tmp/certs/ca.key \ | |
| -CAcreateserial -out /tmp/certs/server.crt \ | |
| -days 365 -extfile /tmp/certs/extfile.cnf | |
| # Install and configure microceph with TLS | |
| sudo snap install microceph | |
| sudo microceph cluster bootstrap | |
| sudo microceph disk add loop,1G,3 | |
| SERVER_CRT_B64=$(sudo base64 -w0 /tmp/certs/server.crt) | |
| SERVER_KEY_B64=$(sudo base64 -w0 /tmp/certs/server.key) | |
| sudo microceph enable rgw \ | |
| --ssl-certificate "$SERVER_CRT_B64" \ | |
| --ssl-private-key "$SERVER_KEY_B64" | |
| # Wait for RGW to be ready | |
| for i in $(seq 1 30); do | |
| if curl -sk "https://$IP/" > /dev/null 2>&1; then | |
| echo "RGW is ready" | |
| break | |
| fi | |
| echo "Waiting for RGW... ($i/30)" | |
| sleep 5 | |
| done | |
| sudo microceph.radosgw-admin user create --uid test --display-name test --access-key=foo --secret-key=bar | |
| { | |
| echo "S3_SERVER_URL=https://$IP" | |
| echo "S3_ACCESS_KEY=foo" | |
| echo "S3_SECRET_KEY=bar" | |
| echo "S3_CA_BUNDLE_PATH=/tmp/certs/ca.crt" | |
| } > .env | |
| - name: Download packed charm(s) | |
| uses: actions/download-artifact@v8 | |
| with: | |
| artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} | |
| merge-multiple: true | |
| - id: cache-images | |
| name: Cache Images Locally | |
| run: | | |
| # Download images for avoiding time out | |
| IMAGES=( | |
| $(yq '.resources.kyuubi-image.upstream-source' metadata.yaml) | |
| $(python3 -c "from src.constants import JOB_OCI_IMAGE;print(JOB_OCI_IMAGE, end='')") | |
| ) | |
| for IMAGE in "${IMAGES[@]}"; do | |
| docker pull $IMAGE | |
| docker save $IMAGE -o image.tar | |
| sudo microk8s ctr images import --base-name $IMAGE image.tar | |
| docker rmi $IMAGE | |
| rm image.tar | |
| done | |
| - name: Select tests | |
| id: select-tests | |
| run: | | |
| if [ "${{ github.event_name }}" == "schedule" ] | |
| then | |
| echo Running unstable and stable tests | |
| echo "mark_expression=" >> $GITHUB_OUTPUT | |
| else | |
| echo Skipping unstable tests | |
| echo "mark_expression=not unstable" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Run integration tests | |
| run: tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --keep-models | |
| env: | |
| CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} | |
| - name: Collect logs if job failed | |
| if: ${{ failure() }} | |
| run: | | |
| juju-crashdump | |
| # get pods | |
| kubectl get pods -A | |
| # replay juju-debug logs | |
| sudo snap install yq | |
| for model in $(juju models --format yaml | yq '.models[] | .name'); | |
| do | |
| echo "Model: $model" | |
| if [[ ${model} != *"controller"* ]];then | |
| juju debug-log --model $model --replay | |
| fi | |
| done | |
| # events | |
| kubectl events -A | |
| echo "Done" | |
| integration-gpu-amd: | |
| name: GPU tests (amd64) | |
| needs: | |
| - lint | |
| - unit-tests | |
| - build | |
| uses: ./.github/workflows/integration-gpu.yaml | |
| with: | |
| arch: amd64 | |
| secrets: inherit | |
| integration-gpu-arm: | |
| name: GPU tests (arm64) | |
| # TODO(zk-arm): Missing arm64 support for ZK | |
| # TODO(GH#165): Convert to Canonical K8s | |
| if: false | |
| needs: | |
| - lint | |
| - unit-tests | |
| - build | |
| uses: ./.github/workflows/integration-gpu.yaml | |
| with: | |
| arch: arm64 | |
| secrets: inherit | |
| refresh-tests: | |
| name: Inplace upgrades | |
| needs: | |
| - lint | |
| - unit-tests | |
| - build | |
| uses: ./.github/workflows/refresh-tests.yaml | |
| secrets: inherit |