feat: add simple object storage cleaner script #6673
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - release | |
| pull_request: | |
| jobs: | |
| lint: | |
| if: '! github.event.pull_request.draft' | |
| name: Lint code base | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Pre-commit | |
| env: | |
| # set the `no-commit-to-branch` to be skipped, otherwise the check fails when we merge to `master` branch | |
| SKIP: no-commit-to-branch | |
| uses: pre-commit/[email protected] | |
| test: | |
| name: Run tests | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| django_app: | |
| - authentication | |
| - notifs | |
| - subscription | |
| - core | |
| - filestorage | |
| - __flaky__ | |
| storage: | |
| - default | |
| - legacy_storage | |
| continue-on-error: true | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Check env vars configuration | |
| run: | | |
| scripts/check_envvars.sh | |
| - name: Export the env variables file | |
| run: | | |
| cp .env.example .env | |
| sed -ri 's/^ENVIRONMENT=(.*)/ENVIRONMENT=test/g' .env | |
| sed -ri 's/^COMPOSE_FILE=(.*)/COMPOSE_FILE=\1:docker-compose.override.test.yml/g' .env | |
| eval $(egrep "^[^#;]" .env | xargs -d'\n' -n1 | sed -E 's/(\w+)=(.*)/export \1='"'"'\2'"'"'/g') | |
| cat <<EOF >> .env | |
| STORAGES='{ | |
| "legacy_storage": { | |
| "BACKEND": "qfieldcloud.filestorage.backend.QfcS3Boto3Storage", | |
| "OPTIONS": { | |
| "access_key": "minioadmin", | |
| "secret_key": "minioadmin", | |
| "bucket_name": "qfieldcloud-local-legacy", | |
| "region_name": "", | |
| "endpoint_url": "http://172.17.0.1:8009" | |
| }, | |
| "QFC_IS_LEGACY": true | |
| }, | |
| "webdav": { | |
| "BACKEND": "qfieldcloud.filestorage.backend.QfcWebDavStorage", | |
| "OPTIONS": { | |
| "webdav_url": "http://qfc_webdav_user:qfc_webdav_pwd@webdav", | |
| "public_url": "http://webdav", | |
| "basic_auth": "qfc_webdav_user:qfc_webdav_pwd" | |
| }, | |
| "QFC_IS_LEGACY": false | |
| }, | |
| "default": { | |
| "BACKEND": "qfieldcloud.filestorage.backend.QfcS3Boto3Storage", | |
| "OPTIONS": { | |
| "access_key": "minioadmin", | |
| "secret_key": "minioadmin", | |
| "bucket_name": "qfieldcloud-local", | |
| "region_name": "", | |
| "endpoint_url": "http://172.17.0.1:8009" | |
| }, | |
| "QFC_IS_LEGACY": false | |
| } | |
| }' | |
| EOF | |
| echo "STORAGES_PROJECT_DEFAULT_STORAGE=${{ matrix.storage }}" >> .env | |
| - name: Pull docker containers | |
| run: docker compose pull | |
| - name: Build and run docker containers | |
| run: | | |
| docker compose up -d --build | |
| - name: Initial manage.py commands | |
| run: | | |
| docker compose run app python manage.py makemigrations --no-input --check | |
| docker compose run app python manage.py migrate | |
| docker compose run app python manage.py collectstatic | |
| - name: Run mandatory unit and integration tests | |
| if: matrix.django_app != '__flaky__' | |
| run: | | |
| docker compose run app python manage.py test --keepdb -v2 --exclude-tag="flaky" qfieldcloud.${{ matrix.django_app }} | |
| - name: Run flaky unit and integration tests | |
| if: matrix.django_app == '__flaky__' | |
| run: | | |
| docker compose run app python manage.py test --keepdb -v2 --tag="flaky" qfieldcloud | |
| - name: Run Object Storage Cleaner tests | |
| if: matrix.django_app == 'filestorage' && matrix.storage == 'default' | |
| run: | | |
| docker compose run --rm \ | |
| -e AWS_ACCESS_KEY_ID=minioadmin \ | |
| -e AWS_SECRET_ACCESS_KEY=minioadmin \ | |
| -e AWS_ENDPOINT_URL=http://172.17.0.1:8009 \ | |
| -e AWS_DEFAULT_REGION=us-east-1 \ | |
| -v ${{ github.workspace }}/scripts:/usr/src/scripts \ | |
| app python /usr/src/scripts/object_storage_cleaner/test.py | |
| - name: "failure logs" | |
| if: failure() | |
| run: | | |
| docker compose logs | |
| - name: Post Google Chat message on failure | |
| if: failure() && matrix.django_app != '__flaky__' | |
| uses: julb/action-post-googlechat-message@v1 | |
| with: | |
| message: | | |
| Failed job run for branch `${{ github.head_ref || github.ref_name }}`, check ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} . | |
| gchat_webhook_url: ${{ secrets.GOOGLE_CHAT_WEBHOOK_URL }} | |
| # - name: Setup tmate session | |
| # if: ${{ failure() }} | |
| # uses: mxschmitt/action-tmate@v3 | |
| # timeout-minutes: 30 | |
| # with: | |
| # limit-access-to-actor: true |