[#2917] Add SSD mock service to docker stack #15225
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - develop | |
| tags: | |
| - '**' | |
| pull_request: | |
| workflow_dispatch: | |
| env: | |
| # Must match the `web` service's `image:` in docker-compose.yml -- that's | |
| # what `bin/stack.sh up` actually builds and tags, and what the docker commands | |
| # below reference. | |
| IMAGE_NAME: maykinmedia/open-inwoner | |
| permissions: {} | |
| jobs: | |
| # | |
| # Unit and integration tests | |
| # | |
| tests: | |
| name: Run the Django test suite | |
| runs-on: | |
| ${{ matrix.test-type == 'main' && 'maykin-runner' || 'ubuntu-latest' }} | |
| permissions: | |
| contents: read | |
| strategy: | |
| matrix: | |
| test-type: [main, elastic, migrations] | |
| services: | |
| postgres: | |
| image: postgis/postgis:15-3.4 | |
| env: | |
| POSTGRES_HOST_AUTH_METHOD: trust | |
| ports: | |
| - 5432:5432 | |
| # needed because the postgres container does not provide a healthcheck | |
| options: | |
| --health-cmd pg_isready --health-interval 10s --health-timeout 5s | |
| --health-retries 5 --name postgres | |
| elasticsearch: | |
| image: elasticsearch:9.0.3 | |
| env: | |
| discovery.type: single-node | |
| ES_JAVA_OPTS: '-Xms512m -Xmx512m' | |
| xpack.security.enabled: true | |
| ELASTIC_PASSWORD: elastic | |
| ports: | |
| - 9200:9200 | |
| - 9300:9300 | |
| redis: | |
| image: redis:6 | |
| ports: | |
| - 6379:6379 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Set up backend environment | |
| uses: maykinmedia/setup-django-backend@4a6c5facc65bb1dd510cf06be20871db11a156b3 # v1.4.2 | |
| with: | |
| apt-packages: | |
| 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext | |
| postgresql-client libgdal-dev gdal-bin' | |
| python-version: '3.13' | |
| optimize-postgres: 'yes' | |
| pg-service: 'postgres' | |
| setup-node: 'yes' | |
| npm-ci-flags: '--legacy-peer-deps' | |
| - name: Run tests | |
| run: | | |
| if [ "${{ matrix.test-type }}" = "main" ]; then | |
| coverage run -p --concurrency=multiprocessing \ | |
| src/manage.py test src \ | |
| --parallel \ | |
| --exclude-tag=e2e \ | |
| --exclude-tag=elastic \ | |
| --exclude-tag=migrations \ | |
| -v 2 | |
| elif [ "${{ matrix.test-type }}" = "elastic" ]; then | |
| coverage run -p src/manage.py test src \ | |
| --tag=elastic \ | |
| --exclude-tag=e2e \ | |
| --exclude-tag=migrations \ | |
| -v 2 | |
| elif [ "${{ matrix.test-type }}" = "migrations" ]; then | |
| coverage run -p --concurrency=multiprocessing \ | |
| src/manage.py test src --tag=migrations --parallel 2 -v 2 | |
| else | |
| echo "Error: Unknown test type '${{ matrix.test-type }}'" | |
| exit 1 | |
| fi | |
| env: | |
| DJANGO_SETTINGS_MODULE: open_inwoner.conf.ci | |
| SECRET_KEY: dummy | |
| DB_USER: postgres | |
| DB_PASSWORD: '' | |
| ES_USERNAME: elastic | |
| ES_PASSWORD: elastic | |
| OTEL_SDK_DISABLED: true | |
| - name: Persist coverage data files | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: ${{ matrix.test-type }} | |
| path: .coverage* | |
| include-hidden-files: true | |
| if-no-files-found: error | |
| upload_coverage: | |
| needs: tests | |
| name: Upload coverage information to codecov | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Set up Git repository | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version-file: 'pyproject.toml' | |
| - name: Install coverage.py | |
| # We only need coverage, but we need to match the version that was used | |
| # to generate the coverage files. Grab it from the dependencies. | |
| run: grep "^coverage" requirements/ci.txt | xargs -r pip install | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| path: coverages | |
| - name: Combine the coverage files | |
| run: | | |
| mv coverages/main/.coverage* . | |
| mv coverages/elastic/.coverage* . | |
| mv coverages/migrations/.coverage* . | |
| coverage combine | |
| - name: Publish coverage report | |
| uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| # | |
| # End-to-end tests | |
| # | |
| e2etests: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| matrix: | |
| browser: | |
| - chromium | |
| - firefox | |
| - webkit | |
| - msedge | |
| name: End-to-end tests, ${{ matrix.browser }} | |
| services: | |
| postgres: | |
| image: postgis/postgis:15-3.4 | |
| env: | |
| POSTGRES_HOST_AUTH_METHOD: trust | |
| ports: | |
| - 5432:5432 | |
| # Needed because the postgres container does not provide a healthcheck | |
| options: | |
| --health-cmd pg_isready --health-interval 10s --health-timeout 5s | |
| --health-retries 5 --name postgres | |
| elasticsearch: | |
| image: elasticsearch:9.0.3 | |
| env: | |
| discovery.type: single-node | |
| ES_JAVA_OPTS: '-Xms512m -Xmx512m' | |
| xpack.security.enabled: true | |
| ELASTIC_PASSWORD: elastic | |
| ports: | |
| - 9200:9200 | |
| - 9300:9300 | |
| clamav: | |
| image: clamav/clamav:stable | |
| ports: | |
| - 3310:3310 | |
| options: >- | |
| --health-cmd "/usr/local/bin/clamdcheck.sh" --health-interval 60s | |
| --health-timeout 30s --health-retries 5 --health-start-period 120s | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Set up backend environment | |
| uses: maykinmedia/setup-django-backend@4a6c5facc65bb1dd510cf06be20871db11a156b3 # v1.4.2 | |
| with: | |
| apt-packages: | |
| 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext | |
| postgresql-client libgdal-dev gdal-bin' | |
| python-version: '3.13' | |
| optimize-postgres: 'yes' | |
| pg-service: 'postgres' | |
| setup-node: 'yes' | |
| npm-ci-flags: '--legacy-peer-deps' | |
| # Keycloak backs the `keycloak`-tagged OIDC login tests, which only run | |
| # under chromium: the authorization-code flow relies on cross-origin | |
| # cookies that Firefox/WebKit/Edge drop by default (see docs/testing.rst). | |
| # So we only boot Keycloak on the chromium leg and skip the tag elsewhere. | |
| - name: Start Keycloak (imports the test realm) | |
| if: matrix.browser == 'chromium' | |
| run: | | |
| bin/ensure_dev_network.sh | |
| docker compose -f docker/docker-compose.keycloak.yml --project-directory docker up -d | |
| echo "Waiting for Keycloak to import the test realm..." | |
| for i in $(seq 1 45); do | |
| if curl -sf http://localhost:8080/realms/test/.well-known/openid-configuration >/dev/null; then | |
| echo "Keycloak is ready"; exit 0 | |
| fi | |
| sleep 2 | |
| done | |
| echo "Keycloak did not become ready in time" | |
| docker compose -f docker/docker-compose.keycloak.yml --project-directory docker logs | |
| exit 1 | |
| - name: Install playwright deps | |
| run: playwright install --with-deps ${{ matrix.browser }} | |
| - name: Run testsuite | |
| run: | | |
| python src/manage.py collectstatic --noinput --link | |
| if [ "${{ matrix.browser }}" = "chromium" ]; then | |
| src/manage.py test src --tag=e2e | |
| else | |
| src/manage.py test src --tag=e2e --exclude-tag=keycloak | |
| fi | |
| env: | |
| DJANGO_SETTINGS_MODULE: open_inwoner.conf.ci | |
| SECRET_KEY: dummy | |
| DB_USER: postgres | |
| DB_PASSWORD: '' | |
| E2E_DRIVER: ${{ matrix.browser }} | |
| E2E_KEYCLOAK_REALM_URL: http://localhost:8080/realms/test | |
| ES_USERNAME: elastic | |
| ES_PASSWORD: elastic | |
| OTEL_SDK_DISABLED: true | |
| CLAMAV_HOST: localhost | |
| CLAMAV_PORT: 3310 | |
| - name: Upload test results | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: always() | |
| with: | |
| name: e2e-test-results-${{ matrix.browser }} | |
| path: test-results/ | |
| retention-days: 7 | |
| # | |
| # Full application stack | |
| # | |
| full-app-stack-test: | |
| name: Full Docker application stack | |
| runs-on: maykin-runner # Many containers, much resources | |
| timeout-minutes: 20 | |
| permissions: | |
| contents: read | |
| outputs: | |
| tag: ${{ steps.vars.outputs.tag }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Set tag | |
| id: vars | |
| run: | | |
| # Strip git ref prefix from version | |
| VERSION=$(echo "${GITHUB_REF}" | sed -e 's,.*/\(.*\),\1,') | |
| # Strip "v" prefix from tag name (if present at all) | |
| [[ "${GITHUB_REF}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') | |
| # Use Docker `latest` tag convention | |
| [ "$VERSION" == "develop" ] && VERSION=latest | |
| echo ::set-output name=tag::${VERSION} | |
| # `bin/stack.sh up` always runs `docker compose build web`. We use | |
| # TAG/COMMIT_HASH to ensure that build produces exactly the image the | |
| # `docker` job will push, tagged the same way, so this is the only build: | |
| # no separate/duplicate build later. | |
| - name: Bring up the full stack | |
| env: | |
| TAG: ${{ steps.vars.outputs.tag }} | |
| COMMIT_HASH: ${{ github.sha }} | |
| run: bin/stack.sh up | |
| - name: Wait for the app containers to come up healthy | |
| run: | | |
| # $2 = "healthy" to require a passing healthcheck, "running" to | |
| # just require the container hasn't crashed. | |
| wait_for() { | |
| local service="$1" want="$2" cid status | |
| for i in $(seq 1 60); do | |
| cid=$(bin/stack.sh ps -q "$service") | |
| if [ -n "$cid" ]; then | |
| status=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$cid") | |
| echo "$service: $status" | |
| [ "$status" = "$want" ] && return 0 | |
| case "$status" in exited | restarting) | |
| echo "$service crashed (status: $status)" >&2 | |
| return 1 | |
| ;; | |
| esac | |
| fi | |
| sleep 5 | |
| done | |
| echo "$service never reached '$want'" >&2 | |
| return 1 | |
| } | |
| # web, celery, celery-low-latency and celery-beat all have fast | |
| # healthchecks -- celery-beat's via the "Beat health sentinel" | |
| # periodic task (conf/base.py), which keeps its liveness file | |
| # fresh every minute. Wait for a real "healthy" on all four. | |
| wait_for web healthy | |
| wait_for celery healthy | |
| wait_for celery-low-latency healthy | |
| wait_for celery-beat healthy | |
| # celery-monitor has no healthcheck at all, so it can only ever | |
| # report "running" (docker inspect never returns a Health.Status | |
| # for it) -- just needs to still be running, not crashed. | |
| wait_for celery-monitor running | |
| - name: Save the image for the docker job to push | |
| env: | |
| TAG: ${{ steps.vars.outputs.tag }} | |
| run: docker save "$IMAGE_NAME:$TAG" | gzip > open-inwoner-image.tar.gz | |
| - name: Upload the image | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: docker-image | |
| path: open-inwoner-image.tar.gz | |
| if-no-files-found: error | |
| retention-days: 1 | |
| - name: Dump service status and logs on failure | |
| if: failure() | |
| run: | | |
| bin/stack.sh ps | |
| bin/stack.sh logs --no-color --tail=200 | |
| - name: Tear down | |
| if: always() | |
| run: bin/stack.sh down -v | |
| # | |
| # Docs | |
| # | |
| docs: | |
| name: Build and check documentation | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| submodules: 'true' | |
| persist-credentials: false | |
| - name: Set up backend environment | |
| uses: maykinmedia/setup-django-backend@4a6c5facc65bb1dd510cf06be20871db11a156b3 # v1.4.2 | |
| with: | |
| apt-packages: | |
| 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext libgdal-dev | |
| gdal-bin graphviz' | |
| python-version: '3.13' | |
| setup-node: 'no' | |
| - name: Build and test docs | |
| run: | | |
| export OPENSSL_CONF=$(pwd)/openssl.conf | |
| pytest check_sphinx.py -v --tb=auto | |
| working-directory: docs | |
| env: | |
| DJANGO_SETTINGS_MODULE: open_inwoner.conf.ci | |
| # | |
| # Docker | |
| # | |
| docker: | |
| needs: [tests, full-app-stack-test] | |
| if: github.event_name == 'push' | |
| name: Push Docker image | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| env: | |
| TAG: ${{ needs.full-app-stack-test.outputs.tag }} | |
| steps: | |
| # The image itself was already built (and proven healthy), so nothing | |
| # here rebuilds it. | |
| - name: Download the image built by full-app-stack-test | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: docker-image | |
| - name: Load the image | |
| run: gunzip -c open-inwoner-image.tar.gz | docker load | |
| - name: Log into registry | |
| run: | |
| echo "${{ secrets.DOCKER_TOKEN }}" | docker login -u ${{ | |
| secrets.DOCKER_USERNAME }} --password-stdin | |
| - name: Push the Docker image | |
| run: docker push "$IMAGE_NAME:$TAG" |