Skip to content

V2 django

V2 django #59

Workflow file for this run

name: CI
on:
push:
branches: [main, develop]
tags: ['v*']
pull_request:
branches: [main, develop]
workflow_dispatch:
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
# ===========================================================================
# Stage 1: Path Detection (runs in parallel with linting)
# ===========================================================================
detect-changes:
name: Detect Changes
runs-on: ubuntu-latest
outputs:
python: ${{ steps.filter.outputs.python }}
frontend: ${{ steps.filter.outputs.frontend }}
go: ${{ steps.filter.outputs.go }}
docker: ${{ steps.filter.outputs.docker }}
steps:
- uses: actions/checkout@v4
- uses: dorny/paths-filter@v3
id: filter
with:
filters: |
python:
- 'skyspy_django/**'
- 'skyspy_common/**'
- 'requirements*.txt'
- 'pyproject.toml'
frontend:
- 'web/**'
- '!web/e2e/**'
- '!web/playwright-report/**'
- '!web/test-results/**'
go:
- 'skyspy-go/**'
docker:
- 'Dockerfile'
- 'docker-compose*.yaml'
- 'docker-compose*.yml'
# ===========================================================================
# Stage 1b: Linting (runs in parallel with path detection)
# ===========================================================================
lint-python:
name: Lint Python
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
- run: uv tool run ruff check --output-format=github .
- run: uv tool run ruff format --check .
lint-frontend:
name: Lint Frontend
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: '22'
cache: 'npm'
cache-dependency-path: web/package-lock.json
- run: npm ci
working-directory: web
- run: npm run lint && npm run format:check
working-directory: web
lint-go:
name: Lint Go
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: '1.22'
cache-dependency-path: skyspy-go/go.sum
- uses: golangci/golangci-lint-action@v6
with:
version: latest
working-directory: skyspy-go
- name: Check gofmt
working-directory: skyspy-go
run: test -z "$(gofmt -l .)"
# ===========================================================================
# Stage 2: Security & Unit Tests (parallel after path detection)
# ===========================================================================
security-python:
name: Security Python
runs-on: ubuntu-latest
needs: detect-changes
if: needs.detect-changes.outputs.python == 'true'
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v5
- name: Run security scans
run: |
uv tool run bandit -r skyspy_django skyspy_common -x "*/tests/*,*/test_*" -ll -ii
# Filter out editable installs (-e) which pip-audit can't resolve in CI
grep -v '^-e ' skyspy_django/requirements.txt > /tmp/requirements-audit.txt
uv tool run pip-audit --requirement /tmp/requirements-audit.txt --ignore-vuln PYSEC-2024-48
security-frontend:
name: Security Frontend
runs-on: ubuntu-latest
needs: detect-changes
if: needs.detect-changes.outputs.frontend == 'true'
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: '22'
cache: 'npm'
cache-dependency-path: web/package-lock.json
- run: npm ci && npm audit --audit-level=high
working-directory: web
test-python-unit:
name: Python Unit Tests
runs-on: ubuntu-latest
needs: detect-changes
if: needs.detect-changes.outputs.python == 'true'
timeout-minutes: 10
services:
postgres:
image: postgres:16-alpine
env:
POSTGRES_USER: adsb
POSTGRES_PASSWORD: adsb
POSTGRES_DB: adsb_test
options: --health-cmd pg_isready --health-interval 5s --health-timeout 3s --health-retries 5
ports: ['5432:5432']
redis:
image: redis:7-alpine
options: --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 3s --health-retries 5
ports: ['6379:6379']
env:
DJANGO_SETTINGS_MODULE: skyspy.tests.test_settings
DJANGO_SECRET_KEY: test-secret-key-for-ci
DATABASE_URL: postgresql://adsb:adsb@localhost:5432/adsb_test
REDIS_URL: redis://localhost:6379/0
FEEDER_LAT: '48.6969'
FEEDER_LON: '-122.6767'
SAFETY_MONITORING_ENABLED: 'true'
PHOTO_CACHE_ENABLED: 'false'
S3_ENABLED: 'false'
WHISPER_ENABLED: 'false'
TRANSCRIPTION_ENABLED: 'false'
OPENSKY_DB_ENABLED: 'false'
LLM_ENABLED: 'false'
PROMETHEUS_ENABLED: 'false'
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- uses: astral-sh/setup-uv@v5
- name: Install dependencies
working-directory: skyspy_django
run: |
uv venv
uv pip install -r requirements.txt
uv pip install pytest pytest-cov pytest-django pytest-asyncio
uv pip install -e ../skyspy_common
- name: Run tests
working-directory: skyspy_django
run: |
source .venv/bin/activate
python manage.py migrate --noinput
pytest -v --tb=short \
--ignore=skyspy/tests/e2e \
--ignore=skyspy/tests/integration \
--cov=skyspy --cov-report=xml --cov-fail-under=80
- uses: codecov/codecov-action@v5
with:
files: skyspy_django/coverage.xml
flags: python-unit
fail_ci_if_error: false
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
test-go:
name: Go Tests
runs-on: ubuntu-latest
needs: detect-changes
if: needs.detect-changes.outputs.go == 'true'
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: '1.22'
cache-dependency-path: skyspy-go/go.sum
- name: Run tests
working-directory: skyspy-go
run: |
go test -v -race -coverprofile=coverage.out -covermode=atomic ./...
COVERAGE=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//')
echo "Coverage: ${COVERAGE}%"
(( $(echo "$COVERAGE >= 70" | bc -l) )) || exit 1
- uses: codecov/codecov-action@v5
with:
files: skyspy-go/coverage.out
flags: go
fail_ci_if_error: false
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
test-frontend-unit:
name: Frontend Unit Tests
runs-on: ubuntu-latest
needs: detect-changes
if: needs.detect-changes.outputs.frontend == 'true'
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: '22'
cache: 'npm'
cache-dependency-path: web/package-lock.json
- run: npm ci
working-directory: web
- run: npm run test:unit:coverage
working-directory: web
- uses: codecov/codecov-action@v5
with:
files: web/coverage/coverage-final.json
flags: frontend
fail_ci_if_error: false
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
build-frontend:
name: Build Frontend
runs-on: ubuntu-latest
needs: detect-changes
if: needs.detect-changes.outputs.frontend == 'true'
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: '22'
cache: 'npm'
cache-dependency-path: web/package-lock.json
- run: npm ci && npm run build
working-directory: web
# ===========================================================================
# Stage 3: Integration Tests
# ===========================================================================
test-python-integration:
name: Python Integration Tests
runs-on: ubuntu-latest
needs: [detect-changes, test-python-unit]
if: needs.detect-changes.outputs.python == 'true'
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: docker/setup-buildx-action@v3
- name: Create test env
run: |
cat > .env.test << 'EOF'
POSTGRES_USER=adsb
POSTGRES_PASSWORD=adsb
POSTGRES_DB=adsb_test
DJANGO_SECRET_KEY=test-secret-key-for-ci
DJANGO_SETTINGS_MODULE=skyspy.tests.test_settings
DEBUG=False
ALLOWED_HOSTS=*
DATABASE_URL=postgresql://adsb:adsb@pgbouncer:5432/adsb_test
REDIS_URL=redis://redis:6379/0
ULTRAFEEDER_HOST=ultrafeeder
ULTRAFEEDER_PORT=80
DUMP978_HOST=dump978
DUMP978_PORT=80
FEEDER_LAT=48.6969
FEEDER_LON=-122.6767
POLLING_INTERVAL=2
DB_STORE_INTERVAL=5
SESSION_TIMEOUT_MINUTES=30
SAFETY_MONITORING_ENABLED=true
ACARS_ENABLED=true
ACARS_PORT=5555
VDLM2_PORT=5556
OPENSKY_DB_ENABLED=false
PHOTO_CACHE_ENABLED=false
S3_ENABLED=false
WHISPER_ENABLED=false
TRANSCRIPTION_ENABLED=false
LLM_ENABLED=false
PROMETHEUS_ENABLED=false
PYTHONDONTWRITEBYTECODE=1
PYTHONUNBUFFERED=1
MOCK_ENABLED=true
MOCK_INTERVAL_MIN=1.0
MOCK_INTERVAL_MAX=2.0
STATION_ID=CI-TEST-STATION
EOF
- name: Build and run tests
run: |
docker buildx bake -f docker-compose.test.yaml \
--set "*.cache-from=type=gha" \
--set "*.cache-to=type=gha,mode=max" \
--load
docker compose -f docker-compose.test.yaml --env-file .env.test up -d --wait postgres pgbouncer redis ultrafeeder dump978
docker compose -f docker-compose.test.yaml --env-file .env.test --profile test run --rm api-test
env:
DOCKER_BUILDKIT: 1
- uses: actions/upload-artifact@v4
if: always()
with:
name: integration-test-results
path: test-results/
retention-days: 7
- uses: codecov/codecov-action@v5
if: success()
with:
files: test-results/coverage.xml
flags: python-integration
fail_ci_if_error: false
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- if: always()
run: docker compose -f docker-compose.test.yaml --env-file .env.test down -v
# ===========================================================================
# Stage 4: E2E Tests
# ===========================================================================
test-e2e:
name: E2E Tests
runs-on: ubuntu-latest
needs: [detect-changes, test-python-integration, build-frontend]
if: needs.detect-changes.outputs.python == 'true' || needs.detect-changes.outputs.frontend == 'true'
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: docker/setup-buildx-action@v3
- name: Create test env
run: |
cat > .env.test << 'EOF'
POSTGRES_USER=adsb
POSTGRES_PASSWORD=adsb
POSTGRES_DB=adsb_test
DJANGO_SECRET_KEY=test-secret-key-for-ci
DJANGO_SETTINGS_MODULE=skyspy.settings
DEBUG=False
ALLOWED_HOSTS=*
DATABASE_URL=postgresql://adsb:adsb@pgbouncer:5432/adsb_test
REDIS_URL=redis://redis:6379/0
ULTRAFEEDER_HOST=ultrafeeder
ULTRAFEEDER_PORT=80
DUMP978_HOST=dump978
DUMP978_PORT=80
FEEDER_LAT=48.6969
FEEDER_LON=-122.6767
POLLING_INTERVAL=2
DB_STORE_INTERVAL=5
SESSION_TIMEOUT_MINUTES=30
SAFETY_MONITORING_ENABLED=true
ACARS_ENABLED=true
ACARS_PORT=5555
VDLM2_PORT=5556
OPENSKY_DB_ENABLED=false
PHOTO_CACHE_ENABLED=false
S3_ENABLED=false
WHISPER_ENABLED=false
TRANSCRIPTION_ENABLED=false
LLM_ENABLED=false
PROMETHEUS_ENABLED=false
PYTHONDONTWRITEBYTECODE=1
PYTHONUNBUFFERED=1
MOCK_ENABLED=true
MOCK_INTERVAL_MIN=1.0
MOCK_INTERVAL_MAX=2.0
STATION_ID=CI-TEST-STATION
EOF
- name: Start backend
run: |
docker buildx bake -f docker-compose.test.yaml \
--set "*.cache-from=type=gha" \
--set "*.cache-to=type=gha,mode=max" \
--load
docker compose -f docker-compose.test.yaml --env-file .env.test --profile dev up -d --wait
env:
DOCKER_BUILDKIT: 1
- uses: actions/setup-node@v4
with:
node-version: '22'
cache: 'npm'
cache-dependency-path: web/package-lock.json
- run: npm ci
working-directory: web
- uses: actions/cache@v4
id: playwright-cache
with:
path: ~/.cache/ms-playwright
key: playwright-${{ runner.os }}-${{ hashFiles('web/package-lock.json') }}
- name: Install Playwright
working-directory: web
run: npx playwright install --with-deps chromium
if: steps.playwright-cache.outputs.cache-hit != 'true'
- name: Install Playwright deps
working-directory: web
run: npx playwright install-deps chromium
if: steps.playwright-cache.outputs.cache-hit == 'true'
- run: npm run test:e2e
working-directory: web
env:
CI: true
- uses: actions/upload-artifact@v4
if: always()
with:
name: playwright-report
path: web/playwright-report/
retention-days: 7
- uses: actions/upload-artifact@v4
if: always()
with:
name: e2e-test-results
path: web/test-results/
retention-days: 7
- if: always()
run: docker compose -f docker-compose.test.yaml --env-file .env.test down -v
# ===========================================================================
# Stage 5: ReadMe Sync (each job has its own concurrency group)
# ===========================================================================
export-openapi:
name: Sync API to ReadMe
runs-on: ubuntu-latest
needs: [detect-changes, test-python-unit]
if: |
github.ref == 'refs/heads/main' &&
github.event_name == 'push' &&
needs.detect-changes.outputs.python == 'true'
timeout-minutes: 10
concurrency:
group: readme-api-sync
cancel-in-progress: false
services:
postgres:
image: postgres:16-alpine
env:
POSTGRES_USER: adsb
POSTGRES_PASSWORD: adsb
POSTGRES_DB: adsb_test
options: --health-cmd pg_isready --health-interval 5s --health-timeout 3s --health-retries 5
ports: ['5432:5432']
redis:
image: redis:7-alpine
options: --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 3s --health-retries 5
ports: ['6379:6379']
env:
DJANGO_SETTINGS_MODULE: skyspy.settings
DJANGO_SECRET_KEY: openapi-export-key
DATABASE_URL: postgresql://adsb:adsb@localhost:5432/adsb_test
REDIS_URL: redis://localhost:6379/0
FEEDER_LAT: '48.6969'
FEEDER_LON: '-122.6767'
SAFETY_MONITORING_ENABLED: 'false'
PHOTO_CACHE_ENABLED: 'false'
S3_ENABLED: 'false'
WHISPER_ENABLED: 'false'
OPENSKY_DB_ENABLED: 'false'
LLM_ENABLED: 'false'
PROMETHEUS_ENABLED: 'false'
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- uses: astral-sh/setup-uv@v5
- name: Install dependencies
working-directory: skyspy_django
run: |
uv venv
uv pip install -r requirements.txt
uv pip install -e ../skyspy_common
- name: Export OpenAPI schema
working-directory: skyspy_django
run: |
source .venv/bin/activate
python manage.py migrate --noinput
python manage.py spectacular --file openapi.json --validate
- name: Upload to ReadMe
uses: readmeio/rdme@v8
with:
rdme: openapi upload skyspy_django/openapi.json --key=${{ secrets.README_API_KEY }} --slug=skyspy-api --confirm-overwrite
sync-docs:
name: Sync Docs to ReadMe
runs-on: ubuntu-latest
needs: [detect-changes, test-python-unit]
if: |
github.ref == 'refs/heads/main' &&
github.event_name == 'push'
timeout-minutes: 5
concurrency:
group: readme-docs-sync
cancel-in-progress: false
steps:
- uses: actions/checkout@v4
- name: Sync documentation to ReadMe
uses: readmeio/rdme@v8
with:
rdme: docs upload docs --key=${{ secrets.README_API_KEY }}
# ===========================================================================
# Stage 6: Docker Build & Push
# ===========================================================================
docker-build-push:
name: Build ${{ matrix.name }}
runs-on: ubuntu-latest
needs: [detect-changes, test-python-unit, test-go, test-python-integration, test-e2e]
if: |
always() &&
(github.event_name == 'push' || github.event_name == 'workflow_dispatch') &&
!contains(needs.*.result, 'failure') &&
!contains(needs.*.result, 'cancelled')
timeout-minutes: 20
permissions:
contents: read
packages: write
id-token: write
attestations: write
strategy:
fail-fast: true
matrix:
include:
- name: API
dockerfile: Dockerfile
image-suffix: ""
context: .
- name: rtl-airband-uploader
dockerfile: rtl-airband-uploader/Dockerfile
image-suffix: -rtl-airband-uploader
context: rtl-airband-uploader
- name: 1090 Mock
dockerfile: test/mock-1090/Dockerfile
image-suffix: -1090-mock
context: test/mock-1090
- name: ACARS Mock
dockerfile: test/acars-mock/Dockerfile
image-suffix: -acarshub-mock
context: test/acars-mock
steps:
- uses: actions/checkout@v4
- uses: docker/setup-qemu-action@v3
- uses: docker/setup-buildx-action@v3
- uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/metadata-action@v5
id: meta
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image-suffix }}
tags: |
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha,prefix={{branch}}-
type=raw,value=latest,enable={{is_default_branch}}
- uses: docker/build-push-action@v6
id: build
with:
context: ${{ matrix.context }}
file: ${{ matrix.dockerfile }}
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
sbom: true
provenance: true
- uses: actions/attest-build-provenance@v2
with:
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ matrix.image-suffix }}
subject-digest: ${{ steps.build.outputs.digest }}
push-to-registry: true
# ===========================================================================
# CI Gate
# ===========================================================================
ci-success:
name: CI Success
runs-on: ubuntu-latest
needs: [lint-python, lint-frontend, lint-go, security-python, security-frontend, test-python-unit, test-go, test-frontend-unit, build-frontend, test-python-integration, test-e2e, export-openapi, sync-docs]
if: always()
steps:
- name: Check results
run: |
RESULTS='${{ toJSON(needs.*.result) }}'
echo "Job results: $RESULTS"
if echo "$RESULTS" | grep -qE '"failure"|"cancelled"'; then
echo "CI failed"
exit 1
fi
echo "CI passed"