Merge branch 'develop'

Author: ronardcaktus

.dockerignore

@@ -20,3 +20,5 @@ deploy/
 .github
 nc/notebooks
 venv
+.venv
+bin/

.github/workflows/deploy.yaml

@@ -2,52 +2,75 @@ name: deploy
 
 on:
   push:
-    branches: [main, develop]
+    branches: [main, develop, CU-868em67g4_Upgrade-to-Django-52]
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
+    permissions:
+      # Allow GitHub's OIDC provider to create a JSON Web Token for every run
+      # https://github.com/aws-actions/configure-aws-credentials?tab=readme-ov-file#oidc-configuration
+      id-token: write
+      # This is required for actions/checkout
+      contents: read
+      # This is needed for bobheadxi/deployments
+      actions: read
+      deployments: write
     env:
       ENV: staging
       GITHUB_ENV: https://staging.nccopwatch.org/
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      DOCKER_REPO: 606178775542.dkr.ecr.us-east-2.amazonaws.com/traff-appli-gvyudgfsjhrz
       AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
       - name: Set env vars (production)
         if: endsWith(github.ref, '/main')
         run: |
           echo "ENV=production" >> $GITHUB_ENV
           echo "ENV_URL=https://nccopwatch.org/" >> $GITHUB_ENV
-      - uses: actions/setup-python@v4
+      - uses: astral-sh/setup-uv@v6
         with:
-          python-version: '3.12'
-          cache: 'pip'
-          cache-dependency-path: 'requirements/*/*.txt'
+          enable-cache: true
       - name: Install dependencies
         run: |
-          python -m pip install pip-tools
-          pip-sync requirements/base/base.txt requirements/dev/dev.txt
+          uv sync --locked
       - name: Start deployment
         uses: bobheadxi/deployments@v1
         id: deployment
         with:
           step: start
           token: ${{ secrets.GITHUB_TOKEN }}
           env: ${{ env.ENV }}
+      - name: Set DOCKER_TAG variable
+        run: |
+          echo DOCKER_TAG="$ENV-$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: us-east-2
+          role-to-assume: arn:aws:iam::606178775542:role/GitHubActionsDeploy
+          role-session-name: GitHubActionsDeploy
       - name: Login to Docker
         id: docker-login
         run: |
           echo "env is $ENV"
-          inv $ENV aws.docker-login
-      - name: Build, tag, push, and deploy image
-        id: build-tag-push-deploy
+          uv run inv $ENV aws.docker-login
+      - name: Build image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          pull: true
+          push: true
+          tags: "${{ env.DOCKER_REPO }}:${{ env.DOCKER_TAG }}"
+          target: "deploy"
+      - name: Deploy image
         run: |
           echo "env is $ENV"
-          inv $ENV image deploy --verbosity=0
+          uv run inv $ENV deploy --tag=${{ env.DOCKER_TAG }} --verbosity=0
       - name: Update deployment status
         uses: bobheadxi/deployments@v1
         if: always()

.github/workflows/test.yaml

@@ -3,7 +3,7 @@ name: test
 on:
   pull_request:
   push:
-    branches: [main, develop, actions-cd]
+    branches: [main, develop]
 
 jobs:
   tests:
@@ -25,30 +25,28 @@ jobs:
         ports:
           - 5432:5432
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v5
+      - uses: astral-sh/setup-uv@v6
         with:
-          python-version: '3.12'
-          cache: 'pip'
-          cache-dependency-path: 'requirements/*/*.txt'
+          enable-cache: true
       - name: Install dependencies
         run: |
           sudo apt update
           sudo apt install -y --no-install-recommends postgresql-client
-          python -m pip install pip-tools
-          pip-sync requirements/base/base.txt requirements/dev/dev.txt requirements/test/test.txt
+          uv sync --locked
+      - run: uv run pre-commit run --show-diff-on-failure --color=always --all-files
       - name: Create NC database
         run: |
           psql $DATABASE_URL -c 'CREATE DATABASE traffic_stops_nc;'
         env:
           DATABASE_URL: postgres://postgres:postgres@localhost:5432/traffic_stops
       - name: Run tests
         run: |
-           pytest
+           uv run pytest
         env:
           DJANGO_SETTINGS_MODULE: traffic_stops.settings.dev
           DATABASE_URL: postgres://postgres:postgres@localhost:5432/traffic_stops
           DATABASE_URL_NC: postgres://postgres:postgres@localhost:5432/traffic_stops_nc
       - name: Test deploy image build
         run: |
-          inv image.build
+          uv run inv image.build

.pre-commit-config.yaml

@@ -1,9 +1,14 @@
 repos:
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    # uv version.
+    rev: 0.8.9
+    hooks:
+      - id: uv-lock
   - repo: https://github.com/psf/black
     rev: 24.10.0
     hooks:
       - id: black
-        language_version: python3.12
+        language_version: python3.13
         exclude: migrations
   - repo: https://github.com/PyCQA/flake8
     rev: 7.1.1
@@ -30,3 +35,12 @@ repos:
       - id: check-merge-conflict
       - id: debug-statements
       - id: detect-private-key
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.20.0
+    hooks:
+      - id: pyupgrade
+  - repo: https://github.com/adamchainz/django-upgrade
+    rev: 1.25.0
+    hooks:
+      - id: django-upgrade
+        args: [--target-version, "5.2"]

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.17.0-bullseye-slim AS static_files
+FROM node:22.18.0-bullseye-slim AS static_files
 
 WORKDIR /code
 ENV PATH=/code/node_modules/.bin:$PATH
@@ -7,7 +7,17 @@ RUN npm install --silent
 COPY frontend/ /code/
 RUN npm run build
 
-FROM python:3.12-slim-bullseye AS base
+FROM ghcr.io/astral-sh/uv:python3.13-bookworm AS base
+
+# Enable bytecode compilation
+ENV UV_COMPILE_BYTECODE=1
+
+# Copy from the cache instead of linking since it's a mounted volume
+ENV UV_LINK_MODE=copy
+
+# Use a custom VIRTUAL_ENV with uv to avoid conflicts with local developer's
+# .venv/ while running tests in Docker
+ENV VIRTUAL_ENV=/venv
 
 # Create a group and user to run our app
 ARG APP_USER=appuser
@@ -32,27 +42,28 @@ RUN set -ex \
     && apt-get update && apt-get install -y --no-install-recommends $RUN_DEPS \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy in your requirements file
-# ADD requirements.txt /requirements.txt
-
-# OR, if you're using a directory for your requirements, copy everything (comment out the above and uncomment this if so):
-ADD requirements /requirements
-
 # Install build deps, then run `pip install`, then remove unneeded build deps all in a single step.
 # Correct the path to your production requirements file, if needed.
-RUN set -ex \
+ARG UV_OPTS="--no-dev --group deploy"
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    set -ex \
     && BUILD_DEPS=" \
     build-essential \
     libpcre3-dev \
     libpq-dev \
     git-core \
     " \
     && apt-get update && apt-get install -y --no-install-recommends $BUILD_DEPS \
-    && pip install -U -q pip-tools \
-    && pip-sync requirements/base/base.txt requirements/deploy/deploy.txt \
+    && uv venv $VIRTUAL_ENV \
+    && uv sync --active --locked --no-install-project $UV_OPTS \
     && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false $BUILD_DEPS \
     && rm -rf /var/lib/apt/lists/*
 
+# Add uv venv to PATH
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
 FROM base AS deploy
 
 # Copy your application code to the container (make sure you create a .dockerignore file if any large files or directories should be excluded)
@@ -98,7 +109,7 @@ ENTRYPOINT ["/code/docker-entrypoint.sh"]
 # Start uWSGI
 CMD ["newrelic-admin", "run-program", "uwsgi", "--single-interpreter", "--enable-threads", "--show-config"]
 
-FROM python:3.12-slim-bullseye AS dev
+FROM ghcr.io/astral-sh/uv:python3.13-bookworm AS dev
 
 ARG USERNAME=appuser
 ARG USER_UID=1000
@@ -153,8 +164,10 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/
     # docker
     && curl https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor | tee /etc/apt/trusted.gpg.d/docker.gpg >/dev/null \
     && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \
-    # nodejs
-    && sh -c 'echo "deb https://deb.nodesource.com/node_18.x $(lsb_release -cs) main" > /etc/apt/sources.list.d/nodesource.list' \
+    # nodejs (https://github.com/nodesource/distributions/wiki/Repository-Manual-Installation#debian-systems)
+    && mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && sh -c 'echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_22.x nodistro main" > /etc/apt/sources.list.d/nodesource.list' \
     && wget --quiet -O- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
     # PostgreSQL
     && sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \

deploy/ansible.cfg

@@ -8,4 +8,4 @@ error_on_undefined_vars = true
 
 # Make errors more readable
 # human-readable stdout/stderr results display
-stdout_callback = yaml
+result_format=yaml

deploy/group_vars/all.yml

@@ -97,7 +97,7 @@ k8s_aws_load_balancer_type: nlb
 # ----------------------------------------------------------------------------
 
 # New Relic Account: [email protected]
-k8s_newrelic_chart_version: "6.0.1"
+k8s_newrelic_chart_version: "6.0.10"
 k8s_newrelic_logging_enabled: true
 k8s_newrelic_license_key: !vault |
   $ANSIBLE_VAULT;1.1;AES256

docs/deploy.rst

@@ -67,15 +67,13 @@ Create PostgreSQL database
 
 3. Create environment-specific databases, e.g.::
 
-```sql
     CREATE ROLE trafficstops_staging WITH LOGIN NOSUPERUSER INHERIT CREATEDB NOCREATEROLE NOREPLICATION PASSWORD '<password>';
     CREATE DATABASE trafficstops_staging;
     GRANT CONNECT ON DATABASE trafficstops_staging TO trafficstops_staging;
     GRANT ALL PRIVILEGES ON DATABASE trafficstops_staging TO trafficstops_staging;
     CREATE DATABASE trafficstops_staging_nc;
     GRANT CONNECT ON DATABASE trafficstops_staging_nc TO trafficstops_staging;
     GRANT ALL PRIVILEGES ON DATABASE trafficstops_staging_nc TO trafficstops_staging;
-```
 
 
 Configure cluster for deploying web applications

docs/dev-setup.rst

@@ -5,9 +5,9 @@ Below you will find basic setup and deployment instructions for the NC Traffic
 Stops project. To begin you should have the following applications installed on
 your local development system:
 
-- Python 3.12
-- NodeJS >= 12.6.0
-- `pip >= 8 or so <http://www.pip-installer.org/>`_
+- Python 3.13
+- NodeJS >= 22
+- `Install uv <https://docs.astral.sh/uv/getting-started/installation/>`_
 - Postgres >= 16
 
 Getting Started (Docker 🐳)
@@ -22,7 +22,8 @@ pack for VS Code.
 2. **Install Python requirements:** Create virtual environment and install Python requirements::
 
     python3 -m venv /code/venv
-    make setup
+    uv python install
+    uv sync --locked
     (cd frontend; npm install)
 
 3. **Setup pre-commit:** Install pre-commit to enforce a variety of community standards::
@@ -85,10 +86,14 @@ To use ``psql`` locally, make sure you have the following env variables loaded
 To setup your local environment you should create a virtualenv and install the
 necessary requirements::
 
-    $ which python3.12  # make sure you have Python 3.10 installed
-    $ mkvirtualenv --python=`which python3.10` traffic-stops
-    (traffic-stops)$ pip install -U pip
-    (traffic-stops)$ make setup
+    $ uv python install
+    $ uv sync --locked
+
+Set up ``uv`` and ``direnv``::
+
+    echo "PATH_add .venv/bin/" >> .envrc
+    echo "uv sync --locked" >> .envrc
+    direnv allow
 
 Next, we'll set up our local environment variables. We use `django-dotenv
 <https://github.com/jpadilla/django-dotenv>`_ to help with this. It reads environment variables

docs/index.rst

@@ -28,6 +28,8 @@ Contents:
    dev-setup
    data-import
    api
+   census
+   deploy
 
 
 Indices and tables