Django v4.2 and Package Upgrades

.devcontainer/devcontainer.json

@@ -3,40 +3,30 @@
 // If you want to run as a non-root user in the container, see .devcontainer/docker-compose.yml.
 {
 	"name": "CopWatch",
-
 	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
 	// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
 	"dockerComposeFile": [
 		"../docker-compose.yml",
 		"docker-compose.yml"
 	],
-
 	// The 'service' property is the name of the service for the container that VS Code should
 	// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
 	"service": "django",
-
 	// The optional 'workspaceFolder' property is the path VS Code should open by default when
 	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
 	"workspaceFolder": "/code",
-
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],
-
 	// Uncomment the next line if you want start specific services in your Docker Compose config.
 	// "runServices": [],
-
 	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
 	"shutdownAction": "stopCompose",
-
 	// Uncomment the next line to run commands after the container is created - for example installing curl.
 	"postCreateCommand": "sudo chown appuser /home/appuser/.cache /code/frontend/node_modules /code/venv /code/public /var/run/docker.sock",
-
 	// Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "appuser",
-
 	// A command to run each time the container is successfully started.
 	"postStartCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",
-
 	"customizations": {
 		"vscode": {
 			"extensions": [

.github/workflows/deploy.yaml

@@ -2,7 +2,7 @@ name: deploy
 
 on:
   push:
-    branches: [main, develop, CU-868ee8pa2-import-emails]
+    branches: [main, develop]
 
 jobs:
   deploy:
@@ -24,7 +24,7 @@ jobs:
           echo "ENV_URL=https://nccopwatch.org/" >> $GITHUB_ENV
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.12'
           cache: 'pip'
           cache-dependency-path: 'requirements/*/*.txt'
       - name: Install dependencies

.github/workflows/test.yaml

@@ -28,7 +28,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.12'
           cache: 'pip'
           cache-dependency-path: 'requirements/*/*.txt'
       - name: Install dependencies

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     rev: 24.10.0
     hooks:
       - id: black
-        language_version: python3.10
+        language_version: python3.12
         exclude: migrations
   - repo: https://github.com/PyCQA/flake8
     rev: 7.1.1

Dockerfile

@@ -7,7 +7,7 @@ RUN npm install --silent
 COPY frontend/ /code/
 RUN npm run build
 
-FROM python:3.10-slim-bullseye AS base
+FROM python:3.12-slim-bullseye AS base
 
 # Create a group and user to run our app
 ARG APP_USER=appuser
@@ -98,7 +98,7 @@ ENTRYPOINT ["/code/docker-entrypoint.sh"]
 # Start uWSGI
 CMD ["newrelic-admin", "run-program", "uwsgi", "--single-interpreter", "--enable-threads", "--show-config"]
 
-FROM python:3.10-slim-bullseye AS dev
+FROM python:3.12-slim-bullseye AS dev
 
 ARG USERNAME=appuser
 ARG USER_UID=1000

Makefile

@@ -9,7 +9,7 @@ update_requirements:
 	pip-compile --output-file=requirements/base/base.txt requirements/base/base.in
 	pip-compile --output-file=requirements/test/test.txt requirements/test/test.in
 	pip-compile --output-file=requirements/dev/dev.txt requirements/dev/dev.in
-	pip-compile --upgrade --output-file=requirements/deploy/deploy.txt requirements/deploy/deploy.in
+	pip-compile --output-file=requirements/deploy/deploy.txt requirements/deploy/deploy.in
 
 install_requirements:
 	@echo 'Installing pip-tools...'

docs/conf.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # NC Traffic Stops documentation build configuration file, created by
 # sphinx-quickstart on Sat Sep  6 15:13:52 2014.

docs/dev-setup.rst

@@ -5,7 +5,7 @@ Below you will find basic setup and deployment instructions for the NC Traffic
 Stops project. To begin you should have the following applications installed on
 your local development system:
 
-- Python 3.10
+- Python 3.12
 - NodeJS >= 12.6.0
 - `pip >= 8 or so <http://www.pip-installer.org/>`_
 - Postgres >= 16
@@ -85,8 +85,8 @@ To use ``psql`` locally, make sure you have the following env variables loaded
 To setup your local environment you should create a virtualenv and install the
 necessary requirements::
 
-    $ which python3.11  # make sure you have Python 3.11 installed
-    $ mkvirtualenv --python=`which python3.11` traffic-stops
+    $ which python3.12  # make sure you have Python 3.10 installed
+    $ mkvirtualenv --python=`which python3.10` traffic-stops
     (traffic-stops)$ pip install -U pip
     (traffic-stops)$ make setup
 

nc/admin.py

@@ -1,6 +1,6 @@
-from ckeditor.widgets import CKEditorWidget
 from django import forms
 from django.contrib import admin
+from django_ckeditor_5.widgets import CKEditor5Widget
 
 from nc.models import (
     Agency,
@@ -81,8 +81,8 @@ class InlineResourceFile(admin.StackedInline):
 
 class ResourceForm(forms.ModelForm):
     # https://django-ckeditor.readthedocs.io/en/latest/#widget
-    title = forms.CharField(widget=CKEditorWidget())
-    description = forms.CharField(widget=CKEditorWidget())
+    title = forms.CharField(widget=CKEditor5Widget())
+    description = forms.CharField(widget=CKEditor5Widget())
 
     class Meta:
         model = Resource

nc/data/download_from_nc.py

@@ -31,18 +31,18 @@ def nc_download_and_unzip_data(destination, prefix="state-"):
     # make sure destination exists or create a temporary directory
     if not destination:
         destination = tempfile.mkdtemp(prefix=prefix)
-        logger.debug("Created temp directory {}".format(destination))
+        logger.debug(f"Created temp directory {destination}")
     else:
         if not os.path.exists(destination):
             os.makedirs(destination)
-            logger.info("Created {}".format(destination))
+            logger.info(f"Created {destination}")
     zip_basename = date.today().strftime("NC_STOPS_Extract_%Y_%m_%d.zip")
     zip_filename = os.path.join(destination, zip_basename)
     # don't re-download data if raw data file already exists
     if os.path.exists(zip_filename):
-        logger.debug("{} exists, skipping download".format(zip_filename))
+        logger.debug(f"{zip_filename} exists, skipping download")
     else:
-        logger.debug("Downloading data to {}".format(zip_filename))
+        logger.debug(f"Downloading data to {zip_filename}")
         nc_data_site = settings.NC_FTP_HOST
         nc_data_file = "STOPS_Extract.zip"
         nc_data_directory = "/TSTOPextract"
@@ -52,11 +52,11 @@ def nc_download_and_unzip_data(destination, prefix="state-"):
         listing = ftps.retrlines("LIST", show_ftp_listing)
         line = listing.split("\n")[0]
         if not line.startswith("226 "):  # server's "Transfer complete" message
-            raise ValueError("Expected 226 response from ftp server, got %r" % listing)
+            raise ValueError(f"Expected 226 response from ftp server, got {listing!r}")
         logger.info('Downloading "%s"...', nc_data_file)
         with open(zip_filename, "wb") as f:
-            ftps.retrbinary("RETR %s" % nc_data_file, f.write)
-        logger.info('File written to "%s"' % zip_filename)
+            ftps.retrbinary(f"RETR {nc_data_file}", f.write)
+        logger.info(f'File written to "{zip_filename}"')
 
     unzip_data(destination, zip_path=zip_filename)
     return destination

nc/data/importer.py

@@ -159,8 +159,8 @@ def to_standard_csv(input_path, output_path):
         quoting=csv.QUOTE_MINIMAL,
         skipinitialspace=False,
     )
-    with open(input_path, "rt") as input:
-        with open(output_path, "wt") as output:
+    with open(input_path) as input:
+        with open(output_path, "w") as output:
             reader = csv.reader(input, dialect="nc_data_in")
             writer = csv.writer(output, dialect="nc_data_out")
             headings_written = False
@@ -186,20 +186,20 @@ def convert_to_csv(destination):
             continue
         csv_path = data_path.replace(".txt", ".csv")
         if os.path.exists(csv_path):
-            logger.info("{} already exists, skipping csv conversion".format(csv_path))
+            logger.info(f"{csv_path} already exists, skipping csv conversion")
             continue
-        logger.info("Converting {} > {}".format(data_path, csv_path))
+        logger.info(f"Converting {data_path} > {csv_path}")
         # Edit source data .txt file in-place to remove NUL bytes
         # (only seen in Stop.txt)
-        call([r"sed -i 's/\x0//g' {}".format(data_path)], shell=True)
+        call([rf"sed -i 's/\x0//g' {data_path}"], shell=True)
         to_standard_csv(data_path, csv_path)
         data_count = line_count(data_path)
         csv_count = line_count(csv_path)
         if data_count == (csv_count - 1):
-            logger.debug("CSV line count matches original data file: {}".format(data_count))
+            logger.debug(f"CSV line count matches original data file: {data_count}")
         else:
-            logger.error("DAT {}".format(data_count))
-            logger.error("CSV {}".format(csv_count))
+            logger.error(f"DAT {data_count}")
+            logger.error(f"CSV {csv_count}")
 
 
 def update_nc_agencies(nc_csv_path, destination):

nc/migrations/0001_initial.py

@@ -1,6 +1,3 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations, models
 
 

nc/migrations/0002_agency_census_profile_id.py

@@ -1,6 +1,3 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations, models
 
 

nc/migrations/0003_auto_20180115_1141.py

@@ -1,6 +1,3 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations, models
 
 

nc/models.py

@@ -207,7 +207,7 @@ class Agency(models.Model):
     census_profile_id = models.CharField(max_length=16, blank=True, default="")
     last_reported_stop = models.DateField(null=True)
 
-    class Meta(object):
+    class Meta:
         verbose_name_plural = "Agencies"
 
     def __str__(self):

nc/prime_cache.py

@@ -1,8 +1,8 @@
 import logging
 import time
 
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Generator
 
 import boto3
 import httpx

nc/tests/api/test_arrests.py

@@ -10,7 +10,6 @@
 from nc.tests.urls import reverse_querystring
 from nc.views.arrests import sort_by_stop_purpose
 
-
 class ArrestUtilityTests(TestCase):
     def test_sort_by_stop_purpose(self):
         """Sort DataFrame by stop_purpose column in order of the IntegerChoices"""

nc/tests/api/test_basic_search.py

@@ -8,7 +8,7 @@
 pytestmark = pytest.mark.django_db
 
 
-RACE_VALUES = set([v[0] for v in RACE_CHOICES])
+RACE_VALUES = {v[0] for v in RACE_CHOICES}
 
 
 def test_no_agency(client, search_url):

nc/tests/api/test_timezones.py

@@ -30,7 +30,7 @@ def test_stop_date_after_august_excludes_july_stop(client, search_url, durham, j
         data={"agency": durham.pk, "stop_date_after": dt.date(2020, 8, 1)},
         format="json",
     )
-    stop_ids = set([stop["stop_id"] for stop in response.data["results"]])
+    stop_ids = {stop["stop_id"] for stop in response.data["results"]}
     assert july_person.stop.stop_id not in stop_ids
 
 
@@ -41,7 +41,7 @@ def test_stop_date_after_august_includes_august_stop(client, search_url, durham,
         data={"agency": durham.pk, "stop_date_after": dt.date(2020, 8, 1)},
         format="json",
     )
-    stop_ids = set([stop["stop_id"] for stop in response.data["results"]])
+    stop_ids = {stop["stop_id"] for stop in response.data["results"]}
     assert {august_person.stop.stop_id} == stop_ids
     assert august_person.stop.date == response.data["results"][0]["date"]
 
@@ -53,6 +53,6 @@ def test_stop_date_after_july_includes_both(client, search_url, durham, july_per
         data={"agency": durham.pk, "stop_date_after": dt.date(2020, 7, 1)},
         format="json",
     )
-    stop_ids = set([stop["stop_id"] for stop in response.data["results"]])
+    stop_ids = {stop["stop_id"] for stop in response.data["results"]}
     assert july_person.stop.stop_id in stop_ids
     assert august_person.stop.stop_id in stop_ids

nc/tests/test_api.py

@@ -138,7 +138,7 @@ def test_officer_stops_count(self):
         )
         StopSummary.refresh()
         url = reverse("nc:agency-api-stops", args=[agency.pk])
-        url = "{}?officer={}".format(url, p1.stop.officer_id)
+        url = f"{url}?officer={p1.stop.officer_id}"
         response = self.client.get(url, format="json")
         response_data = response.json()
         self.assertEqual(response.status_code, status.HTTP_200_OK)

nc/urls.py

@@ -1,5 +1,4 @@
-from django.conf.urls import include
-from django.urls import path, re_path
+from django.urls import include, path
 from django.views.decorators.csrf import csrf_exempt
 from rest_framework.routers import DefaultRouter
 
@@ -13,7 +12,7 @@
 
 
 urlpatterns = [  # noqa
-    re_path(r"^api/", include(router.urls)),
+    path("api/", include(router.urls)),
     path("api/about/contact/", csrf_exempt(views.ContactView.as_view()), name="contact-form"),
     path(
         "api/agency/<agency_id>/year-range/",

nc/views/main.py

@@ -1361,6 +1361,7 @@ def get_values(race):
             ],
         }
 
+
     def get(self, request, agency_id):
         stop_qs = StopSummary.objects.all().annotate(year=ExtractYear("date"))
         search_qs = StopSummary.objects.filter(search_type__isnull=False).annotate(

requirements/base/base.in

@@ -1,33 +1,34 @@
 # base requirements.in
-django==3.2.25
+django==4.2.18
 celery==5.4.0
 census==0.8.24
 us
 dealer
 boto
-boto3==1.35.76
-botocore==1.35.76
-click==8.1.7
+boto3
+botocore
+click
 # django-cache-machine is no longer used, remains for legacy migrations
-django-ckeditor==6.7.0
-django-click==2.3.0
+django-ckeditor-5
+django-click
 django-crispy-forms
 django-dotenv
 django-extensions
 django-filter
 django-memoize
-django-pgviews-redux==0.8.0
+
+django-pgviews-redux
 django-redis
-django-storages==1.13.2
-djangorestframework==3.12.4
+django-storages
+djangorestframework
 dj-database-url
-drf-extensions==0.7.1
-psycopg2==2.9.9
-brotli==1.1.0
+drf-extensions
+psycopg2
+brotli
 httpx[http2]
-requests==2.32.3
-urllib3==2.2.1
+requests
+urllib3
 six
 whitenoise
-pandas==2.2.2
-vine==5.1.0
+pandas
+vine