Merge branch 'develop'

Author: copelco

.github/workflows/deploy.yaml

@@ -2,7 +2,7 @@ name: deploy
 
 on:
   push:
-    branches: [main, develop, CU-868em67g4_Upgrade-to-Django-52]
+    branches: [main, develop, 366-broken-resource-links]
 
 jobs:
   deploy:

Dockerfile

@@ -129,7 +129,7 @@ RUN groupadd --gid $USER_GID $USERNAME \
 #   openssh-client -- for git over SSH
 #   sudo -- to run commands as superuser
 #   vim -- enhanced vi editor for commits
-ENV KUBE_CLIENT_VERSION="v1.32.6"
+ENV KUBE_CLIENT_VERSION="v1.33.0"
 ENV HELM_VERSION="3.18.3"
 ENV POSTGRESQL_CLIENT_VERSION="16"
 RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \

deploy/ansible.cfg

@@ -4,8 +4,8 @@ inventory = inventory
 vault_password_file = ./echo-vault-pass.sh
 
 any_errors_fatal = true
-error_on_undefined_vars = true
 
 # Make errors more readable
 # human-readable stdout/stderr results display
 result_format=yaml
+callback_result_format = yaml

deploy/group_vars/all.yml

@@ -37,7 +37,7 @@ cloudformation_stack_region: '{{ aws_region }}'
 cloudformation_stack_name: "{{ app_name }}-stack"
 cloudformation_stack_termination_protection: true
 cloudformation_stack_template_bucket: "aws-web-stacks-{{ app_name }}"
-cloudformation_stack_template_local_path: '{{ playbook_dir + "/stack/eks-no-nat.yml" }}'
+cloudformation_stack_template_local_path: '{{ playbook_dir + "/stack/eks-no-nat-2.4.0.yml" }}'
 cloudformation_stack_template_parameters:
   PrimaryAZ: "{{ aws_region }}a"
   SecondaryAZ: "{{ aws_region }}b"
@@ -61,6 +61,8 @@ cloudformation_stack_template_parameters:
   AssetsCloudFrontDomain: files.nccopwatch.org
   AssetsCloudFrontCertArn: arn:aws:acm:us-east-1:606178775542:certificate/379950bb-4b29-4308-8418-122674fe1076
   AssetsUseCloudFront: "true"
+  CustomEKSAMI: "" # Optional when CustomAMIImageType is set. Preferrably do not set as AWS will select best optimized image if CustomAMIImageType is set.
+  CustomAMIImageType: AL2023_x86_64_STANDARD
 cloudformation_stack_tags:
   Environment: "{{ app_name }}"
 
@@ -84,9 +86,9 @@ k8s_iam_users: [copelco]
 # Pin ingress-nginx and cert-manager to current versions so future upgrades of this
 # role will not upgrade these charts without your intervention:
 # https://github.com/kubernetes/ingress-nginx/releases
-k8s_ingress_nginx_chart_version: "4.12.3"
+k8s_ingress_nginx_chart_version: "4.13.0"
 # https://github.com/jetstack/cert-manager/releases
-k8s_cert_manager_chart_version: "v1.17.2"
+k8s_cert_manager_chart_version: "v1.18"
 # AWS only:
 # Use the newer load balancer type (NLB). DO NOT edit k8s_aws_load_balancer_type after
 # creating your Service.
@@ -97,7 +99,7 @@ k8s_aws_load_balancer_type: nlb
 # ----------------------------------------------------------------------------
 
 # New Relic Account: [email protected]
-k8s_newrelic_chart_version: "6.0.10"
+k8s_newrelic_chart_version: "6.0.11"
 k8s_newrelic_logging_enabled: true
 k8s_newrelic_license_key: !vault |
   $ANSIBLE_VAULT;1.1;AES256
@@ -113,7 +115,7 @@ k8s_install_descheduler: yes
 # You must set the k8s_descheduler_chart_version to match the Kubernetes
 # node version (0.23.x -> K8s 1.23.x); see:
 # https://github.com/kubernetes-sigs/descheduler#compatibility-matrix
-k8s_descheduler_chart_version: v0.32.2
+k8s_descheduler_chart_version: v0.33.0
 # See values.yaml for options:
 # https://github.com/kubernetes-sigs/descheduler/blob/master/charts/descheduler/values.yaml#L63
 k8s_descheduler_release_values:

deploy/stack/eks-no-nat.yml deploy/stack/eks-no-nat-2.4.0.ymldeploy/stack/eks-no-nat.yml renamed to deploy/stack/eks-no-nat-2.4.0.yml

@@ -50,6 +50,14 @@ Conditions:
     - !Equals
       - !Ref 'DatabaseReplication'
       - 'true'
+  UseCustomAMI: !Not
+    - !Equals
+      - !Ref 'CustomEKSAMI'
+      - ''
+  UseCustomAMIType: !Not
+    - !Equals
+      - !Ref 'CustomAMIImageType'
+      - ''
   EitherCacheCondition: !Or
     - !Condition 'UsingMemcached'
     - !Condition 'UsingRedis'
@@ -193,6 +201,15 @@ Metadata:
           - RedisNumCacheClusters
           - RedisSnapshotRetentionLimit
           - RedisAutomaticFailover
+      - Label:
+          default: Elastic Kubernetes Service (EKS)
+        Parameters:
+          - EnableEksEncryptionConfig
+          - EksPublicAccessCidrs
+          - EksClusterName
+          - CustomEKSAMI
+          - CustomAMIImageType
+          - EksClusterVersion
     ParameterLabels:
       AdministratorIPAddress:
         default: Admin IP Address
@@ -210,6 +227,10 @@ Metadata:
         default: Instance Type
       ContainerVolumeSize:
         default: Root Volume Size
+      CustomAMIImageType:
+        default: Custom AMI Image Type
+      CustomEKSAMI:
+        default: Custom EKS AMI
       CustomerManagedCmkArn:
         default: Customer managed key ARN
       DatabaseAllocatedStorage:
@@ -609,6 +630,14 @@ Parameters:
     Default: '20'
     Description: Size of instance EBS root volume (in GB)
     Type: Number
+  CustomAMIImageType:
+    Default: ''
+    Description: The image type to match the custom AMI. E.g., AL2023_x86_64_STANDARD, AL2_x86_64
+    Type: String
+  CustomEKSAMI:
+    Default: ''
+    Description: Custom AMI ID for EKS node group
+    Type: String
   CustomerManagedCmkArn:
     Default: ''
     Description: KMS CMK ARN to encrypt stack resources (except for public buckets).
@@ -1587,10 +1616,14 @@ Resources:
     DependsOn:
       - EksCluster
     Properties:
+      AmiType: !If
+        - UseCustomAMIType
+        - !Ref 'CustomAMIImageType'
+        - !Ref 'AWS::NoValue'
       ClusterName: !Ref 'EksCluster'
-      DiskSize: !Ref 'ContainerVolumeSize'
-      InstanceTypes:
-        - !Ref 'ContainerInstanceType'
+      LaunchTemplate:
+        Id: !Ref 'NodegroupLaunchTemplate'
+        Version: !GetAtt 'NodegroupLaunchTemplate.LatestVersionNumber'
       NodeRole: !GetAtt 'ContainerInstanceRole.Arn'
       ScalingConfig:
         DesiredSize: !Ref 'DesiredScale'
@@ -1602,6 +1635,29 @@ Resources:
       Tags:
         Key: Value
     Type: AWS::EKS::Nodegroup
+  NodegroupLaunchTemplate:
+    Properties:
+      LaunchTemplateData:
+        BlockDeviceMappings:
+          - DeviceName: /dev/xvda
+            Ebs:
+              DeleteOnTermination: true
+              Encrypted: !Ref 'UseAES256Encryption'
+              KmsKeyId: !If
+                - CmkArnCondition
+                - !Ref 'CustomerManagedCmkArn'
+                - !Ref 'AWS::NoValue'
+              VolumeSize: !Ref 'ContainerVolumeSize'
+              VolumeType: gp3
+        ImageId: !If
+          - UseCustomAMI
+          - !Ref 'CustomEKSAMI'
+          - !Ref 'AWS::NoValue'
+        InstanceType: !Ref 'ContainerInstanceType'
+        MetadataOptions:
+          HttpPutResponseHopLimit: 3
+          HttpTokens: required
+    Type: AWS::EC2::LaunchTemplate
   PrivateAssetsBucket:
     DeletionPolicy: Retain
     Properties:

pyproject.toml

@@ -13,7 +13,7 @@ dependencies = [
     "census>=0.8.24",
     "click>=8.2.0",
     "dj-database-url>=3.0.0",
-    "django>=5.2.0",
+    "django>=5.2.8",
     "django-ckeditor-5>=0.2.15",
     "django-click>=2.4.1",
     "django-crispy-forms>=2.4.0",
@@ -85,7 +85,7 @@ dev = [
     "ipython>=8.32.0",
     "isort>=6.0.0",
     "kubernetes==32.*",
-    "kubernetes-validate~=1.32.0",
+    "kubernetes-validate~=1.33.0",
     "openshift>=0.13.2",
     "pre-commit>=4.1.0",
     "pytest>=8.3.3",

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
-testpaths = nc tsdata
+testpaths = nc tsdata traffic_stops
 python_files = tests.py test_*.py *_tests.py
-addopts = --ds=traffic_stops.settings.test -p no:warnings --cov-config=.coveragerc --cov-fail-under=44 --cov=nc --cov=tsdata --cov-report=html --cov-report=term-missing:skip-covered -vvv
+addopts = --ds=traffic_stops.settings.test -p no:warnings --cov-config=.coveragerc --cov-fail-under=44 --cov=nc --cov=tsdata --cov=traffic_stops --cov-report=html --cov-report=term-missing:skip-covered -vvv

traffic_stops/settings/base.py

@@ -112,10 +112,20 @@ def __init__(self, tz_name=None):
 STATIC_URL = "/static/"
 
 MEDIA_ROOT = os.path.join(BASE_DIR, "media")
-MEDIA_URL = "/media/"
 MEDIA_STORAGE_BUCKET_NAME = os.getenv("MEDIA_STORAGE_BUCKET_NAME", "")
 MEDIA_LOCATION = os.getenv("MEDIA_LOCATION", "")
 MEDIA_S3_CUSTOM_DOMAIN = os.getenv("MEDIA_S3_CUSTOM_DOMAIN", "")
+
+# Set MEDIA_URL based on whether we're using S3 storage
+if MEDIA_S3_CUSTOM_DOMAIN:
+    # When using S3 with custom domain, construct the full URL
+    MEDIA_URL = f"https://{MEDIA_S3_CUSTOM_DOMAIN}/"
+    if MEDIA_LOCATION:
+        MEDIA_URL += f"{MEDIA_LOCATION}/"
+else:
+    # Fall back to local media serving
+    MEDIA_URL = "/media/"
+
 STORAGES = {
     "default": {
         "BACKEND": os.getenv("DEFAULT_FILE_STORAGE", "django.core.files.storage.FileSystemStorage")

traffic_stops/tests/__init__.py

@@ -0,0 +1,62 @@
+"""Tests for settings configuration."""
+
+import os
+
+from unittest import mock
+
+import pytest
+
+
+@pytest.mark.parametrize(
+    "custom_domain,media_location,expected_url",
+    [
+        # With custom domain and no location
+        ("files.nccopwatch.org", "", "https://files.nccopwatch.org/"),
+        # With custom domain and location
+        ("files.nccopwatch.org", "media", "https://files.nccopwatch.org/media/"),
+        # With custom domain and nested location
+        ("files.example.com", "uploads/files", "https://files.example.com/uploads/files/"),
+        # Without custom domain (local development)
+        ("", "", "/media/"),
+        (None, "", "/media/"),
+    ],
+)
+def test_media_url_configuration(custom_domain, media_location, expected_url):
+    """Test that MEDIA_URL is correctly set based on S3 configuration."""
+    # Mock environment variables
+    env_vars = {
+        "MEDIA_S3_CUSTOM_DOMAIN": custom_domain or "",
+        "MEDIA_LOCATION": media_location,
+        "DEFAULT_FILE_STORAGE": "django.core.files.storage.FileSystemStorage",
+    }
+
+    with mock.patch.dict(os.environ, env_vars, clear=False):
+        # Import settings module to trigger configuration
+        # We need to reload to pick up the mocked environment
+        import importlib
+
+        from traffic_stops.settings import base
+
+        importlib.reload(base)
+
+        assert base.MEDIA_URL == expected_url
+
+
+def test_media_url_with_s3_storage():
+    """Test MEDIA_URL when using S3 storage backend."""
+    env_vars = {
+        "MEDIA_S3_CUSTOM_DOMAIN": "files.nccopwatch.org",
+        "MEDIA_LOCATION": "",
+        "DEFAULT_FILE_STORAGE": "traffic_stops.storages.MediaBoto3Storage",
+    }
+
+    with mock.patch.dict(os.environ, env_vars, clear=False):
+        import importlib
+
+        from traffic_stops.settings import base
+
+        importlib.reload(base)
+
+        # When S3 storage is configured with custom domain, MEDIA_URL should be absolute
+        assert base.MEDIA_URL.startswith("https://")
+        assert "files.nccopwatch.org" in base.MEDIA_URL