Merge pull request #2842 from bcgov/DEP-261-object-storage-normalize-urls

DEP-261: update object storage service to normalize URLs to keys

Author: NatSquared

CHANGELOG.MD

@@ -1,3 +1,10 @@
+## April 30, 2026
+
+- **Feature** Updated all S3 uploaded objects to dynamically generate document URLs from the bucket and object key, rather than storing the full URL in the database. [🎟️ DEP-261](https://citz-gdx.atlassian.net/browse/DEP-261)
+  - Updated the API to generate S3 object URLs on-the-fly using the bucket name and object key, rather than storing the full URL in the database. This allows for more flexibility in changing storage configurations in the future without needing to update existing database records.
+  - Updated relevant API endpoints to reflect this change, including those for retrieving engagement details, survey details, and any other endpoints that return data with S3 object references.
+  - Added basic unit tests for S3 service and URL generation test to ensure that URLs are generated correctly.
+
 ## April 22, 2026
 
 - **Bugfix** Move confirmation logic out of subscribe widget [🎟️ DEP-243](https://citz-gdx.atlassian.net/browse/DEP-243)
@@ -6,7 +13,7 @@
   - Updated notify-api sample.env with all needed values (many were previously missing)
   - Updated notify-api make scripts to remove update from install commands, which was breaking functionality
   - Removed subscribe widget as an option from widget selector and removed associated tables via migration
-      - This completes the work for [🎟️ DEP-241](https://citz-gdx.atlassian.net/browse/DEP-241)
+    - This completes the work for [🎟️ DEP-241](https://citz-gdx.atlassian.net/browse/DEP-241)
   - Added Keycloak environment variable fallbacks to the API that are compatible with Vault secret injection.
 
 ## April 20, 2026

api/migrations/versions/8e50e47b7d32_normalize_object_storage_urls_to_keys.py

@@ -0,0 +1,142 @@
+"""Normalize object storage URLs to object keys.
+
+Previously, widget_image.image_url, widget_documents.url (for uploaded files),
+and tenant.hero_image_url stored full S3 URLs
+(e.g. https://host/bucket/my-file.png). Going forward only the object key
+(e.g. my-file.png) is stored; URLs are constructed dynamically at read time
+using the S3_BUCKET / S3_HOST environment variables.
+
+This migration strips the bucket+host prefix from any rows that still contain
+a full URL, leaving only the object key.  The downgrade puts them back using
+the values of those env vars at the time it is run.
+
+Revision ID: 8e50e47b7d32
+Revises: d756d0fe942c
+Create Date: 2026-04-29 00:00:00.000000
+"""
+import os
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '8e50e47b7d32'
+down_revision = 'd756d0fe942c'
+branch_labels = None
+depends_on = None
+
+
+def _url_prefix() -> str | None:
+    """Return the full URL prefix that was previously prepended to object keys.
+
+    Returns None when S3_HOST or S3_BUCKET are not configured, in which case
+    nothing needs to be migrated (no full URLs could have been stored).
+    """
+    host = os.getenv('S3_HOST')
+    bucket = os.getenv('S3_BUCKET')
+    if not host or not bucket:
+        return None
+    return f'https://{host}/{bucket}/'
+
+
+def upgrade():
+    prefix = _url_prefix()
+    if not prefix:
+        return
+
+    conn = op.get_bind()
+
+    # ---- widget_image.image_url ----------------------------------------
+    conn.execute(
+        sa.text(
+            """
+            UPDATE widget_image
+            SET    image_url = SUBSTRING(image_url FROM :prefix_len)
+            WHERE  image_url LIKE :prefix_pattern
+            """
+        ),
+        {
+            'prefix_len': len(prefix) + 1,   # 1-based SUBSTRING index
+            'prefix_pattern': prefix + '%',
+        },
+    )
+
+    # ---- widget_documents.url (uploaded files only) --------------------
+    conn.execute(
+        sa.text(
+            """
+            UPDATE widget_documents
+            SET    url = SUBSTRING(url FROM :prefix_len)
+            WHERE  is_uploaded = TRUE
+              AND  url LIKE :prefix_pattern
+            """
+        ),
+        {
+            'prefix_len': len(prefix) + 1,
+            'prefix_pattern': prefix + '%',
+        },
+    )
+
+    # ---- tenant.hero_image_url -----------------------------------------
+    conn.execute(
+        sa.text(
+            """
+            UPDATE tenant
+            SET    hero_image_url = SUBSTRING(hero_image_url FROM :prefix_len)
+            WHERE  hero_image_url LIKE :prefix_pattern
+            """
+        ),
+        {
+            'prefix_len': len(prefix) + 1,
+            'prefix_pattern': prefix + '%',
+        },
+    )
+
+
+def downgrade():
+    prefix = _url_prefix()
+    if not prefix:
+        return
+
+    conn = op.get_bind()
+
+    # Prepend the prefix back to any rows that do NOT already start with https://
+    conn.execute(
+        sa.text(
+            """
+            UPDATE widget_image
+            SET    image_url = :prefix || image_url
+            WHERE  image_url IS NOT NULL
+              AND  image_url NOT LIKE 'https://%'
+              AND  image_url <> ''
+            """
+        ),
+        {'prefix': prefix},
+    )
+
+    conn.execute(
+        sa.text(
+            """
+            UPDATE widget_documents
+            SET    url = :prefix || url
+            WHERE  is_uploaded = TRUE
+              AND  url IS NOT NULL
+              AND  url NOT LIKE 'https://%'
+              AND  url <> ''
+            """
+        ),
+        {'prefix': prefix},
+    )
+
+    conn.execute(
+        sa.text(
+            """
+            UPDATE tenant
+            SET    hero_image_url = :prefix || hero_image_url
+            WHERE  hero_image_url IS NOT NULL
+              AND  hero_image_url NOT LIKE 'https://%'
+              AND  hero_image_url <> ''
+            """
+        ),
+        {'prefix': prefix},
+    )

api/src/api/schemas/tenant.py

@@ -1,11 +1,15 @@
 """Tenant schema class."""
 
-from marshmallow import EXCLUDE, Schema, fields
+from marshmallow import EXCLUDE, Schema, fields, post_dump
+
+from api.services.object_storage_service import ObjectStorageService
 
 
 class TenantSchema(Schema):
     """Schema for tenant."""
 
+    _object_storage = ObjectStorageService()
+
     class Meta:  # pylint: disable=too-few-public-methods
         """Exclude unknown fields in the deserialized output."""
 
@@ -20,3 +24,18 @@ class Meta:  # pylint: disable=too-few-public-methods
     contact_email = fields.Str(data_key='contact_email')
     hero_image_credit = fields.Str(data_key='hero_image_credit')
     hero_image_description = fields.Str(data_key='hero_image_description')
+
+    @post_dump(pass_collection=True)
+    def _attach_hero_image_url(self, data, many, **kwargs):
+        if not many:
+            hero_image_url = data.get('hero_image_url')
+            if hero_image_url:
+                data['hero_image_url'] = self._object_storage.get_url(hero_image_url)
+            return data
+
+        for item in data:
+            hero_image_url = item.get('hero_image_url')
+            if hero_image_url:
+                item['hero_image_url'] = self._object_storage.get_url(hero_image_url)
+
+        return data

api/src/api/schemas/widget_documents.py

@@ -1,16 +1,33 @@
 """Widget schema class."""
+from marshmallow import post_dump
 
 from api.models.widget_documents import WidgetDocuments as WidgetDocumentModel
+from api.services.object_storage_service import ObjectStorageService
 
 from .base_schema import BaseSchema
 
 
 class WidgetDocumentsSchema(BaseSchema):
     """Widget Documents schema."""
 
+    _object_storage = ObjectStorageService()
+
     class Meta(BaseSchema.Meta):  # pylint: disable=too-few-public-methods
         """Exclude unknown fields in the deserialized output."""
 
         model = WidgetDocumentModel
         include_fk = True
         fields = ('id', 'title', 'type', 'parent_document_id', 'url', 'sort_index', 'is_uploaded')
+
+    @post_dump(pass_collection=True)
+    def _attach_document_url(self, data, many, **kwargs):
+        if not many:
+            if data.get('is_uploaded') and data.get('url'):
+                data['url'] = self._object_storage.get_url(data.get('url'))
+            return data
+
+        for item in data:
+            if item.get('is_uploaded') and item.get('url'):
+                item['url'] = self._object_storage.get_url(item.get('url'))
+
+        return data

api/src/api/schemas/widget_image.py

@@ -12,15 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Widget image schema definition."""
+from marshmallow import post_dump
 
 from api.models.widget_image import WidgetImage as WidgetImageModel
+from api.services.object_storage_service import ObjectStorageService
 
 from .base_schema import BaseSchema
 
 
 class WidgetImageSchema(BaseSchema):
     """This is the schema for the image model."""
 
+    _object_storage = ObjectStorageService()
+
     class Meta(BaseSchema.Meta):  # pylint: disable=too-few-public-methods
         """Images all of the Widget Image fields to a default schema."""
 
@@ -34,3 +38,18 @@ class Meta(BaseSchema.Meta):  # pylint: disable=too-few-public-methods
             'alt_text',
             'description',
         )
+
+    @post_dump(pass_collection=True)
+    def _attach_image_url(self, data, many, **kwargs):
+        if not many:
+            image_url = data.get('image_url')
+            if image_url:
+                data['image_url'] = self._object_storage.get_url(image_url)
+            return data
+
+        for item in data:
+            image_url = item.get('image_url')
+            if image_url:
+                item['image_url'] = self._object_storage.get_url(image_url)
+
+        return data

api/src/api/services/object_storage_service.py

@@ -2,6 +2,7 @@
 import os
 import uuid
 from typing import List
+from urllib.parse import urlparse
 
 import requests
 from aws_requests_auth.aws_auth import AWSRequestsAuth
@@ -28,10 +29,28 @@ def __init__(self):
 
     def get_url(self, filename: str):
         """Get the object url."""
-        if (not self.s3_auth.aws_host or not self.s3_bucket or not filename):
+        object_key = self.get_object_key(filename)
+        if (not self.s3_auth.aws_host or not self.s3_bucket or not object_key):
             return ''
 
-        return f'https://{self.s3_auth.aws_host}/{self.s3_bucket}/{filename}'
+        return f'https://{self.s3_auth.aws_host}/{self.s3_bucket}/{object_key}'
+
+    def get_object_key(self, filename: str):
+        """Extract the object key from a stored filename/path/url."""
+        if not filename:
+            return ''
+
+        parsed = urlparse(filename)
+        if parsed.scheme and parsed.netloc:
+            path = parsed.path.lstrip('/')
+            if not path:
+                return ''
+            bucket_prefix = f'{self.s3_bucket}/' if self.s3_bucket else ''
+            if bucket_prefix and path.startswith(bucket_prefix):
+                return path[len(bucket_prefix):]
+            return path
+
+        return filename.lstrip('/')
 
     def get_auth_headers(self, documents: List[Document]):
         """Get the S3 auth headers for the provided documents."""

api/src/api/services/tenant_service.py

@@ -6,6 +6,7 @@
 from api.models.tenant import Tenant as TenantModel
 from api.schemas.tenant import TenantSchema
 from api.services import authorization
+from api.services.object_storage_service import ObjectStorageService
 from api.utils.roles import Role
 from ..utils.cache import cache
 
@@ -15,6 +16,14 @@
 class TenantService:
     """Tenant management service."""
 
+    _object_storage = ObjectStorageService()
+
+    @classmethod
+    def _normalize_hero_image_url(cls, data: dict):
+        if 'hero_image_url' in data:
+            data['hero_image_url'] = cls._object_storage.get_object_key(data.get('hero_image_url'))
+        return data
+
     @classmethod
     def build_all_tenant_cache(cls):
         """Build cache for all tenant values."""
@@ -47,7 +56,8 @@ def create(cls, data: dict):
             Role.SUPER_ADMIN.value,
         )
         authorization.check_auth(one_of_roles=one_of_roles)
-        tenant = TenantModel(**data)
+        normalized_data = cls._normalize_hero_image_url(dict(data))
+        tenant = TenantModel(**normalized_data)
         try:
             tenant.save()
         except SQLAlchemyError as e:
@@ -66,7 +76,7 @@ def update(cls, tenant_id: str, data: dict):
         if not tenant:
             raise ValueError(NOT_FOUND_MSG, cls, tenant_id)
         try:
-            tenant.update(data)
+            tenant.update(cls._normalize_hero_image_url(dict(data)))
         except SQLAlchemyError as e:
             current_app.logger.error('Error updating tenant {}', e)
             raise ValueError('Error updating tenant.') from e