Merge pull request #2892 from bcgov/DEP-277-formio-fileupload-backend

DEP-277: Add CHEFS + Object storage file upload functionality to DEP (+fix survey listing)

Author: NatSquared

CHANGELOG.MD

@@ -1,4 +1,12 @@
-## Jul 7, 2026
+## July 07, 2026
+
+- **Feature** Added ability to upload files to Object Storage during survey submission [🎟️ DEP-277](https://citz-gdx.atlassian.net/browse/DEP-277)
+  - Added a new public document endpoint to the API that generates pre-signed URLs for uploading files to Object Storage. This endpoint takes the file name and content type as input and returns a pre-signed URL that can be used to upload the file directly to Object Storage.
+    - The new API can also be used to generate pre-signed URLs for downloading files from Object Storage, as well as deleting files from Object Storage.
+  - Updated the Formio options to include callbacks for handling file uploads, downloads, and deletions. These callbacks use the new API to generate URLs and perform privileged operations on Object Storage, allowing users to upload files during survey submission without needing high-level access to the storage service.
+  - Cleaned up survey-related component code
+  - Updated the survey pages to use React Router loaders to fetch the survey data instead of using useState and useEffect, as well as making sure the loader returns promises (as opposed to awaiting the data) so that the survey page can be rendered while the data is being fetched. This improves the user experience by allowing the page to render faster and show a loading state while the survey data is being retrieved.
+  - Fixed authorization bug that did not allow Super Admins to view surveys
 
 - **Feature** Integrated new landing hero section and introduction [🎟️ DEP-315](https://citz-gdx.atlassian.net/browse/DEP-315)
   - Refactored existing landing code for structural simplicity

api/src/api/models/submission.py

@@ -4,7 +4,7 @@
 """
 from __future__ import annotations
 
-from typing import List
+from typing import List, Optional
 
 from sqlalchemy import ForeignKey
 from sqlalchemy.dialects import postgresql
@@ -26,20 +26,27 @@ class Submission(BaseModel):  # pylint: disable=too-few-public-methods
     __tablename__ = 'submission'
 
     id = db.Column(db.Integer, primary_key=True, autoincrement=True)
-    submission_json = db.Column(postgresql.JSONB(astext_type=db.Text()), nullable=False, server_default='{}')
-    survey_id = db.Column(db.Integer, ForeignKey('survey.id', ondelete='CASCADE'), nullable=False)
-    engagement_id = db.Column(db.Integer, ForeignKey('engagement.id', ondelete='CASCADE'), nullable=False)
-    participant_id = db.Column(db.Integer, ForeignKey('participant.id'), nullable=True)
+    submission_json = db.Column(postgresql.JSONB(
+        astext_type=db.Text()), nullable=False, server_default='{}')
+    survey_id = db.Column(db.Integer, ForeignKey(
+        'survey.id', ondelete='CASCADE'), nullable=False)
+    engagement_id = db.Column(db.Integer, ForeignKey(
+        'engagement.id', ondelete='CASCADE'), nullable=False)
+    participant_id = db.Column(
+        db.Integer, ForeignKey('participant.id'), nullable=True)
     reviewed_by = db.Column(db.String(50))
     review_date = db.Column(db.DateTime)
-    comment_status_id = db.Column(db.Integer, ForeignKey('comment_status.id', ondelete='SET NULL'))
+    comment_status_id = db.Column(db.Integer, ForeignKey(
+        'comment_status.id', ondelete='SET NULL'))
     has_personal_info = db.Column(db.Boolean, nullable=True)
     has_profanity = db.Column(db.Boolean, nullable=True)
     rejected_reason_other = db.Column(db.String(500), nullable=True)
     has_threat = db.Column(db.Boolean, nullable=True)
     notify_email = db.Column(db.Boolean(), default=True)
-    comments = db.relationship('Comment', backref='submission', cascade='all, delete')
-    staff_note = db.relationship('StaffNote', backref='submission', cascade='all, delete')
+    comments = db.relationship(
+        'Comment', backref='submission', cascade='all, delete')
+    staff_note = db.relationship(
+        'StaffNote', backref='submission', cascade='all, delete')
 
     @classmethod
     def get_by_survey_id(cls, survey_id) -> List[SubmissionSchema]:
@@ -118,7 +125,7 @@ def update(cls, submission: SubmissionSchema, session=None) -> Submission:
         return query.first()
 
     @classmethod
-    def update_comment_status(cls, submission_id, comment: dict, session=None) -> Submission:
+    def update_comment_status(cls, submission_id, comment: dict, session=None) -> Optional[Submission]:
         """Update comment status."""
         status_id = comment.get('status_id', None)
         has_personal_info = comment.get('has_personal_info', None)

api/src/api/resources/document.py

@@ -14,22 +14,47 @@
 """API endpoints for managing documents resource."""
 
 from http import HTTPStatus
+from typing import Any, cast
 
 from flask import jsonify, request
 from flask_cors import cross_origin
 from flask_restx import Namespace, Resource
 
+from api.models import Engagement as EngagementModel
+from api.models import Survey as SurveyModel
 from api.schemas.document import Document
+from api.schemas.public_upload import PublicObjectAccessRequestSchema, PublicUploadAuthorizationRequestSchema
+from api.services.email_verification_service import EmailVerificationService
 from api.services.object_storage_service import ObjectStorageService
 from api.utils.roles import Role
 from api.utils.tenant_validator import require_role
 from api.utils.util import allowedorigins, cors_preflight
 
 
-API = Namespace('document', description='Endpoints for Document Storage Management')
+API = Namespace(
+    'document', description='Endpoints for Document Storage Management')
 """Custom exception messages"""
 
 
+def _get_public_upload_scope(verification_token: str):
+    """Return the validated public upload scope for the provided verification token."""
+    verification = cast(
+        dict[str, Any], EmailVerificationService().get_active(verification_token))
+    survey_id = verification.get('survey_id')
+    if not survey_id:
+        raise ValueError('Survey not found.')
+
+    survey = SurveyModel.get_open(survey_id)
+    if not survey:
+        raise PermissionError('Engagement not open to submissions.')
+
+    engagement = EngagementModel.find_by_id(survey.engagement_id)
+    if not engagement:
+        raise ValueError('Engagement not found.')
+
+    return verification, survey, engagement
+
+
 @cors_preflight('GET,OPTIONS')
 @API.route('/')
 class DocumentStorage(Resource):
@@ -42,9 +67,133 @@ def post():
         """Retrieve authentication properties for document storage."""
         try:
             requestfilejson = request.get_json()
-            documents = Document().load(requestfilejson, many=True)
+            documents = cast(list[dict[str, Any]],
+                             Document().load(requestfilejson, many=True))
             return jsonify(ObjectStorageService().get_auth_headers(documents)), HTTPStatus.OK
         except KeyError as err:
             return str(err), HTTPStatus.INTERNAL_SERVER_ERROR
         except ValueError as err:
             return str(err), HTTPStatus.INTERNAL_SERVER_ERROR
+
+
+@cors_preflight('POST,OPTIONS')
+@API.route('/public')
+class PublicDocumentUploadAuthorization(Resource):
+    """Token-scoped public upload/download/delete authorization controller."""
+
+    @staticmethod
+    @cross_origin(origins=allowedorigins())
+    def post():
+        """Retrieve upload authorization for a public survey file upload."""
+        try:
+            request_json = request.get_json()
+            upload_request = cast(
+                dict[str, Any],
+                PublicUploadAuthorizationRequestSchema().load(request_json),
+            )
+
+            verification, survey, _ = _get_public_upload_scope(
+                upload_request['verification_token'])
+
+            object_storage_service = ObjectStorageService()
+            object_key, unique_filename = object_storage_service.build_public_upload_key(
+                tenant_id=survey.tenant_id,
+                survey_id=survey.id,
+                verification_id=verification['id'],
+                filename=upload_request['filename'],
+            )
+            signed_upload = object_storage_service.get_signed_upload_details(
+                object_key=object_key,
+                content_type=upload_request.get('content_type'),
+            )
+
+            return {
+                **signed_upload,
+                'uniquefilename': unique_filename,
+                'filename': upload_request['filename'],
+                'content_type': upload_request['content_type'],
+                'size': upload_request['size'],
+            }, HTTPStatus.OK
+        except KeyError as err:
+            return str(err), HTTPStatus.BAD_REQUEST
+        except PermissionError as err:
+            return {'message': str(err)}, HTTPStatus.FORBIDDEN
+        except ValueError as err:
+            return {'message': str(err)}, HTTPStatus.BAD_REQUEST
+
+    @staticmethod
+    @cross_origin(origins=allowedorigins())
+    def get():
+        """Retrieve download authorization for a public survey file upload."""
+        try:
+            file_id = request.args.get('file_id')
+            verification_token = request.headers.get('Verification-Token')
+            if not file_id or not verification_token:
+                return {'message': 'Missing required parameters.'}, HTTPStatus.BAD_REQUEST
+            access_request = cast(
+                dict[str, Any],
+                PublicObjectAccessRequestSchema().load({
+                    'file_id': request.args.get('file_id'),
+                    'verification_token': request.headers.get('Verification-Token'),
+                }),
+            )
+            verification, survey, _ = _get_public_upload_scope(
+                access_request['verification_token'])
+
+            object_storage_service = ObjectStorageService()
+            object_key = object_storage_service.get_object_key(
+                access_request['file_id'])
+            expected_prefix = object_storage_service.build_public_upload_prefix(
+                tenant_id=survey.tenant_id,
+                survey_id=survey.id,
+                verification_id=verification['id'],
+            )
+            if not object_key.startswith(expected_prefix):
+                return {'message': 'Invalid file requested.'}, HTTPStatus.FORBIDDEN
+
+            return object_storage_service.get_signed_request_details('GET', object_key), HTTPStatus.OK
+        except KeyError as err:
+            return str(err), HTTPStatus.BAD_REQUEST
+        except PermissionError as err:
+            return {'message': str(err)}, HTTPStatus.FORBIDDEN
+        except ValueError as err:
+            return {'message': str(err)}, HTTPStatus.BAD_REQUEST
+
+    @staticmethod
+    @cross_origin(origins=allowedorigins())
+    def delete():
+        """Delete a public survey upload for a valid verification token."""
+        try:
+            headers = request.headers
+            args = request.args
+            if not args.get('file_id') or not headers.get('Verification-Token'):
+                return {'message': 'Missing required parameters.'}, HTTPStatus.BAD_REQUEST
+            access_request = cast(
+                dict[str, Any],
+                PublicObjectAccessRequestSchema().load({
+                    'file_id': args.get('file_id'),
+                    'verification_token': headers.get('Verification-Token'),
+                }),
+            )
+            verification, survey, _ = _get_public_upload_scope(
+                access_request['verification_token'])
+
+            object_storage_service = ObjectStorageService()
+            object_key = object_storage_service.get_object_key(
+                access_request['file_id'])
+            expected_prefix = object_storage_service.build_public_upload_prefix(
+                tenant_id=survey.tenant_id,
+                survey_id=survey.id,
+                verification_id=verification['id'],
+            )
+            if not object_key.startswith(expected_prefix):
+                return {'message': 'Invalid file requested.'}, HTTPStatus.FORBIDDEN
+
+            object_storage_service.delete_file(object_key)
+            return {}, HTTPStatus.NO_CONTENT
+        except KeyError as err:
+            return str(err), HTTPStatus.BAD_REQUEST
+        except PermissionError as err:
+            return {'message': str(err)}, HTTPStatus.FORBIDDEN
+        except ValueError as err:
+            return {'message': str(err)}, HTTPStatus.BAD_REQUEST

api/src/api/resources/survey.py

@@ -87,14 +87,21 @@ def get():
                 exclude_hidden=args.get('exclude_hidden', False, bool),
                 exclude_template=args.get('exclude_template', False, bool),
                 search_text=args.get('search_text', '', str),
-                is_unlinked=args.get('is_unlinked', default=False, type=lambda v: v.lower() == 'true'),
-                is_linked=args.get('is_linked', default=False, type=lambda v: v.lower() == 'true'),
-                is_hidden=args.get('is_hidden', default=False, type=lambda v: v.lower() == 'true'),
-                is_template=args.get('is_template', default=False, type=lambda v: v.lower() == 'true'),
-                created_date_from=args.get('created_date_from', None, type=str),
+                is_unlinked=args.get(
+                    'is_unlinked', default=False, type=lambda v: v.lower() == 'true'),
+                is_linked=args.get('is_linked', default=False,
+                                   type=lambda v: v.lower() == 'true'),
+                is_hidden=args.get('is_hidden', default=False,
+                                   type=lambda v: v.lower() == 'true'),
+                is_template=args.get(
+                    'is_template', default=False, type=lambda v: v.lower() == 'true'),
+                created_date_from=args.get(
+                    'created_date_from', None, type=str),
                 created_date_to=args.get('created_date_to', None, type=str),
-                published_date_from=args.get('published_date_from', None, type=str),
-                published_date_to=args.get('published_date_to', None, type=str),
+                published_date_from=args.get(
+                    'published_date_from', None, type=str),
+                published_date_to=args.get(
+                    'published_date_to', None, type=str),
             )
 
             survey_records = SurveyService()\

api/src/api/schemas/public_upload.py

@@ -0,0 +1,53 @@
+"""Schemas for public object-storage upload authorization."""
+
+from marshmallow import EXCLUDE, Schema, fields, validate
+
+
+class PublicUploadAuthorizationRequestSchema(Schema):
+    """Request schema for token-scoped public upload authorization."""
+
+    class Meta:  # pylint: disable=too-few-public-methods
+        """Exclude unknown fields in the deserialized output."""
+
+        unknown = EXCLUDE
+
+    filename = fields.Str(
+        data_key='filename',
+        required=True,
+        validate=validate.Length(min=1, max=2048),
+    )
+    content_type = fields.Str(
+        data_key='content_type',
+        required=True,
+        validate=validate.Length(min=1, max=255),
+    )
+    size = fields.Int(
+        data_key='size',
+        required=True,
+        validate=validate.Range(min=1),
+    )
+    verification_token = fields.Str(
+        data_key='verification_token',
+        required=True,
+        validate=validate.Length(min=1, max=255),
+    )
+
+
+class PublicObjectAccessRequestSchema(Schema):
+    """Request schema for token-scoped access to a previously uploaded object."""
+
+    class Meta:  # pylint: disable=too-few-public-methods
+        """Exclude unknown fields in the deserialized output."""
+
+        unknown = EXCLUDE
+
+    file_id = fields.Str(
+        data_key='file_id',
+        required=True,
+        validate=validate.Length(min=1, max=4096),
+    )
+    verification_token = fields.Str(
+        data_key='verification_token',
+        required=True,
+        validate=validate.Length(min=1, max=255),
+    )