Merge pull request #235 from HydrologicEngineeringCenter/156-blob-endpoint-fails-to-store-excelfails-to-determine-encoding

add test for excel files

Author: Enovotny

.github/workflows/CDA-testing.yml

@@ -2,7 +2,7 @@ name: CI
 
 on:
   push:
-    branches: [main, githubAction-testing]
+    branches: [main]
   pull_request:
     branches: [main]
   workflow_dispatch:
@@ -14,40 +14,42 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - name: set up backend
-        run: docker compose up --build -d
+      - name: Set up backend
+        run: |
+          docker compose pull
+          docker compose up -d
 
       - name: Set Up Python
         uses: actions/setup-python@v6
         with:
           python-version: '3.9.X'
 
-      # Unlike the code-check workflow, this job requires the dev dependencies to be
-      # installed to make sure we have the necessary, tools, stub files, etc.
-      - name: Install Poetry
+      # Use actions-poetry to handle installation
+      - name: Install Poetry and Dependencies
         uses: abatilo/actions-poetry@v4
 
-      - name: Add Poetry to PATH (for act)
-        if: env.ACT
-        run: echo "/root/.local/bin" >> $GITHUB_PATH
+      # Set Poetry to use an in-project virtual environment
+      - name: Configure Poetry for in-project venv
+        run: poetry config virtualenvs.in-project true
 
-      - name: Cache Virtual Environment
+          # Poetry will handle installation and caching
+      - name: Cache Python dependencies
         uses: actions/cache@v4
+        id: cache-poetry-venv
         with:
-          path: ./.venv
-          key: venv-${{ hashFiles('poetry.lock') }}
+          path: .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
 
-      - name: Install Dependencies
-        run: poetry install
+      # Install dependencies only if cache is missed
+      - name: Install dependencies
+        if: steps.cache-poetry-venv.outputs.cache-hit != 'true'
+        run: poetry install --no-root
 
       # Run pytest and generate coverage report data.
-      - name: Run Tests
-        run: poetry run pytest tests/cda/ --doctest-modules --cov --cov-report=xml:out/coverage.xml
-
-      # Run mypy with strict mode enabled. Only the main source code is type checked (test
-      # and example code is excluded).
-      - name: Check Types
-        run: poetry run mypy --strict cwms/
+      - name: Run Tests and Check Types
+        run: |
+          poetry run pytest tests/cda/ --doctest-modules --cov --cov-report=xml:out/coverage.xml
+          poetry run mypy --strict cwms/
 
       - name: Generate Coverage Report
         uses: irongut/CodeCoverageSummary@v1.3.0

docker-compose.yml

@@ -10,10 +10,9 @@ services:
       - CWMS_PASSWORD=simplecwmspasswD1
       - OFFICE_ID=HQ
       - OFFICE_EROC=s0
-    ports:
-      - "1526:1521"
+    ports: ["1526:1521"]
     healthcheck:
-      test: ["CMD","tnsping", "FREEPDB1"]
+      test: ["CMD", "tnsping", "FREEPDB1"]
       interval: 30s
       timeout: 50s
       retries: 50
@@ -32,9 +31,9 @@ services:
       - INSTALLONCE=1
       - QUIET=1
     command: >
-      sh -xc "sqlplus CWMS_20/$$CWMS_PASSWORD@$$DB_HOST_PORT$$DB_NAME @/setup_sql/users $$OFFICE_EROC"
-    volumes:
-      - ./compose_files/sql:/setup_sql:ro
+      sh -xc "sqlplus CWMS_20/$$CWMS_PASSWORD@$$DB_HOST_PORT$$DB_NAME @/setup_sql/users
+      $$OFFICE_EROC"
+    volumes: [./compose_files/sql:/setup_sql:ro]
     depends_on:
       db:
         condition: service_healthy
@@ -43,7 +42,6 @@ services:
       traefik:
         condition: service_healthy
 
-
   data-api:
     depends_on:
       auth:
@@ -75,10 +73,9 @@ services:
       - cwms.dataapi.access.openid.altAuthUrl=http://localhost:${APP_PORT:-8082}
       - cwms.dataapi.access.openid.useAltWellKnown=true
       - cwms.dataapi.access.openid.issuer=http://localhost:${APP_PORT:-8082}/auth/realms/cwms
-    expose:
-      - 7000
+    expose: [7000]
     healthcheck:
-      test: ["CMD","/usr/bin/curl", "-I","localhost:7000/cwms-data/offices/HEC"]
+      test: ["CMD", "/usr/bin/curl", "-I", "localhost:7000/cwms-data/offices/HEC"]
       interval: 5s
       timeout: 1s
       retries: 100
@@ -90,9 +87,12 @@ services:
 
   auth:
     image: quay.io/keycloak/keycloak:19.0.1
-    command: ["start-dev", "--features-disabled=admin2","--import-realm"]
+    command: ["start-dev", "--import-realm"]
     healthcheck:
-      test: "/usr/bin/curl -If localhost:${APP_PORT:-8082}/auth/health/ready || exit 1"
+      test:
+        - "CMD-SHELL"
+        - "/usr/bin/curl -If localhost:${APP_PORT:-8082}/auth/health/ready || exit\
+          \ 1"
       interval: 5s
       timeout: 1s
       retries: 100
@@ -108,6 +108,8 @@ services:
       - KC_PROXY=none
       - KC_HTTP_ENABLED=true
       - KC_HTTP_RELATIVE_PATH=/auth
+      - KC_HOSTNAME=localhost
+      - KC_DB=dev-file
     volumes:
       - ./compose_files/keycloak/realm.json:/opt/keycloak/data/import/realm.json:ro
     labels:
@@ -119,17 +121,12 @@ services:
       traefik:
         condition: service_healthy
 
-
-
   # Proxy for HTTPS for OpenID
   traefik:
     image: traefik:v3.3.3
-    ports:
-      - "${APP_PORT:-8082}:80"
-    expose:
-      - "8080"
-    volumes:
-      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+    ports: ["${APP_PORT:-8082}:80"]
+    expose: ["8080"]
+    volumes: ["/var/run/docker.sock:/var/run/docker.sock:ro"]
     healthcheck:
       test: traefik healthcheck --ping
     command:
@@ -142,4 +139,4 @@ services:
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.traefik.rule=PathPrefix(`/traefik`)"
-      - "traefik.http.routers.traefik.service=api@internal"
+      - "traefik.http.routers.traefik.service=api@internal"

tests/cda/blobs/blob_CDA_test.py

@@ -1,13 +1,16 @@
 # tests/test_blob.py
 from __future__ import annotations
 
+import base64
+import mimetypes
 from datetime import datetime, timezone
+from pathlib import Path
 from typing import Optional
 
 import pandas as pd
 import pytest
 
-import cwms
+import cwms.catalog.blobs as blobs
 
 TEST_OFFICE = "MVP"
 TEST_BLOB_ID = "PYTEST_BLOB_ALPHA"
@@ -25,12 +28,12 @@
 def ensure_clean_slate():
     """Delete the test blob (if it exists) before/after running this module."""
     try:
-        cwms.delete_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID)
+        blobs.delete_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID)
     except Exception:
         pass
     yield
     try:
-        cwms.delete_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID)
+        blobs.delete_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID)
     except Exception:
         pass
 
@@ -44,7 +47,7 @@ def _find_blob_row(office: str, blob_id: str) -> Optional[pd.Series]:
     """
     Helper: return the row for blob_id from cwms.get_blobs(...).df if present.
     """
-    res = cwms.get_blobs(office_id=office, blob_id_like=blob_id)
+    res = blobs.get_blobs(office_id=office, blob_id_like=blob_id)
     df = res if isinstance(res, pd.DataFrame) else getattr(res, "df", None)
     if df is None or df.empty:
         return None
@@ -55,6 +58,28 @@ def _find_blob_row(office: str, blob_id: str) -> Optional[pd.Series]:
     return match.iloc[0] if not match.empty else None
 
 
+def test_store_blob_excel():
+    excel_file_path = Path(__file__).parent.parent / "resources" / "blob_test.xlsx"
+    with open(excel_file_path, "rb") as f:
+        file_data = f.read()
+    mime_type, _ = mimetypes.guess_type(excel_file_path)
+    excel_blob_id = "TEST_BLOB_EXCEL"
+    payload = {
+        "office-id": TEST_OFFICE,
+        "id": excel_blob_id,
+        "description": "testing excel file",
+        "media-type-id": mime_type,
+        "value": base64.b64encode(file_data).decode("utf-8"),
+    }
+    blobs.store_blobs(data=payload)
+    try:
+        row = _find_blob_row(TEST_OFFICE, excel_blob_id)
+        assert row is not None, "Stored blob not found in listing"
+    finally:
+        # Cleanup excel
+        blobs.delete_blob(blob_id=excel_blob_id, office_id=TEST_OFFICE)
+
+
 def test_store_blob():
     # Build request JSON for store_blobs
     payload = {
@@ -64,7 +89,7 @@ def test_store_blob():
         "media-type-id": TEST_MEDIA_TYPE,
         "value": TEST_TEXT,
     }
-    cwms.store_blobs(payload, fail_if_exists=True)
+    blobs.store_blobs(payload, fail_if_exists=True)
 
     # Verify via listing metadata
     row = _find_blob_row(TEST_OFFICE, TEST_BLOB_ID)
@@ -76,14 +101,14 @@ def test_store_blob():
         assert TEST_DESC in str(row["description"])
 
     # Verify content by downloading
-    content = cwms.get_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID)
+    content = blobs.get_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID)
     assert isinstance(content, str) and content, "Empty blob content"
     assert TEST_TEXT in content
 
 
 def test_get_blob():
     # Do a simple read of the blob created in test_store_blob
-    content = cwms.get_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID)
+    content = blobs.get_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_ID)
     assert TEST_TEXT in content
     assert len(content) >= len(TEST_TEXT)
 
@@ -97,7 +122,7 @@ def test_update_blob():
         "media-type-id": TEST_MEDIA_TYPE,
         "value": TEST_TEXT_UPDATED,
     }
-    cwms.update_blob(update, fail_if_not_exists=True)
+    blobs.update_blob(update, fail_if_not_exists=True)
 
     # Confirm updated metadata
     row = _find_blob_row(TEST_OFFICE, TEST_BLOB_UPDATED_ID)
@@ -106,5 +131,5 @@ def test_update_blob():
         assert TEST_DESC_UPDATED in str(row["description"])
 
     # Verify new content
-    content = cwms.get_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_UPDATED_ID)
+    content = blobs.get_blob(office_id=TEST_OFFICE, blob_id=TEST_BLOB_UPDATED_ID)
     assert TEST_TEXT_UPDATED in content