DBC22-6067: removed legacy images pulling logic (#1275)

Author: bcgov-brwang

.env.example

@@ -64,10 +64,7 @@ REACT_APP_LEGACY_URL=<legacy URL for the old DriveBC site>
 
 # API
 DRIVEBC_INLAND_FERRY_API_BASE_URL=<inland ferry api url>
-DRIVEBC_IMAGE_API_BASE_URL=<image api url>
 DRIVEBC_IMAGE_BASE_URL=<base url for local images>
-DRIVEBC_IMAGE_PROXY_URL=<image proxy url>
-DRIVEBC_WEBCAM_API_BASE_URL=<camera api url>
 DRIVEBC_OPEN_511_API_BASE_URL=<open511 api url>
 DRIVEBC_DIT_API_BASE_URL=<CARS api url>
 DRIVEBC_ROUTE_PLANNER_API_BASE_URL=<route planner url>

src/backend/apps/consumer/processor.py

@@ -307,6 +307,7 @@ def process_camera_rows(rows):
             'update_period_stddev': 60,
             'dbc_mark': row.cam_internetdbc_mark if hasattr(row, 'cam_internetdbc_mark') else '',
             'is_on': not row.cam_controldisabled if hasattr(row, 'cam_controldisabled') else True,
+            'cam_controldisappear': not row.cam_controldisappear if hasattr(row, 'cam_controldisappear') else True,
             'cam_maintenanceis_on_demand': row.cam_maintenanceis_on_demand if hasattr(row, 'cam_maintenanceis_on_demand') else False,
             'is_new': row.isnew if hasattr(row, 'isnew') else False,
             'seq': row.seq if hasattr(row, 'seq') else 0,
@@ -467,8 +468,6 @@ def save_watermarked_image_to_drivebc_pvc(camera_id: str, image_bytes: bytes, is
             f.write(image_bytes)
         if is_on:
             logger.info(f"Watermarked image saved to drivebc PVC at {filepath}")
-        else:
-            logger.info(f"Blank out image saved to drivebc PVC at {filepath}")
     except Exception as e:
         logger.error(f"Error saving image to drivebc PVC {filepath}: {e}")
 
@@ -480,13 +479,10 @@ def delete_watermarked_image_from_pvc(camera_id: str):
         return
 
     try:
-        deleted_count = 0
         for filename in os.listdir(save_dir):
             filepath = os.path.join(save_dir, filename)
             if os.path.isfile(filepath):
                 os.remove(filepath)
-                deleted_count += 1
-        logger.info(f"Deleted {deleted_count} watermarked image(s) from {save_dir}")
     except Exception as e:
         logger.error(f"Error deleting watermarked images from PVC {save_dir}: {e}")
 

src/backend/apps/consumer/tasks.py

@@ -32,18 +32,9 @@ def generate_offline_camera_images():
             watermarked = blank_out_image(camera, dummy_image_bytes, tz, timestamp)
 
             if watermarked:
-                # Convert timestamp to UTC for storage
-                local_tz = pytz.timezone(tz)
-                naive_dt = datetime.strptime(timestamp, "%Y%m%d%H%M%S%f")
-                local_dt = local_tz.localize(naive_dt)
-                utc_dt = local_dt.astimezone(pytz.utc)
-                utc_timestamp_str = utc_dt.strftime("%Y%m%d%H%M%S")
-                
                 # Delete all the images for replay the day
                 delete_watermarked_image_from_pvc(camera_id)
                 # Delete all the records from image index table for offline cams
                 async_to_sync(delete_offline_webcam_records)(camera_id)
-                # Save blank image for replay the day
-                save_watermarked_image_to_pvc(camera_id, watermarked, utc_timestamp_str, False)
                 # Save blank image for current image displaying
                 save_watermarked_image_to_drivebc_pvc(camera_id, watermarked, False)

src/backend/apps/consumer/tests/test_processor.py

@@ -43,6 +43,7 @@ def test_process_camera_rows_with_valid_data(self):
                 cam_locationselevation=100,
                 cam_internetdbc_mark='MARK1',
                 cam_controldisabled=False,
+                cam_controldisappear=False,
                 cam_maintenanceis_on_demand=False,
                 isnew=False,
                 seq=1,

src/backend/apps/consumer/tests/test_tasks.py

@@ -53,7 +53,7 @@ def test_generates_for_offline_cameras(self, mock_process_rows, mock_get_db):
 
             generate_offline_camera_images()
 
-            mock_save_pvc.assert_called_once()
+            mock_save_pvc.assert_not_called()
             mock_save_drivebc.assert_called_once()
 
     @patch('apps.consumer.tasks.get_all_from_db')
@@ -84,7 +84,7 @@ def test_handles_watermark_failure(self, mock_process_rows, mock_get_db):
 
             generate_offline_camera_images()
 
-            mock_save_pvc.assert_called_once()
+            mock_save_pvc.assert_not_called()
             mock_save_drivebc.assert_called_once()
 
     @patch('apps.consumer.tasks.get_all_from_db')
@@ -103,4 +103,4 @@ def test_handles_is_on_key_missing(self, mock_process_rows, mock_get_db):
             generate_offline_camera_images()
 
             mock_watermark.assert_not_called()
-            mock_save_pvc.assert_not_called()
+            mock_save_pvc.assert_not_called()

src/backend/apps/feed/client.py

@@ -18,7 +18,7 @@
     REGIONAL_WEATHER,
     REGIONAL_WEATHER_AREAS,
     REST_STOP,
-    WEBCAM,
+    # WEBCAM,
     WILDFIRE,
 )
 from apps.feed.serializers import (
@@ -40,6 +40,8 @@
 from django.core.cache import cache
 from rest_framework.exceptions import ValidationError
 from rest_framework.response import Response
+from apps.weather.models import CurrentWeather
+from apps.webcam.models import CameraSource, Region, RegionHighway
 
 # Maps the key for our client API's serializer fields to the matching pair of
 # the source API's DataSetName and DisplayName fields
@@ -88,9 +90,6 @@ class FeedClient:
 
     def __init__(self):
         self.resource_map: Dict[str, dict] = {
-            WEBCAM: {
-                "base_url": settings.DRIVEBC_WEBCAM_API_BASE_URL,
-            },
             OPEN511: {
                 "base_url": settings.DRIVEBC_OPEN_511_API_BASE_URL,
             },
@@ -264,13 +263,16 @@ def get_list_feed(self, resource_type, resource_name, serializer_cls, params=Non
             new_serializer.is_valid(raise_exception=True)
             return new_serializer.validated_data
 
-    # Webcam
-    def get_webcam(self, webcam):
-        return self.get_single_feed(webcam, WEBCAM, 'webcams/', WebcamFeedSerializer)
 
     def get_webcam_list(self):
-        return self.get_list_feed(WEBCAM, 'webcams', WebcamAPISerializer)
+        try:
+            cameras = list(CameraSource.objects.using("mssql").all())
+            return cameras
 
+        except Exception as e:
+            logger.error(f"Failed to query camera from Webcam database: {e}")
+            return []
+        
     # Events
     def get_event(self, event):
         return self.get_single_feed(event, OPEN511, 'events/', EventFeedSerializer)

src/backend/apps/feed/tests/test_feed_client.py

@@ -3,11 +3,14 @@
 import pytest
 from django.core.cache import cache
 from httpx import HTTPStatusError
-from apps.feed.client import FeedClient, WEBCAM, OPEN511
+from apps.feed.client import FeedClient, OPEN511
 from apps.feed.constants import REST_STOP
 from apps.feed.serializers import WebcamAPISerializer, EventAPISerializer
 from apps.shared.tests import BaseTest
 from django.conf import settings
+from apps.webcam.models import CameraSource
+from pathlib import Path
+import json
 
 logging.getLogger().setLevel(logging.CRITICAL)
 
@@ -27,18 +30,6 @@ def setUp(self):
         super().setUp()
         self.client = FeedClient()
 
-    def test_get_endpoint(self):
-        endpoint = self.client._get_endpoint(WEBCAM, 'webcams/123')
-        self.assertIn('webcams/123', endpoint)
-
-    def test_get_auth_headers_with_key(self):
-        self.client.resource_map[WEBCAM]['auth_key'] = 'test_api_key'
-        headers = self.client._get_auth_headers(WEBCAM)
-        self.assertEqual(headers, {'apiKey': 'test_api_key'})
-
-    def test_get_auth_headers_without_key(self):
-        headers = self.client._get_auth_headers(WEBCAM)
-        self.assertEqual(headers, {})
 
     def test_get_response_data_or_raise_success(self):
         mock_response = create_mock_response(200, {'key': 'value'})
@@ -174,49 +165,6 @@ def setUp(self):
             }
         }
 
-    @patch('apps.feed.client.httpx.get')
-    @patch('apps.feed.serializers.WebcamFeedSerializer')
-    def test_get_single_feed(self, mock_serializer_cls, mock_get):
-        mock_response = create_mock_response(200, self.valid_webcam_data)
-        mock_get.return_value = mock_response
-
-        mock_serializer = MagicMock()
-        mock_serializer.is_valid = MagicMock(return_value=True)
-        mock_serializer.validated_data = {'id': 123}
-        mock_serializer_cls.return_value = mock_serializer
-
-        from apps.feed.serializers import WebcamFeedSerializer
-        result = self.client.get_single_feed('123', WEBCAM, 'webcams/', WebcamFeedSerializer)
-        self.assertIsNotNone(result)
-
-    @patch('apps.feed.client.httpx.get')
-    @patch('apps.feed.serializers.WebcamFeedSerializer')
-    def test_get_single_feed_returns_list(self, mock_serializer_cls, mock_get):
-        mock_response = create_mock_response(200, [self.valid_webcam_data])
-        mock_get.return_value = mock_response
-
-        mock_serializer = MagicMock()
-        mock_serializer.is_valid = MagicMock(return_value=True)
-        mock_serializer.validated_data = {'id': 123}
-        mock_serializer_cls.return_value = mock_serializer
-
-        from apps.feed.serializers import WebcamFeedSerializer
-        result = self.client.get_single_feed('123', WEBCAM, 'webcams/', WebcamFeedSerializer)
-        self.assertIsNotNone(result)
-
-    @patch('apps.feed.client.httpx.get')
-    @patch('apps.feed.client.WebcamFeedSerializer')
-    def test_get_single_feed_as_serializer(self, mock_serializer_cls, mock_get):
-        mock_response = create_mock_response(200, {'id': '123', 'name': 'Test'})
-        mock_get.return_value = mock_response
-
-        mock_serializer = MagicMock()
-        mock_serializer_cls.return_value = mock_serializer
-        mock_serializer.is_valid = MagicMock()
-        mock_serializer.save = MagicMock()
-
-        result = self.client.get_single_feed('123', WEBCAM, 'webcams/', mock_serializer_cls, as_serializer=True)
-        self.assertEqual(result, mock_serializer)
 
 
 class TestFeedClientGetListFeed(BaseTest):
@@ -226,13 +174,6 @@ def setUp(self):
         super().setUp()
         self.client = FeedClient()
 
-    @patch('apps.feed.client.httpx.get')
-    def test_get_list_feed_success(self, mock_get):
-        mock_response = create_mock_response(200, {'webcams': []})
-        mock_get.return_value = mock_response
-
-        result = self.client.get_list_feed(WEBCAM, 'webcams', WebcamAPISerializer)
-        self.assertIn('webcams', result)
 
     @patch('apps.feed.client.httpx.get')
     def test_get_list_feed_with_params(self, mock_get):
@@ -245,28 +186,47 @@ def test_get_list_feed_with_params(self, mock_get):
 
 class TestFeedClientWebcam(BaseTest):
     """Test webcam feed methods."""
+    
+    def populate_webcam_data(self, mock_data=None):
+        for cam in mock_data['webcams']:
+            CameraSource.objects.using('mssql').create(
+                id=cam['id'],
+                cam_internetname=cam['cam_internetname'],
+                cam_internetcaption=cam['cam_internetcaption'],
+                cam_internetcredit=cam['cam_internetcredit'],
+                cam_internetdbc_mark=cam['cam_internetdbc_mark'],
+                cam_locationsregion=cam['cam_locationsregion'],
+                cam_locationshighway=cam['cam_locationshighway'],
+                cam_controldisabled=cam['cam_controldisabled'],
+                cam_controldisappear=cam['cam_controldisappear'],
+                isnew=cam['isnew'],
+                cam_maintenanceis_on_demand=cam['cam_maintenanceis_on_demand'],
+                seq=cam['seq']
+            )
 
     def setUp(self):
         super().setUp()
         self.client = FeedClient()
 
-    @patch('apps.feed.client.httpx.get')
-    def test_get_webcam_list(self, mock_get):
-        mock_response = create_mock_response(200, {'webcams': []})
-        mock_get.return_value = mock_response
-
-        result = self.client.get_webcam_list()
-        self.assertIn('webcams', result)
-
-    @patch('apps.feed.client.FeedClient.get_single_feed')
-    def test_get_webcam(self, mock_get_single):
-        mock_get_single.return_value = {'id': 123, 'name': 'Test'}
+    def test_get_webcam_list(self):
+        fake_cam = MagicMock()
+        fake_cam.cam_internetname = "TestWebCam New"
+        with patch("apps.webcam.models.CameraSource.objects.using") as mock_using:
+            mock_qs = MagicMock()
+            mock_qs.all.return_value = [fake_cam]
+            mock_using.return_value = mock_qs
+            result = self.client.get_webcam_list()
+            self.assertEqual(len(result), 1)
+            self.assertEqual(result[0].cam_internetname, "TestWebCam New")
+            
+    def test_get_webcam_list_exception(self):
+        with patch("apps.webcam.models.CameraSource.objects.using") as mock_using:
+            mock_using.side_effect = Exception("DB error")
 
-        class MockWebcam:
-            id = 123
+            result = self.client.get_webcam_list()
 
-        result = self.client.get_webcam(MockWebcam())
-        self.assertEqual(result, {'id': 123, 'name': 'Test'})
+            assert result == []
+            
 
 
 class TestFeedClientEvent(BaseTest):
@@ -324,15 +284,6 @@ def setUp(self):
         super().setUp()
         self.client = FeedClient()
 
-    @patch('apps.feed.client.httpx.get')
-    def test_get_list_feed_empty_response(self, mock_get):
-        from apps.feed.serializers import WebcamAPISerializer
-
-        mock_response = create_mock_response(200, {'webcams': []})
-        mock_get.return_value = mock_response
-
-        result = self.client.get_list_feed(WEBCAM, 'webcams', WebcamAPISerializer)
-        self.assertIn('webcams', result)
 
 
 class TestFeedClientFerry(BaseTest):

src/backend/apps/webcam/enums.py

@@ -2,9 +2,13 @@
 CAMERA_DIFF_FIELDS = [
     'name',
     'caption',
-    'last_update_modified',
     'is_on',
     'should_appear',
-    'marked_stale',
-    'marked_delayed',
 ]
+
+CAMERA_FIELD_MAPPING = {
+    'name': 'cam_internetname',
+    'caption': 'cam_internetcaption',
+    'is_on': 'isOn',
+    'should_appear': 'should_appear',
+}

src/backend/apps/webcam/models.py

@@ -161,6 +161,7 @@ class CameraSource(models.Model):
     cam_locationsupdated_by = models.CharField(max_length=255, blank=True, null=True)
     cam_locationslast_updated = models.DateTimeField(blank=True, null=True)
     cam_controldisabled = models.BooleanField(default=False)
+    cam_controldisappear = models.BooleanField(default=False)
     isnew = models.BooleanField(default=False)
     cam_maintenanceis_on_demand = models.BooleanField(default=False)
     seq = models.IntegerField(blank=True, null=True)

src/backend/apps/webcam/serializers.py

@@ -39,20 +39,12 @@ def get_caption(self, obj):
 
     def get_links(self, obj):
         local_root = settings.DRIVEBC_IMAGE_BASE_URL
-        proxy_root = settings.DRIVEBC_IMAGE_PROXY_URL
         webcam_id = obj.id
 
         timestamp = int(time.time())
-        if obj.https_cam:
-            links = {
+        links = {
                 "imageDisplay": f"{local_root}images/{webcam_id}.jpg?t={timestamp}",
-                "replayTheDay": f"{local_root}api/webcams/{webcam_id}/replayTheDay/",
-            }
-
-        else:
-            links = {
-                "imageDisplay": f"{local_root}images/{webcam_id}.jpg",
-                "replayTheDay": f"{proxy_root}ReplayTheDay/json/{webcam_id}.json",
+                "replayTheDay": f"{local_root}api/webcams/{webcam_id}/replayTheDay/",                
             }
 
         return links