BUG: only return data available in both DB and S3

sds_data_manager/constructs/sds_api_manager_construct.py

@@ -187,6 +187,7 @@ def __init__(
             environment={
                 "REGION": env.region,
                 "SECRET_NAME": db_secret_name,
+                "S3_BUCKET": data_bucket.bucket_name,
             },
             layers=layers,
         )

sds_data_manager/lambda_code/SDSCode/api_lambdas/query_api.py

@@ -3,8 +3,10 @@
 import datetime
 import json
 import logging
+import os
 from collections import namedtuple
 
+import boto3
 from sqlalchemy import func, select
 
 from ..api_lambdas.utils import is_authenticated_user
@@ -16,7 +18,7 @@
 logger.setLevel(logging.INFO)
 
 
-def lambda_handler(event, context):  # noqa: PLR0912
+def lambda_handler(event, context):  # noqa: PLR0912 PLR0915
     """Entry point to the query API lambda.
 
     Parameters
@@ -131,6 +133,16 @@ def lambda_handler(event, context):  # noqa: PLR0912
     # Convert the search results (list of tuples) to a list of dicts
     search_results = [result._asdict() for result in search_results]
 
+    # Check if those files exists in S3 before returning them
+    s3_client = boto3.client("s3")
+    data_bucket = boto3.resource("s3").Bucket(os.environ["S3_BUCKET"])
+    existing_files = []
+    for result in search_results:
+        s3_key = result["file_path"]
+        if s3_client.head_object(Bucket=data_bucket.bucket_name, Key=s3_key):
+            existing_files.append(result)
+
+    search_results = existing_files
     # Convert datetimes to string values of format 'YYYYMMDD'
     # Also remove values that are not needed by users
     for result in search_results:

tests/lambda_endpoints/test_query_api.py

@@ -2,7 +2,9 @@
 
 import datetime
 import json
+from unittest.mock import MagicMock
 
+import boto3
 import pytest
 
 from sds_data_manager.lambda_code.SDSCode.api_lambdas import query_api
@@ -38,6 +40,33 @@ def _populate_test_data(session):
     session.commit()
 
 
+@pytest.fixture(autouse=True)
+def mock_head_object(monkeypatch):
+    """Patch boto3's S3 client head_object to always return success."""
+    # Create a mock that always returns a success response
+    mock_head_object = MagicMock(
+        return_value={"ResponseMetadata": {"HTTPStatusCode": 200}}
+    )
+
+    # Also patch the boto3.resource to handle the bucket name issue
+    mock_bucket = MagicMock()
+    mock_bucket.name = "mock-bucket-name"  # Add a name attribute that can be used
+
+    # Patch both s3_client.head_object and the bucket name access
+    def mock_client(*args, **kwargs):
+        mock = MagicMock()
+        mock.head_object = mock_head_object
+        return mock
+
+    def mock_resource(*args, **kwargs):
+        mock = MagicMock()
+        mock.Bucket.return_value = mock_bucket
+        return mock
+
+    monkeypatch.setattr(boto3, "client", mock_client)
+    monkeypatch.setattr(boto3, "resource", mock_resource)
+
+
 @pytest.fixture
 def expected_response():
     """Return the expected response."""