Merge pull request #946 from laspsandoval/quick_latency_fix

Author: laspsandoval

sds_data_manager/constructs/ialirt_api_manager_construct.py

@@ -223,12 +223,14 @@ def __init__(
             handler="IAlirtCode.ialirt_db_query_api.lambda_handler",
             runtime=lambda_.Runtime.PYTHON_3_12,
             timeout=cdk.Duration.minutes(1),
-            memory_size=1000,
+            # Lambda allocates CPU proportionally to memory,
+            # and DynamoDB queries are often CPU-bound due to
+            # JSON parsing and network serialization.
+            memory_size=2048,  # MB
             environment={
                 "ALGORITHM_TABLE": algorithm_table.table_name,
                 "REGION": env.region,
             },
-            layers=layers,
         )
 
         # Grant the lambda function read/write permissions on the DynamoDB table.
@@ -250,12 +252,14 @@ def __init__(
             handler="IAlirtCode.ialirt_db_query_api_formatted.lambda_handler",
             runtime=lambda_.Runtime.PYTHON_3_12,
             timeout=cdk.Duration.minutes(1),
-            memory_size=1000,
+            # Lambda allocates CPU proportionally to memory,
+            # and DynamoDB queries are often CPU-bound due to
+            # JSON parsing and network serialization.
+            memory_size=2048,
             environment={
                 "ALGORITHM_TABLE": algorithm_table.table_name,
                 "REGION": env.region,
             },
-            layers=layers,
         )
 
         # Grant the lambda function read/write permissions on the DynamoDB table.

sds_data_manager/lambda_code/IAlirtCode/ialirt_db_query_api.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import os
+import time
 from decimal import Decimal
 
 import boto3
@@ -11,6 +12,11 @@
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
+table_name = os.environ.get("ALGORITHM_TABLE")
+region = os.environ.get("AWS_DEFAULT_REGION", "us-west-2")
+dynamodb = boto3.resource("dynamodb", region_name=region)
+table = dynamodb.Table(table_name)
+
 
 def process_item_types(item: dict) -> dict:
     """Convert Decimal values to int/float for known fields.
@@ -67,12 +73,11 @@ def lambda_handler(event, context):  # noqa: PLR0912
         and runtime environment.
 
     """
-    table_name = os.environ.get("ALGORITHM_TABLE")
-    region = os.environ.get("AWS_DEFAULT_REGION", "us-west-2")
-    dynamodb = boto3.resource("dynamodb", region_name=region)
-    table = dynamodb.Table(table_name)
+    t1 = time.perf_counter()
 
     logger.info(f"Received event: {json.dumps(event)}")
+
+    # --- Parse event ---
     params = event.get("queryStringParameters", {})
 
     if not params:
@@ -83,7 +88,9 @@ def lambda_handler(event, context):  # noqa: PLR0912
 
     key_expr = Key("apid").eq(478)
     query_kwargs = {"KeyConditionExpression": key_expr}
+    t2 = time.perf_counter()
 
+    # --- Determine key condition ---
     allowed_params = {
         "met_start",
         "met_end",
@@ -183,10 +190,29 @@ def lambda_handler(event, context):  # noqa: PLR0912
         }
 
     query_kwargs["KeyConditionExpression"] = key_expr
+    t3 = time.perf_counter()
 
+    # --- Query DynamoDB ---
     response = table.query(**query_kwargs)
+    t4 = time.perf_counter()
 
+    # --- Process items ---
     items = response.get("Items", [])
     processed_items = [process_item_types(item) for item in items]
+    t5 = time.perf_counter()
+
+    # --- Serialize to JSON ---
+    json_body = json.dumps(processed_items)
+    t6 = time.perf_counter()
+
+    num_items = len(processed_items)
+
+    text = (
+        f"Param parse: {t2 - t1:.3f}s | KeyCondition setup: {t3 - t2:.3f}s | "
+        f"Query: {t4 - t3:.3f}s | Process: {t5 - t4:.3f}s | "
+        f"JSON: {t6 - t5:.3f}s | TOTAL: {t6 - t1:.3f}s | "
+        f"Items: {num_items}"
+    )
+    logger.info(text)
 
-    return {"statusCode": 200, "body": json.dumps(processed_items)}
+    return {"statusCode": 200, "body": json_body}

sds_data_manager/lambda_code/IAlirtCode/ialirt_db_query_api_formatted.py

@@ -12,6 +12,11 @@
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
+table_name = os.environ.get("ALGORITHM_TABLE")
+region = os.environ.get("AWS_DEFAULT_REGION", "us-west-2")
+dynamodb = boto3.resource("dynamodb", region_name=region)
+table = dynamodb.Table(table_name)
+
 
 def process_item_types(item: dict) -> dict:
     """Convert Decimal values to int/float for known fields.
@@ -75,11 +80,6 @@ def lambda_handler(event, context):  # noqa: PLR0912, PLR0915
     [-2.058, 3.792, -3.989]],
     'time_tag_utc': ['2025-10-02T07:07:13', '2025-10-02T07:07:17'], ...}
     """
-    table_name = os.environ.get("ALGORITHM_TABLE")
-    region = os.environ.get("AWS_DEFAULT_REGION", "us-west-2")
-    dynamodb = boto3.resource("dynamodb", region_name=region)
-    table = dynamodb.Table(table_name)
-
     logger.info(f"Received event: {json.dumps(event)}")
     params = event.get("queryStringParameters", {})
 

tests/lambda_endpoints/test_ialirt_db_query_api.py

@@ -1,11 +1,24 @@
 """Tests for the I-ALiRT DB Query API Lambda function."""
+# ruff: noqa: PLC0415
 
+import importlib
 import json
+import os
 from decimal import Decimal
 
 import pytest
 
-from sds_data_manager.lambda_code.IAlirtCode import ialirt_db_query_api
+
+@pytest.fixture
+def ialirt_db_query_api_module(setup_dynamodb):
+    """Mock the import."""
+    os.environ["ALGORITHM_TABLE"] = setup_dynamodb["algorithm_table"].name
+    os.environ["AWS_DEFAULT_REGION"] = "us-west-2"
+
+    from sds_data_manager.lambda_code.IAlirtCode import ialirt_db_query_api
+
+    importlib.reload(ialirt_db_query_api)
+    return ialirt_db_query_api
 
 
 @pytest.fixture
@@ -50,7 +63,7 @@ def algorithm_table(setup_dynamodb):
     return table
 
 
-def test_query_with_met_range(algorithm_table):
+def test_query_with_met_range(algorithm_table, ialirt_db_query_api_module):
     """Test query with met range."""
     # GET <invoke url>/query?met_start=100&met_end=111
     event = {
@@ -59,7 +72,7 @@ def test_query_with_met_range(algorithm_table):
             "met_end": "111",
         }
     }
-    response = ialirt_db_query_api.lambda_handler(event, context=None)
+    response = ialirt_db_query_api_module.lambda_handler(event, context=None)
     items = json.loads(response["body"])
     met = sorted(item["met"] for item in items)
 
@@ -68,38 +81,38 @@ def test_query_with_met_range(algorithm_table):
     assert met == expected_data
 
 
-def test_query_with_met_start(algorithm_table):
+def test_query_with_met_start(algorithm_table, ialirt_db_query_api_module):
     """Test query with met start."""
     # GET <invoke url>/query?met_start=120
     event = {
         "queryStringParameters": {
             "met_start": "120",
         }
     }
-    response = ialirt_db_query_api.lambda_handler(event, context=None)
+    response = ialirt_db_query_api_module.lambda_handler(event, context=None)
     items = json.loads(response["body"])
     met = sorted(item["met"] for item in items)
 
     expected_data = [120, 130]
     assert met == expected_data
 
 
-def test_query_with_met_end(algorithm_table):
+def test_query_with_met_end(algorithm_table, ialirt_db_query_api_module):
     """Test query with met end."""
     # GET <invoke url>/query?met_end=120
     event = {
         "queryStringParameters": {
             "met_end": "120",
         }
     }
-    response = ialirt_db_query_api.lambda_handler(event, context=None)
+    response = ialirt_db_query_api_module.lambda_handler(event, context=None)
 
     assert response["statusCode"] == 400
     expected_message = {"message": "Cannot query by end time without start time"}
     assert json.loads(response["body"]) == expected_message
 
 
-def test_query_with_utc_range(algorithm_table):
+def test_query_with_utc_range(algorithm_table, ialirt_db_query_api_module):
     """Test query_with_utc_range."""
     # GET <invoke url>/query?met_in_utc_start=<met_in_utc_start>&
     # met_in_utc_end=<met_in_utc_end>
@@ -109,7 +122,7 @@ def test_query_with_utc_range(algorithm_table):
             "met_in_utc_end": "2021-01-03T00:00:00",
         }
     }
-    response = ialirt_db_query_api.lambda_handler(event, context=None)
+    response = ialirt_db_query_api_module.lambda_handler(event, context=None)
     items = json.loads(response["body"])
 
     utc = sorted(item["met_in_utc"] for item in items)
@@ -123,15 +136,15 @@ def test_query_with_utc_range(algorithm_table):
     assert utc == expected_utc
 
 
-def test_query_with_utc_start(algorithm_table):
+def test_query_with_utc_start(algorithm_table, ialirt_db_query_api_module):
     """Test with insert time start."""
     # GET <invoke url>/query?utc_start=<utc_start>
     event = {
         "queryStringParameters": {
             "met_in_utc_start": "2021-01-02T00:00:00",
         }
     }
-    response = ialirt_db_query_api.lambda_handler(event, context=None)
+    response = ialirt_db_query_api_module.lambda_handler(event, context=None)
     items = json.loads(response["body"])
 
     utcs = sorted(item["met_in_utc"] for item in items)
@@ -145,21 +158,21 @@ def test_query_with_utc_start(algorithm_table):
     assert utcs == expected_data
 
 
-def test_query_with_utc_end(algorithm_table):
+def test_query_with_utc_end(algorithm_table, ialirt_db_query_api_module):
     """Test query with insert time end."""
     # GET <invoke url>/query?met_in_utc_end=<met_in_utc_end>
     event = {
         "queryStringParameters": {
             "met_in_utc_end": "2021-01-02T00:00:00",
         }
     }
-    response = ialirt_db_query_api.lambda_handler(event, context=None)
+    response = ialirt_db_query_api_module.lambda_handler(event, context=None)
     assert response["statusCode"] == 400
     expected_message = {"message": "Cannot query by end time without start time"}
     assert json.loads(response["body"]) == expected_message
 
 
-def test_query_no_results(algorithm_table):
+def test_query_no_results(algorithm_table, ialirt_db_query_api_module):
     """Test query if there are no results."""
     # GET <invoke url>/query?met_start=<met_start>&met_end=<met_end>
     event = {
@@ -168,12 +181,12 @@ def test_query_no_results(algorithm_table):
             "met_end": "300",
         }
     }
-    response = ialirt_db_query_api.lambda_handler(event, context=None)
+    response = ialirt_db_query_api_module.lambda_handler(event, context=None)
     assert response["statusCode"] == 200
     assert json.loads(response["body"]) == []
 
 
-def test_query_with_multiple_filters(algorithm_table):
+def test_query_with_multiple_filters(algorithm_table, ialirt_db_query_api_module):
     """Test query with multiple filters."""
     # GET <invoke url>/query?met_start=100&met_end=130&product_name=codicelo_product_1
     event = {
@@ -182,13 +195,13 @@ def test_query_with_multiple_filters(algorithm_table):
             "met_end": "130",
         }
     }
-    response = ialirt_db_query_api.lambda_handler(event, context=None)
+    response = ialirt_db_query_api_module.lambda_handler(event, context=None)
 
     items = json.loads(response["body"])
     assert len(items) == 4
 
 
-def test_query_with_different_time_queries(algorithm_table):
+def test_query_with_different_time_queries(algorithm_table, ialirt_db_query_api_module):
     """Test query API with multiple filters."""
     # GET <invoke url>/query?met_start=100&met_end=130&product_name=hit*&
     # met_in_utc_start=2021-01-02T00:00:00.
@@ -199,41 +212,41 @@ def test_query_with_different_time_queries(algorithm_table):
             "met_in_utc_start": "2021-01-02T00:00:00",
         }
     }
-    response = ialirt_db_query_api.lambda_handler(event, context=None)
+    response = ialirt_db_query_api_module.lambda_handler(event, context=None)
     assert response["statusCode"] == 400
     expected_message = {
         "message": "Cannot query multiple time keys (met, met_in_utc, last_modified)"
     }
     assert json.loads(response["body"]) == expected_message
 
 
-def test_query_with_invalid_parameters(algorithm_table):
+def test_query_with_invalid_parameters(algorithm_table, ialirt_db_query_api_module):
     """Test query with invalid parameters."""
     # GET <invoke url>/query?met_bad=100.
     event = {
         "queryStringParameters": {
             "met_bad": "100",
         }
     }
-    response = ialirt_db_query_api.lambda_handler(event, context=None)
+    response = ialirt_db_query_api_module.lambda_handler(event, context=None)
 
     assert response["statusCode"] == 400
     expected_message = {"message": "Unexpected parameters: met_bad"}
     assert json.loads(response["body"]) == expected_message
 
 
-def test_query_with_no_parameters(algorithm_table):
+def test_query_with_no_parameters(algorithm_table, ialirt_db_query_api_module):
     """Test query with no parameters."""
     # GET <invoke url>/query.
     event = {"queryStringParameters": None}
-    response = ialirt_db_query_api.lambda_handler(event, context=None)
+    response = ialirt_db_query_api_module.lambda_handler(event, context=None)
 
     assert response["statusCode"] == 400
     expected_message = {"message": "No query parameters provided"}
     assert json.loads(response["body"]) == expected_message
 
 
-def test_query_with_mixed_parameters(algorithm_table):
+def test_query_with_mixed_parameters(algorithm_table, ialirt_db_query_api_module):
     """Test query with mixed parameters."""
     # GET <invoke url>/query?met_start=100&met_in_utc_end=2021-01-02T00:00:00.
     event = {
@@ -242,7 +255,7 @@ def test_query_with_mixed_parameters(algorithm_table):
             "met_in_utc_end": "2021-01-02T00:00:00",
         }
     }
-    response = ialirt_db_query_api.lambda_handler(event, context=None)
+    response = ialirt_db_query_api_module.lambda_handler(event, context=None)
 
     assert response["statusCode"] == 400
     expected_message = {
@@ -251,7 +264,7 @@ def test_query_with_mixed_parameters(algorithm_table):
     assert json.loads(response["body"]) == expected_message
 
 
-def test_process_item_types():
+def test_process_item_types(ialirt_db_query_api_module):
     """Test process_item_types function."""
     items = [
         {
@@ -265,7 +278,9 @@ def test_process_item_types():
         }
     ]
 
-    processed_items = [ialirt_db_query_api.process_item_types(item) for item in items]
+    processed_items = [
+        ialirt_db_query_api_module.process_item_types(item) for item in items
+    ]
 
     assert processed_items == [
         {