Create a separate lambda-handler service for lambdas (#337)

## Description

Creates a separate package and moves tests from ttc-lambda into that
folder. Updates relevant sections of pyproject.tomls.
## Related Issues

Closes #324 

## Additional Notes

[Add any additional context or notes that reviewers should know about.]

<--------------------- REMOVE THE LINES BELOW BEFORE MERGING
--------------------->

## Checklist

Please review and complete the following checklist before submitting
your pull request:

- [ ] I have ensured that the pull request is of a manageable size,
allowing it to be reviewed within a single session.
- [ ] I have reviewed my changes to ensure they are clear, concise, and
well-documented.
- [ ] I have updated the documentation, if applicable.
- [ ] I have added or updated test cases to cover my changes, if
applicable.
- [ ] I have minimized the number of reviewers to include only those
essential for the review.

## Checklist for Reviewers

Please review and complete the following checklist during the review
process:

- [ ] The code follows best practices and conventions.
- [ ] The changes implement the desired functionality or fix the
reported issue.
- [ ] The tests cover the new changes and pass successfully.
- [ ] Any potential edge cases or error scenarios have been considered.

Author: robertandremitchell

packages/augmentation-lambda/pyproject.toml

@@ -12,4 +12,7 @@ requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
-packages = ["src/augmentation_lambda"]
+packages = ["src/augmentation_lambda"]
+
+[tool.uv.sources]
+lambda-handler = { workspace = true }

packages/index-lambda/pyproject.toml

@@ -2,7 +2,14 @@
 name = "index-lambda"
 version = "0.1.0"
 readme = "README.md"
-dependencies = ["boto3", "opensearch-py", "requests_aws4auth", "aws_lambda_typing>=2.20.0", "botocore"]
+dependencies = [
+    "boto3",
+    "opensearch-py",
+    "requests-aws4auth",
+    "aws-lambda-typing>=2.20.0",
+    "botocore",
+    "lambda-handler",
+]
 
 [dependency-groups]
 dev = []
@@ -12,4 +19,7 @@ requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
-packages = ["src/index_lambda"]
+packages = ["src/index_lambda"]
+
+[tool.uv.sources]
+lambda-handler = { workspace = true }

packages/index-lambda/src/index_lambda/lambda_function.py

@@ -1,8 +1,4 @@
-import os
-import sys
-
-sys.path.append(os.path.join(os.path.dirname(__file__), "../../../text-to-code-lambda/src"))
-from text_to_code_lambda import s3_handler
+import lambda_handler
 
 
 def handler(event: dict, context: dict) -> dict:
@@ -42,9 +38,9 @@ def handler(event: dict, context: dict) -> dict:
     }
 
     # Configure OpenSearch client
-    aws_auth = s3_handler.create_aws_auth()
-    os_client = s3_handler.create_opensearch_client(aws_auth)
-    index_name = s3_handler.require_env("INDEX_NAME")
+    aws_auth = lambda_handler.create_aws_auth()
+    os_client = lambda_handler.create_opensearch_client(aws_auth)
+    index_name = lambda_handler.require_env("INDEX_NAME")
 
     # Create index if it doesn't already exist
     if not os_client.indices.exists(index=index_name):

packages/index-lambda/tests/test_index_lambda_function.py

@@ -43,7 +43,7 @@ def __init__(self, description_vector_type: str) -> None:
         self.indices = MockIndices(description_vector_type)
 
 
-def patch_s3_handler(monkeypatch, description_vector_type: str) -> None:
+def patch_lambda_handler(monkeypatch, description_vector_type: str) -> None:
     def mock_create_aws_auth() -> object:
         """Mock create_aws_auth function that returns a dummy AWS auth object."""
         return object()
@@ -58,19 +58,19 @@ def mock_require_env(name: str) -> str:
             return INDEX_NAME
         raise ValueError(f"Unexpected env var requested: {name}")
 
-    monkeypatch.setattr(lambda_function.s3_handler, "create_aws_auth", mock_create_aws_auth)
+    monkeypatch.setattr(lambda_function.lambda_handler, "create_aws_auth", mock_create_aws_auth)
     monkeypatch.setattr(
-        lambda_function.s3_handler,
+        lambda_function.lambda_handler,
         "create_opensearch_client",
         mock_create_opensearch_client,
     )
-    monkeypatch.setattr(lambda_function.s3_handler, "require_env", mock_require_env)
+    monkeypatch.setattr(lambda_function.lambda_handler, "require_env", mock_require_env)
 
 
 class TestHandler:
     def test_handler_success(self, monkeypatch):
         """Test handler creates the index when it does not exist and returns expected response."""
-        patch_s3_handler(monkeypatch, "knn_vector")
+        patch_lambda_handler(monkeypatch, "knn_vector")
 
         resp = lambda_function.handler({}, {})
 
@@ -88,7 +88,7 @@ def test_handler_success(self, monkeypatch):
 
     def test_handler_recreates_index_when_vector_mapping_incorrect(self, monkeypatch):
         """Test handler recreates the index when description_vector mapping is not knn_vector."""
-        patch_s3_handler(monkeypatch, "keyword")
+        patch_lambda_handler(monkeypatch, "keyword")
 
         resp = lambda_function.handler({}, {})
 

packages/lambda-handler/README.md

@@ -0,0 +1,17 @@
+[project]
+name = "lambda-handler"
+version = "0.1.0"
+readme = "README.md"
+dependencies = [
+    "boto3>=1.40.60",
+]
+
+[dependency-groups]
+dev = ["moto"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/lambda_handler"]

packages/lambda-handler/pyproject.toml

@@ -0,0 +1,11 @@
+from .lambda_handler import check_s3_object_exists as check_s3_object_exists
+from .lambda_handler import create_aws_auth as create_aws_auth
+from .lambda_handler import create_opensearch_client as create_opensearch_client
+from .lambda_handler import create_s3_client as create_s3_client
+from .lambda_handler import get_eventbridge_data_from_s3_event as get_eventbridge_data_from_s3_event
+from .lambda_handler import get_file_content_from_s3 as get_file_content_from_s3
+from .lambda_handler import get_persistence_id as get_persistence_id
+from .lambda_handler import get_s3_credentials as get_s3_credentials
+from .lambda_handler import put_file as put_file
+from .lambda_handler import require_env as require_env
+from .lambda_handler import strip_protocol as strip_protocol

packages/lambda-handler/src/lambda_handler/__init__.py

@@ -0,0 +1,162 @@
+import os
+import typing
+
+import boto3
+from aws_lambda_typing import events as lambda_events
+from botocore.client import BaseClient
+from botocore.credentials import Credentials
+from botocore.exceptions import ClientError
+from opensearchpy import OpenSearch
+from opensearchpy import RequestsHttpConnection
+from requests_aws4auth import AWS4Auth
+
+
+def require_env(name: str) -> str:
+    """Fetch a required environment variable or raise a clear error.
+
+    :param name: The name of the environment variable to fetch.
+    :return: The value of the environment variable.
+    """
+    value = os.getenv(name)
+    if not value:
+        raise ValueError(f"{name} not set as an environment variable.")
+    return value
+
+
+def strip_protocol(url: str) -> str:
+    """Remove http/https from a URL.
+
+    This is sometimes needed for AWS service
+    endpoints (like OpenSearch) that require the URL without the protocol.
+    :param url: The URL to strip.
+    :return: The URL without the protocol.
+    """
+    return url.removeprefix("https://").removeprefix("http://")
+
+
+def get_s3_credentials() -> Credentials:
+    """Fetch AWS credentials from the environment."""
+    return boto3.Session().get_credentials()
+
+
+def create_aws_auth() -> AWS4Auth:
+    """Creates an AWS4Auth object for authenticating with AWS services.
+
+    :return: AWS4Auth object
+    """
+    credentials = get_s3_credentials()
+    return AWS4Auth(
+        credentials.access_key,
+        credentials.secret_key,
+        require_env("AWS_REGION"),
+        "es",
+        session_token=credentials.token,
+    )
+
+
+def create_s3_client() -> BaseClient:
+    """Creates an S3 client.
+
+    :return: S3 client
+    """
+    endpoint_url = os.getenv("S3_ENDPOINT_URL")
+    region_name = require_env("AWS_REGION")
+
+    return boto3.client("s3", endpoint_url=endpoint_url, region_name=region_name)
+
+
+def create_opensearch_client(aws_auth: AWS4Auth) -> OpenSearch:
+    """Creates an OpenSearch client.
+
+    :param aws_auth: AWS4Auth object for authentication
+    :return: OpenSearch client
+    """
+    endpoint_url = require_env("OPENSEARCH_ENDPOINT_URL")
+    return OpenSearch(
+        hosts=[{"host": strip_protocol(endpoint_url), "port": 443}],
+        http_auth=aws_auth,
+        use_ssl=True,
+        verify_certs=True,
+        connection_class=RequestsHttpConnection,
+    )
+
+
+def get_file_content_from_s3(bucket_name: str, object_key: str) -> str:
+    """Extracts the file content from an S3 bucket.
+
+    :param bucket_name: The name of the S3 bucket.
+    :param object_key: The key of the S3 object.
+    :return: The content of the file as a string.
+    """
+    client = create_s3_client()
+
+    # Check if object exists
+    if not check_s3_object_exists(client, bucket_name, object_key):
+        raise FileNotFoundError(f"S3 object not found: {bucket_name}/{object_key}")
+
+    response = client.get_object(Bucket=bucket_name, Key=object_key)
+    return response["Body"].read().decode("utf-8")
+
+
+def get_eventbridge_data_from_s3_event(event: lambda_events.EventBridgeEvent) -> dict:
+    """Extracts the file metadata from an S3 event triggered by a Lambda function.
+
+    :param event: The S3 event containing the bucket and object key information.
+    :return: A dictionary containing the bucket name and object key.
+    """
+    bucket_name = event["detail"]["bucket"]["name"]
+    object_key = event["detail"]["object"]["key"]
+
+    return {"bucket_name": bucket_name, "object_key": object_key}
+
+
+def put_file(file_obj: typing.BinaryIO, bucket_name: str, object_key: str) -> None:
+    """Uploads a file object to a S3 bucket.
+
+    :param file_obj: The file object to upload.
+    :param bucket_name: The name of the S3 bucket to upload to.
+    :param object_key: The key to assign to the uploaded object in S3.
+    """
+    client = create_s3_client()
+    client.put_object(Body=file_obj, Bucket=bucket_name, Key=object_key)
+
+
+def check_s3_object_exists(s3_client: BaseClient, bucket: str, key: str) -> bool:
+    """Checks that an S3 object exists.
+
+    :param s3_client: The S3 client.
+    :param bucket: The name of the S3 bucket.
+    :param key: The key of the S3 object.
+    :raises Exception: If an unexpected error occurs while fetching the S3 object.
+    :return: True if the S3 object exists, False otherwise.
+    """
+    try:
+        s3_client.head_object(Bucket=bucket, Key=key)
+        return True
+    except ClientError as e:
+        error_code = e.response["Error"]["Code"]
+
+        if error_code in ("404", "NoSuchKey"):
+            return False
+
+        msg = f"Unexpected error while fetching file from S3: {bucket}/{key}. Error: {e.response['Error']['Message']}"
+        raise Exception(msg) from e
+
+
+def get_persistence_id(object_key: str, input_prefix: str) -> str:
+    """Get the persistence_id from an S3 object key.
+
+    Object key format: <pipeline-step>/<persistance_id>
+    Example: TTCInput/2026/01/01/0026b704-f510-4494-8d21-11d27217d96e
+    Returns: 2026/01/01/0026b704-f510-4494-8d21-11d27217d96e
+
+    :param object_key: The S3 object key
+    :param input_prefix: The pipeline step prefix (e.g., "TTCInput/")
+    :return: The persistence_id portion of the key
+
+    """
+    if not object_key.startswith(input_prefix):
+        raise ValueError(
+            f"Object key '{object_key}' does not start with expected prefix '{input_prefix}'"
+        )
+    return object_key[len(input_prefix) :]

packages/lambda-handler/src/lambda_handler/lambda_handler.py

@@ -0,0 +1,99 @@
+import json
+import logging
+import os
+
+import boto3
+import moto
+import pytest
+
+
+@pytest.fixture(scope="function")
+def moto_setup(monkeypatch: pytest.MonkeyPatch) -> boto3.client:
+    """Setup test AWS."""
+    with moto.mock_aws():
+        monkeypatch.setenv("AWS_REGION", "us-east-1")
+        monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test_access_key_id")
+        monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test_secret_access_key")
+        bucket_name = "test-bucket"
+        monkeypatch.setenv("OPENSEARCH_ENDPOINT_URL", "https://test-opensearch-endpoint.com")
+
+        # Create the fake S3 bucket
+        s3 = boto3.client(
+            "s3",
+            region_name=os.environ["AWS_REGION"],
+            aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+            aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+        )
+        s3.create_bucket(Bucket=bucket_name)
+
+        # Add convenience attribute for tests
+        s3.bucket_name = bucket_name
+
+        yield s3
+
+
+@pytest.fixture
+def example_s3_event_payload() -> dict:
+    """Inner S3 event payload (what SQS body contains as JSON string).
+
+    This example content comes from APHL
+    """
+    return {
+        "version": "0",
+        "id": "12345678-1234-5678-9012-123456789012",
+        "detail-type": "Object Created",
+        "source": "aws.s3",
+        "account": "111122223333",
+        "time": "2025-09-03T12:34:56Z",
+        "region": "us-east-1",
+        "resources": ["arn:aws:s3:::my-bucket-name"],
+        "detail": {
+            "version": "0",
+            "bucket": {"name": "ecr-bucket"},
+            "object": {
+                "key": "TextToCodeSubmission/2025/09/03/1-5f84c7a5-91d7f5c6a2b7c9e08f0d1234",
+                "size": 1024,
+                "etag": "0123456789abcdef0123456789abcdef",
+                "sequencer": "0055AED6DCD90281E5",
+            },
+            "request-id": "C3D13FE58DE4C810",
+            "requester": "arn:aws:iam::111122223333:user/example-user",
+            "reason": "PutObject",
+        },
+    }
+
+
+@pytest.fixture
+def example_sqs_event(example_s3_event_payload: dict) -> dict:
+    """Full SQS event that mimics real Lambda input."""
+    return {
+        "Records": [
+            {
+                "messageId": "f9ccdff5-0acb-4933-8995-bd7f0ab5f2f7",
+                "receiptHandle": "test-receipt-handle",
+                "body": json.dumps(example_s3_event_payload),
+                "attributes": {
+                    "ApproximateReceiveCount": "1",
+                    "SentTimestamp": "1752691260451",
+                    "SenderId": "AIDAJXNJGGKNS7OSV23OI",
+                    "ApproximateFirstReceiveTimestamp": "1752691260458",
+                },
+                "messageAttributes": {},
+                "md5OfBody": "dummy-md5",
+                "eventSource": "aws:sqs",
+                "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:queue-name",
+                "awsRegion": "us-east-1",
+            }
+        ]
+    }
+
+
+@pytest.fixture
+def caplog_warning(caplog: pytest.LogCaptureFixture) -> logging.Logger:
+    """Capture log warnings for tests.
+
+    :param caplog: Pytest fixture for capturing log output
+    :return: Caplog instance with warning level set
+    """
+    caplog.set_level(logging.WARNING)
+    return caplog