Merge branch 'main' into dependabot/github_actions/codecov/codecov-action-6

Author: nickclyde

.github/workflows/deploy.yaml

@@ -60,7 +60,7 @@ jobs:
       - name: Create ECR repositories
         if: ${{ inputs.apply && !inputs.destroy }}
         working-directory: terraform
-        run: terraform apply -auto-approve -target=aws_ecr_repository.index_lambda -target=aws_ecr_repository.ttc_lambda
+        run: terraform apply -auto-approve -target=aws_ecr_repository.index_lambda -target=aws_ecr_repository.ttc_lambda -target=aws_ecr_repository.augmentation_lambda
 
       - name: Login to Amazon ECR
         if: ${{ inputs.apply && !inputs.destroy }}
@@ -76,22 +76,36 @@ jobs:
           echo "index_ecr_url=$INDEX_ECR_URL" >> "$GITHUB_OUTPUT"
           ECR_URL=$(terraform output -raw ecr_repository_url)
           echo "ecr_url=$ECR_URL" >> "$GITHUB_OUTPUT"
+          AUG_ECR_URL=$(terraform output -raw augmentation_ecr_repository_url)
+          echo "aug_ecr_url=$AUG_ECR_URL" >> "$GITHUB_OUTPUT"
 
       - name: Build and push Index Docker image
         if: ${{ inputs.apply && !inputs.destroy }}
         run: |
           INDEX_ECR_URL="${{ steps.ecr-url.outputs.index_ecr_url }}"
-          docker build -f Dockerfile.index -t "$INDEX_ECR_URL:${{ github.sha }}" -t "$INDEX_ECR_URL:latest" .
+          docker build -f Dockerfile.index -t "$INDEX_ECR_URL:${{ github.sha }}" -t "$INDEX_ECR_URL:latest" --secret id=huggingface_token,env=HF_TOKEN .
           docker push "$INDEX_ECR_URL:${{ github.sha }}"
           docker push "$INDEX_ECR_URL:latest"
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
 
       - name: Build and push TTC Docker image
         if: ${{ inputs.apply && !inputs.destroy }}
         run: |
           ECR_URL="${{ steps.ecr-url.outputs.ecr_url }}"
-          docker build -f Dockerfile.ttc -t "$ECR_URL:${{ github.sha }}" -t "$ECR_URL:latest" .
+          docker build -f Dockerfile.ttc -t "$ECR_URL:${{ github.sha }}" -t "$ECR_URL:latest" --secret id=huggingface_token,env=HF_TOKEN .
           docker push "$ECR_URL:${{ github.sha }}"
           docker push "$ECR_URL:latest"
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+
+      - name: Build and push Augmentation Docker image
+        if: ${{ inputs.apply && !inputs.destroy }}
+        run: |
+          AUG_ECR_URL="${{ steps.ecr-url.outputs.aug_ecr_url }}"
+          docker build -f Dockerfile.augmentation -t "$AUG_ECR_URL:${{ github.sha }}" -t "$AUG_ECR_URL:latest" .
+          docker push "$AUG_ECR_URL:${{ github.sha }}"
+          docker push "$AUG_ECR_URL:latest"
 
       - name: Terraform Plan
         if: ${{ !inputs.apply && !inputs.destroy }}
@@ -101,7 +115,7 @@ jobs:
       - name: Terraform Apply
         if: ${{ inputs.apply && !inputs.destroy }}
         working-directory: terraform
-        run: terraform apply -auto-approve -var="index_lambda_image_tag=${{ github.sha }}" -var="ttc_lambda_image_tag=${{ github.sha }}"
+        run: terraform apply -auto-approve -var="index_lambda_image_tag=${{ github.sha }}" -var="ttc_lambda_image_tag=${{ github.sha }}" -var="augmentation_lambda_image_tag=${{ github.sha }}"
 
       - name: Terraform Destroy
         if: ${{ inputs.destroy }}

.github/workflows/slack_notifier_review.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   slack-ready-for-review:
     runs-on: ubuntu-latest
-    if: ${{ !github.event.pull_request.draft }}
+    if: ${{ !github.event.pull_request.draft && github.actor != 'dependabot[bot]' }}
 
     steps:
       - name: Send GitHub trigger payload to Slack

Dockerfile.ttc

@@ -14,17 +14,27 @@ COPY ./packages/lambda-handler ${LAMBDA_TASK_ROOT}/lambda-handler
 RUN pip install --no-cache-dir "${LAMBDA_TASK_ROOT}/lambda-handler"
 
 COPY ./packages/text-to-code ${LAMBDA_TASK_ROOT}/text-to-code
+# Install CPU-only PyTorch to reduce image size (Lambda doesn't have GPUs)
+RUN pip install --no-cache-dir torch==2.9.1 --index-url https://download.pytorch.org/whl/cpu
 RUN pip install --no-cache-dir "${LAMBDA_TASK_ROOT}/text-to-code"
 
 COPY ./packages/text-to-code-lambda ${LAMBDA_TASK_ROOT}/text-to-code-lambda
 RUN pip install --no-cache-dir "${LAMBDA_TASK_ROOT}/text-to-code-lambda"
 
+# Remove build tools no longer needed at runtime
+RUN rpm -e --nodeps gcc-c++ gcc cpp make && microdnf clean all && rm -rf /var/cache/*
+
 # Download retriever at build time (private repo, needs token)
 RUN --mount=type=secret,id=huggingface_token,env=HF_TOKEN \
-  python -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='NCHS/ttc-retriever-mvp', local_dir='/opt/retriever_model', ignore_patterns=['*.git*', '*.md', 'onnx/*', 'openvino/*', 'pytorch_model.bin'])"
+  python -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='NCHS/ttc-retriever-mvp', local_dir='/opt/retriever_model', ignore_patterns=['*.git*', '*.md', 'onnx/*', 'openvino/*', 'pytorch_model.bin', 'tf_model.h5', 'flax_model.msgpack', 'model.onnx'])"
+
+# Download reranker at build time (private repo, needs token)
+RUN --mount=type=secret,id=huggingface_token,env=HF_TOKEN \
+    python -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='NCHS/ttc-reranker-mvp', local_dir='/opt/reranker_model', ignore_patterns=['*.git*', '*.md', 'onnx/*', 'openvino/*', 'pytorch_model.bin', 'tf_model.h5', 'flax_model.msgpack', 'model.onnx'])" \
+  && rm -rf /root/.cache/huggingface
 
-# Download reranker at build time (public repo)
-RUN python -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='cross-encoder/stsb-roberta-large', local_dir='/opt/reranker_model', ignore_patterns=['*.git*', '*.md', 'onnx/*', 'openvino/*', 'pytorch_model.bin'])"
+# Clean up package source copies
+RUN rm -rf ${LAMBDA_TASK_ROOT}/shared-models ${LAMBDA_TASK_ROOT}/lambda-handler ${LAMBDA_TASK_ROOT}/text-to-code ${LAMBDA_TASK_ROOT}/text-to-code-lambda
 
 ENV RETRIEVER_MODEL_PATH="/opt/retriever_model"
 ENV RERANKER_MODEL_PATH="/opt/reranker_model"

packages/augmentation-lambda/src/augmentation_lambda/lambda_function.py

@@ -5,18 +5,18 @@
 
 from aws_lambda_typing import context as lambda_context
 from aws_lambda_typing import events as lambda_events
+from botocore.client import BaseClient
 
 import lambda_handler
 from augmentation.models import TTCAugmenterConfig
-from botocore.client import BaseClient
 from augmentation.models.application import TTCAugmenterOutput
 from augmentation.services.eicr_augmenter import EICRAugmenter
 from shared_models import TTCAugmenterInput
 
 # Environment variables
 S3_BUCKET = os.getenv("S3_BUCKET", "dibbs-text-to-code")
 AUGMENTED_EICR_PREFIX = os.getenv("AUGMENTED_EICR_PREFIX", "AugmentationEICRV2/")
-AUGMENTATION_METADATA_PREFIX = os.getenv("AUGMENTATION_METADATA_PREFIX", "AugmentationMetadata/")
+AUGMENTATION_METADATA_PREFIX = os.getenv("AUGMENTATION_METADATA_PREFIX", "AugmentationMetadataV2/")
 
 # Cache S3 client to reuse across Lambda invocations
 _cached_s3_client: BaseClient | None = None

packages/text-to-code-lambda/src/text_to_code_lambda/lambda_function.py

@@ -27,10 +27,10 @@
 # Environment variables
 S3_BUCKET = os.getenv("S3_BUCKET", "dibbs-text-to-code")
 EICR_INPUT_PREFIX = os.getenv("EICR_INPUT_PREFIX", "eCRMessageV2/")
-SCHEMATRON_ERROR_PREFIX = os.getenv("SCHEMATRON_ERROR_PREFIX", "schematronErrors/")
-TTC_INPUT_PREFIX = os.getenv("TTC_INPUT_PREFIX", "TextToCodeValidateSubmissionV2/")
-TTC_OUTPUT_PREFIX = os.getenv("TTC_OUTPUT_PREFIX", "TTCOutput/")
-TTC_METADATA_PREFIX = os.getenv("TTC_METADATA_PREFIX", "TTCMetadata/")
+SCHEMATRON_ERROR_PREFIX = os.getenv("SCHEMATRON_ERROR_PREFIX", "ValidationResponseV2/")
+TTC_INPUT_PREFIX = os.getenv("TTC_INPUT_PREFIX", "TextToCodeSubmissionV2/")
+TTC_OUTPUT_PREFIX = os.getenv("TTC_OUTPUT_PREFIX", "TTCAugmentationMetadataV2/")
+TTC_METADATA_PREFIX = os.getenv("TTC_METADATA_PREFIX", "TTCMetadataV2/")
 AWS_REGION = os.getenv("AWS_REGION")
 S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL")
 OPENSEARCH_ENDPOINT_URL = os.getenv("OPENSEARCH_ENDPOINT_URL")
@@ -120,7 +120,8 @@ def process_record(record: SQSRecord, s3_client: BaseClient, opensearch_client:
     # Parse the EventBridge S3 event from the SQS message body
     eventbridge_data = lambda_handler.get_eventbridge_data_from_s3_event(s3_event)
     object_key = eventbridge_data["object_key"]
-    logger.info(f"Processing S3 Object: s3://{S3_BUCKET}/{object_key}")
+    bucket_name = eventbridge_data.get("bucket_name") or S3_BUCKET
+    logger.info(f"Processing S3 Object: s3://{bucket_name}/{object_key}")
 
     # Extract persistence_id from the RR object key
     persistence_id = lambda_handler.get_persistence_id(object_key, TTC_INPUT_PREFIX)
@@ -129,7 +130,7 @@ def process_record(record: SQSRecord, s3_client: BaseClient, opensearch_client:
     with logger.append_context_keys(
         persistence_id=persistence_id,
     ):
-        _process_record_pipeline(persistence_id, s3_client, opensearch_client)
+        _process_record_pipeline(persistence_id, s3_client, opensearch_client, bucket_name)
 
 
 def _initialize_ttc_outputs(persistence_id: str) -> tuple[dict, dict]:
@@ -152,17 +153,20 @@ def _initialize_ttc_outputs(persistence_id: str) -> tuple[dict, dict]:
     return ttc_output, ttc_metadata_output
 
 
-def _load_schematron_data_fields(persistence_id: str, s3_client: BaseClient) -> list:
+def _load_schematron_data_fields(
+    persistence_id: str, s3_client: BaseClient, bucket_name: str
+) -> list:
     """Load Schematron errors from S3 and extract relevant fields.
 
     :param persistence_id: The persistence ID extracted from the S3 object key
     :param s3_client: The S3 client to use for fetching files.
+    :param bucket_name: The S3 bucket name to read from.
     :return: The relevant Schematron data fields for TTC processing.
     """
     object_key = f"{SCHEMATRON_ERROR_PREFIX}{persistence_id}"
-    logger.info("Loading Schematron errors", s3_key=f"s3://{S3_BUCKET}/{object_key}")
+    logger.info("Loading Schematron errors", s3_key=f"s3://{bucket_name}/{object_key}")
     schematron_errors = lambda_handler.get_file_content_from_s3(
-        bucket_name=S3_BUCKET,
+        bucket_name=bucket_name,
         object_key=object_key,
         s3_client=s3_client,
     )
@@ -172,17 +176,18 @@ def _load_schematron_data_fields(persistence_id: str, s3_client: BaseClient) ->
     return schematron_processor.get_data_fields_from_schematron_error(schematron_errors)
 
 
-def _load_original_eicr(persistence_id: str, s3_client: BaseClient) -> str:
+def _load_original_eicr(persistence_id: str, s3_client: BaseClient, bucket_name: str) -> str:
     """Load the original eICR from S3.
 
     :param persistence_id: The persistence ID extracted from the S3 object key
     :param s3_client: The S3 client to use for fetching files.
+    :param bucket_name: The S3 bucket name to read from.
     :return: The original eICR content.
     """
     object_key = f"{EICR_INPUT_PREFIX}{persistence_id}"
-    logger.info(f"Retrieving eICR from s3://{S3_BUCKET}/{object_key}")
+    logger.info(f"Retrieving eICR from s3://{bucket_name}/{object_key}")
     original_eicr_content = lambda_handler.get_file_content_from_s3(
-        bucket_name=S3_BUCKET, object_key=object_key, s3_client=s3_client
+        bucket_name=bucket_name, object_key=object_key, s3_client=s3_client
     )
     logger.info(f"Retrieved eICR content for persistence_id {persistence_id}")
     return original_eicr_content
@@ -283,20 +288,25 @@ def _process_schematron_errors(
 
 
 def _save_ttc_outputs(
-    persistence_id: str, ttc_output: dict, ttc_metadata_output: dict, s3_client: BaseClient
+    persistence_id: str,
+    ttc_output: dict,
+    ttc_metadata_output: dict,
+    s3_client: BaseClient,
+    bucket_name: str,
 ) -> None:
     """Save TTC output and metadata output to S3.
 
     :param persistence_id: The persistence ID extracted from the S3 object key
     :param ttc_output: The TTC output dictionary.
     :param ttc_metadata_output: The TTC metadata output dictionary.
     :param s3_client: The S3 client to use for uploading files.
+    :param bucket_name: The S3 bucket name to write to.
     """
     # Save the TTC output to S3 for the Augmentation Lambda to consume
     logger.info(f"Saving TTC output to S3 for persistence_id {persistence_id}")
     lambda_handler.put_file(
         file_obj=io.BytesIO(json.dumps(ttc_output, default=str).encode("utf-8")),
-        bucket_name=S3_BUCKET,
+        bucket_name=bucket_name,
         object_key=f"{TTC_OUTPUT_PREFIX}{persistence_id}",
         s3_client=s3_client,
     )
@@ -305,7 +315,7 @@ def _save_ttc_outputs(
     logger.info(f"Saving TTC metadata output to S3 for persistence_id {persistence_id}")
     lambda_handler.put_file(
         file_obj=io.BytesIO(json.dumps(ttc_metadata_output, default=str).encode("utf-8")),
-        bucket_name=S3_BUCKET,
+        bucket_name=bucket_name,
         object_key=f"{TTC_METADATA_PREFIX}{persistence_id}",
         s3_client=s3_client,
     )
@@ -315,6 +325,7 @@ def _process_record_pipeline(
     persistence_id: str,
     s3_client: BaseClient,
     opensearch_client: OpenSearch,
+    bucket_name: str,
 ) -> dict:
     """The main pipeline for processing each record.
 
@@ -333,11 +344,12 @@ def _process_record_pipeline(
     :param persistence_id: The persistence ID extracted from the S3 object key
     :param s3_client: The S3 client to use for S3 operations.
     :param opensearch_client: The OpenSearch client.
+    :param bucket_name: The S3 bucket name extracted from the event, or the default.
     """
     ttc_output, ttc_metadata_output = _initialize_ttc_outputs(persistence_id)
 
     logger.info("Starting TTC processing")
-    schematron_data_fields = _load_schematron_data_fields(persistence_id, s3_client)
+    schematron_data_fields = _load_schematron_data_fields(persistence_id, s3_client, bucket_name)
 
     if not schematron_data_fields:
         logger.warning(
@@ -348,13 +360,13 @@ def _process_record_pipeline(
         logger.info(f"Saving TTC metadata output to S3 for persistence_id {persistence_id}")
         lambda_handler.put_file(
             file_obj=io.BytesIO(json.dumps(ttc_metadata_output, default=str).encode("utf-8")),
-            bucket_name=S3_BUCKET,
+            bucket_name=bucket_name,
             object_key=f"{TTC_METADATA_PREFIX}{persistence_id}",
             s3_client=s3_client,
         )
         return ttc_output
 
-    original_eicr_content = _load_original_eicr(persistence_id, s3_client)
+    original_eicr_content = _load_original_eicr(persistence_id, s3_client, bucket_name)
     _populate_eicr_metadata(original_eicr_content, ttc_output, ttc_metadata_output)
     _process_schematron_errors(
         original_eicr_content,
@@ -363,6 +375,6 @@ def _process_record_pipeline(
         ttc_output,
         ttc_metadata_output,
     )
-    _save_ttc_outputs(persistence_id, ttc_output, ttc_metadata_output, s3_client)
+    _save_ttc_outputs(persistence_id, ttc_output, ttc_metadata_output, s3_client, bucket_name)
 
     return {"statusCode": 200, "message": "TTC processed successfully!"}

packages/text-to-code-lambda/tests/conftest.py

@@ -13,10 +13,10 @@
 
 S3_BUCKET = "dibbs-text-to-code"
 EICR_INPUT_PREFIX = "eCRMessageV2/"
-SCHEMATRON_ERROR_PREFIX = "schematronErrors/"
-TTC_INPUT_PREFIX = "TextToCodeValidateSubmissionV2/"
-TTC_OUTPUT_PREFIX = "TTCOutput/"
-TTC_METADATA_PREFIX = "TTCMetadata/"
+SCHEMATRON_ERROR_PREFIX = "ValidationResponseV2/"
+TTC_INPUT_PREFIX = "TextToCodeSubmissionV2/"
+TTC_OUTPUT_PREFIX = "TTCAugmentationMetadataV2/"
+TTC_METADATA_PREFIX = "TTCMetadataV2/"
 AWS_REGION = "us-east-1"
 AWS_ACCESS_KEY_ID = "test_access_key_id"
 AWS_SECRET_ACCESS_KEY = "test_secret_access_key"  # noqa: S105