Merge branch 'dev' into add-repoint-api

Author: greglucas

.github/workflows/deploy.yml

@@ -4,33 +4,73 @@ on:
   push:
     branches:
       - dev
-      - prod
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Environment to deploy to"
+        required: true
+        type: choice
+        options:
+          - dev
+          - prod
+        default: prod
+      ref:
+        description: "Optional git ref (commit SHA, branch, or tag) to deploy (for rollback)"
+        required: false
+        default: ""
+
+permissions:
+  id-token: write
+  contents: write  # needed for tagging and release creation
+  actions: read
 
 jobs:
   cdk-deploy:
     runs-on: ubuntu-latest
-    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
-    permissions:
-      id-token: write
-      contents: read
 
     steps:
-      - name: Set deployment account and role
+      # ----------------------------------------------------
+      # 🧭 Determine environment + AWS role + tagging behavior
+      # ----------------------------------------------------
+      - name: Determine deployment target
         id: set_account
         run: |
-          if [[ "${GITHUB_REF##*/}" == "dev" ]]; then
-            echo "account_name=dev" >> $GITHUB_ENV
-            echo "role_arn=arn:aws:iam::449431850278:role/GitHubDeploy" >> $GITHUB_ENV
-          elif [[ "${GITHUB_REF##*/}" == "prod" ]]; then
-            echo "account_name=prod" >> $GITHUB_ENV
-            echo "role_arn=arn:aws:iam::593025701104:role/GitHubDeploy" >> $GITHUB_ENV
+          if [[ "${GITHUB_EVENT_NAME}" == "push" ]]; then
+            if [[ "${GITHUB_REF##*/}" == "dev" ]]; then
+              echo "account_name=dev" >> $GITHUB_ENV
+              echo "role_arn=arn:aws:iam::449431850278:role/GitHubDeploy" >> $GITHUB_ENV
+              echo "should_tag=false" >> $GITHUB_ENV
+            else
+              echo "Branch not configured for deployment." && exit 1
+            fi
+          elif [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then
+            if [[ "${{ github.event.inputs.environment }}" == "dev" ]]; then
+              echo "account_name=dev" >> $GITHUB_ENV
+              echo "role_arn=arn:aws:iam::449431850278:role/GitHubDeploy" >> $GITHUB_ENV
+              echo "should_tag=false" >> $GITHUB_ENV
+            elif [[ "${{ github.event.inputs.environment }}" == "prod" ]]; then
+              echo "account_name=prod" >> $GITHUB_ENV
+              echo "role_arn=arn:aws:iam::593025701104:role/GitHubDeploy" >> $GITHUB_ENV
+              echo "should_tag=true" >> $GITHUB_ENV
+            else
+              echo "Invalid environment specified." && exit 1
+            fi
           else
-            echo "Branch not configured for deployment." && exit 1
+            echo "Unsupported trigger type" && exit 1
           fi
+
+      # ----------------------------------------------------
+      # 🔄 Checkout code (use ref input if provided for rollback)
+      # ----------------------------------------------------
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.inputs.ref || github.ref }}
+
       - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
+
       - uses: Gr1N/setup-poetry@v8
         with:
           poetry-version: "1.8.0"
@@ -50,16 +90,33 @@ jobs:
           role-to-assume: ${{ env.role_arn }}
           aws-region: us-west-2
 
+      # ----------------------------------------------------
+      # 🧱 CDK Synth + Deploy
+      # ----------------------------------------------------
       - name: Synth
         env:
           ACCOUNT_NAME: ${{ env.account_name }}
         run: |
+          echo "Synthesizing for environment: $ACCOUNT_NAME"
           # poetry run to get the environment we installed everything into
           poetry run cdk synth --context account_name=$ACCOUNT_NAME
 
       - name: Deploy
         env:
           ACCOUNT_NAME: ${{ env.account_name }}
         run: |
+          echo "Deploying to environment: $ACCOUNT_NAME"
           # poetry run to get the environment we installed everything into
           poetry run cdk deploy --all --context account_name=$ACCOUNT_NAME --require-approval never
+
+      # ----------------------------------------------------
+      # 🏷️ Tag Release (prod only)
+      # ----------------------------------------------------
+      - name: Create tag
+        if: env.should_tag == 'true'
+        run: |
+          TAG_NAME="prod-$(date -u +'%Y-%m-%d-%H%M%S')"
+          echo "TAG_NAME=$TAG_NAME" >> $GITHUB_ENV
+          git tag -a "$TAG_NAME" -m "Production deployment on $(date -u)"
+          git push origin "$TAG_NAME"
+          echo "Created tag $TAG_NAME"

lambda_layer/database/requirements.txt

@@ -153,9 +153,9 @@ greenlet==3.2.2 ; python_version < "3.14" and (platform_machine == "aarch64" or
 idna==3.10 ; python_version >= "3.10" and python_version < "4" \
     --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
     --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
-imap-data-access==0.36.0 ; python_version >= "3.10" and python_version < "4" \
-    --hash=sha256:e221c55395bc3a1163432110b67ec544566f87e1e332bd3da2c9f833c765136c \
-    --hash=sha256:edf8a57c89759bcf40979b290e5fa368f71b34094d757c203d1086ad57818a79
+imap-data-access==0.37.0 ; python_version >= "3.10" and python_version < "4" \
+    --hash=sha256:7887ad43c4404c6465035af73621237aef7a8d88d506439dd4618aef1db8cf87 \
+    --hash=sha256:9801aba311311815866336636fdad9e37b530498f4c3f21abce283d860a7c643
 psycopg2-binary==2.9.10 ; python_version >= "3.10" and python_version < "4" \
     --hash=sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff \
     --hash=sha256:056470c3dc57904bbf63d6f534988bafc4e970ffd50f6271fc4ee7daad9498a5 \

lambda_layer/spice/requirements.txt

@@ -97,9 +97,9 @@ charset-normalizer==3.4.2 ; python_version >= "3.10" and python_version < "4" \
 idna==3.10 ; python_version >= "3.10" and python_version < "4" \
     --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
     --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
-imap-data-access==0.36.0 ; python_version >= "3.10" and python_version < "4" \
-    --hash=sha256:e221c55395bc3a1163432110b67ec544566f87e1e332bd3da2c9f833c765136c \
-    --hash=sha256:edf8a57c89759bcf40979b290e5fa368f71b34094d757c203d1086ad57818a79
+imap-data-access==0.37.0 ; python_version >= "3.10" and python_version < "4" \
+    --hash=sha256:7887ad43c4404c6465035af73621237aef7a8d88d506439dd4618aef1db8cf87 \
+    --hash=sha256:9801aba311311815866336636fdad9e37b530498f4c3f21abce283d860a7c643
 numpy==2.2.6 ; python_version >= "3.10" and python_version < "4" \
     --hash=sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff \
     --hash=sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47 \

poetry.lock

@@ -32,7 +32,7 @@ python = ">=3.10,<4"
 # WARNING - Please run this command when updateing `imap-data-access` to ensure that
 # the correct version and hash are used:
 #   poetry export -f requirements.txt -o lambda_layer/database/requirements.txt --with layer-database
-imap-data-access = ">=0.36.0"
+imap-data-access = ">=0.37.0"
 
 # Optional dependencies - install with `poetry install -E docs`
 sphinx = {version="^7.1.0", optional=true}

pyproject.toml

@@ -180,43 +180,63 @@ def filter_conditions(table):
             )
         return conditions
 
-    # First check to see if there are any jobs in progress and get the max version
+    # Step 1: query to get the max version from the processing jobs table
     max_version_record = (
         session.query(models.ProcessingJob)
         .filter(*filter_conditions(models.ProcessingJob))
         .order_by(models.ProcessingJob.version.desc())
         .first()
     )
     if max_version_record:
-        max_version = max_version_record.version
-        # If there is a job already in progress, determine whether the current job
-        # is a duplicate of the in-progress job by checking the dependency file hash.
-        # If the hashes are different, then we know the dependencies have changed and
-        # we should bump the version number and continue with processing.
+        max_version_processing = max_version_record.version
+        # Step 2: If there is a job already in progress, determine whether the current
+        # job is a duplicate of the in-progress job by checking the dependency file
+        # hash. If the hashes are different, then we know the dependencies have changed
+        # and we should bump the version number and continue with processing.
         if max_version_record.status == models.Status.INPROGRESS:
             command = max_version_record.container_command
             if dependency_hash(current_dependencies) in command:
                 # Return the current max version and this job will not proceed if
                 # everything else is the same.
-                return max_version
-            logger.info(
-                f"Job with id: {max_version_record.id} is in progress, but the "
-                f"dependencies have changed. Bumping version number."
-            )
+                return max_version_processing
+            else:
+                # Dependencies have changed, so bump the version number.
+                logger.info(
+                    f"Job with id: {max_version_record.id} is in progress, but the "
+                    f"dependencies have changed. Bumping version number."
+                )
+                return f"v{int(max_version_processing[1:]) + 1:03d}"
+
     else:
-        max_version = None
-    # If the descriptor is "all", we should only check the processing job table. The
-    # ScienceFiles table does not have descriptors of "all" since the products
-    # produced will have their own specific descriptors.
-    if descriptor == "all":
-        return f"v{int(max_version[1:]) + 1:03d}" if max_version else "v001"
-    # If no jobs are in progress, check the science files table for the max version.
-    if not max_version:
-        max_version = (
-            session.query(func.max(models.ScienceFiles.version)).filter(
-                *filter_conditions(models.ScienceFiles)
-            )
-        ).scalar()
+        max_version_processing = None
+    # Step 3: If the descriptor is "all", only use the max version from the processing
+    # job table. The ScienceFiles table does not have descriptors of "all" since the
+    # products produced will have their own specific descriptors.
+    if "all" in descriptor:
+        return (
+            f"v{int(max_version_processing[1:]) + 1:03d}"
+            if max_version_processing
+            else "v001"
+        )
+
+    # Step 4: Get the max version from the science files table.
+    max_version_sci = (
+        session.query(func.max(models.ScienceFiles.version)).filter(
+            *filter_conditions(models.ScienceFiles)
+        )
+    ).scalar()
+
+    # Step 5: By default, use the max version from the science files table unless
+    # it is a spacecraft "pointing-attitude" job. If a so, then use the max version
+    # from the processing jobs table. If the job is a spacecraft pointing-attitude job,
+    # it will produce a SPICE kernel and not a science file. There is no way to
+    # determine the filename of the kernel that will be produced, so we rely on the max
+    # version from the processing jobs table.
+    if instrument == "spacecraft" and descriptor == "pointing-attitude":
+        max_version = max_version_processing
+    else:
+        max_version = max_version_sci
+
     # Bump the version number. "V001" will be returned if max_version is None.
     return f"v{int(max_version[1:]) + 1:03d}" if max_version else "v001"
 

sds_data_manager/lambda_code/SDSCode/pipeline_lambdas/batch_starter.py

@@ -142,9 +142,9 @@ charset-normalizer==3.4.2 ; python_version >= "3.10" and python_version < "4" \
 idna==3.10 ; python_version >= "3.10" and python_version < "4" \
     --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
     --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
-imap-data-access==0.36.0 ; python_version >= "3.10" and python_version < "4" \
-    --hash=sha256:e221c55395bc3a1163432110b67ec544566f87e1e332bd3da2c9f833c765136c \
-    --hash=sha256:edf8a57c89759bcf40979b290e5fa368f71b34094d757c203d1086ad57818a79
+imap-data-access==0.37.0 ; python_version >= "3.10" and python_version < "4" \
+    --hash=sha256:7887ad43c4404c6465035af73621237aef7a8d88d506439dd4618aef1db8cf87 \
+    --hash=sha256:9801aba311311815866336636fdad9e37b530498f4c3f21abce283d860a7c643
 imap-processing==1.0.0 ; python_version >= "3.10" and python_version < "4" \
     --hash=sha256:808c614dbd4d1a198972bd224416bcb16a933b12bfe0a5391cac21ea198aca32 \
     --hash=sha256:f71001020a0047f9ceba9a2803384e34f669821779f88619e14c2cd598fa75fc

sds_data_manager/lambda_code/processing/requirements.txt

@@ -2110,3 +2110,73 @@ def test_lambda_skip_processing_due_to_crid_check(session, caplog):
     # Verify the job was skipped
     assert log in caplog.text
     assert mock_submit.call_count == 0
+
+
+# Add tests to verify that the correct version is calculated.
+def test_determine_job_version_science(session):
+    """Tests ``determine_job_version`` for science jobs."""
+    # For science files, the job version should be determined from the science files
+    # table. Although there is a successful job with v003, the latest science file is
+    # v001, so the next version should be v002. It is possible for a job to have
+    # Status = SUCCEEDED, but no files were produced for the job, which is why we check
+    # the science files table for the version.
+    records = [
+        ScienceFiles(
+            file_path="/path/to/imap_lo_l1a_de_20240101_v002.cdf",
+            instrument="lo",
+            data_level="l1a",
+            descriptor="de",
+            start_date=datetime(2024, 1, 1),
+            version="v001",
+            extension="cdf",
+            ingestion_date=datetime.strptime(
+                "2024-01-25 23:35:26+00:00", "%Y-%m-%d %H:%M:%S%z"
+            ),
+        ),
+        ProcessingJob(
+            status=models.Status.SUCCEEDED,
+            instrument="lo",
+            data_level="l1a",
+            descriptor="de",
+            start_date=datetime(2024, 1, 1),
+            version="v003",
+        ),
+    ]
+    session.add_all(records)
+    session.add_all(records)
+    version = determine_job_version(
+        session, "lo", "l1a", "de", datetime(2024, 1, 1), "test_dependency"
+    )
+    # The version should be v002
+    assert version == "v002"
+
+
+def test_determine_job_version_spacecraft(session):
+    """Tests ``determine_job_version`` for spacecraft jobs."""
+    # the function determine_job_version uses the processing job table to determine
+    # the correct version for a spacecraft pointing-attitude job, since there is no way
+    # to determine the filename and therefore the version from the spice table using the
+    # information given. Assert that the version is calculated from the processing
+    # job table.
+    records = [
+        # Add a processing job with version 1
+        ProcessingJob(
+            status=models.Status.SUCCEEDED,
+            instrument="spacecraft",
+            data_level="l1a",
+            descriptor="pointing-attitude",
+            start_date=datetime(2024, 1, 1),
+            version="v002",
+        )
+    ]
+    session.add_all(records)
+    version = determine_job_version(
+        session,
+        "spacecraft",
+        "l1a",
+        "pointing-attitude",
+        datetime(2024, 1, 1),
+        "test_dependency",
+    )
+    # The version should be v003 since there was a successful job with v002
+    assert version == "v003"