BFD-4576: Make IDR->BFD validation script run automatically on a schedule (#3229)

Author: malessi

.github/workflows/build-container-images.yml

@@ -277,6 +277,8 @@ jobs:
       - name: Generate build-contexts
         if: ${{ !inputs.skipBasePull }}
         id: gen-build-contexts
+        env:
+          NAMED_CONTEXTS: ${{ toJSON(matrix.image.namedContexts) || '{}' }}
         run: |
           build_contexts=()
 
@@ -287,6 +289,15 @@ jobs:
             build_contexts+=("${image}:latest=docker-image://localhost:5000/${image}:${BASE_IMAGES_VERSION}")
           done
 
+          readarray -t named_contexts < <(echo "$NAMED_CONTEXTS" | jq -rc '. // {} | to_entries[]')
+          for named_context in "${named_contexts[@]}"
+          do
+            context_name="$(jq -rc '.key' <<<"$named_context")"
+            context_path="$(jq -rc '.value' <<<"$named_context")"
+
+            build_contexts+=("${context_name}=${context_path}")
+          done
+
           build_contexts_json="$(jq -c -n '$ARGS.positional' --args "${build_contexts[@]}")"
 
           echo "build-contexts=$build_contexts_json" >> "$GITHUB_OUTPUT"

.github/workflows/build-release.yml

@@ -452,7 +452,8 @@ jobs:
         bfd-platform-pipeline-ccw-runner,
         bfd-platform-consume-idr-events,
         bfd-platform-run-idr-pipeline,
-        bfd-platform-codebuild-runner
+        bfd-platform-codebuild-runner,
+        bfd-platform-idr-bfd-validator
       baseImagesVersion: ${{ needs.compute-version-strings.outputs.bfd_release }}
       cleanupImageArtifacts: false # We'll cleanup at the end of build-release, so don't do anything
       tagLatest: ${{ !contains(needs.compute-version-strings.outputs.bfd_release, '-') }}

.github/workflows/build_container_images_matrix.json

@@ -82,5 +82,14 @@
     "dockerfile": "ops/images/bfd-platform-codebuild-runner/Dockerfile",
     "contextDir": "ops/images/bfd-platform-codebuild-runner",
     "platform": "linux/arm64"
+  },
+  {
+    "name": "bfd-platform-idr-bfd-validator",
+    "dockerfile": "apps/utils/idr-bfd-validator/Dockerfile",
+    "contextDir": "apps/utils/idr-bfd-validator",
+    "namedContexts": {
+      "idr-pipeline": "apps/bfd-pipeline-idr"
+    },
+    "platform": "linux/arm64"
   }
 ]

.github/workflows/v3-release-deploy.yml

@@ -385,6 +385,7 @@ jobs:
         database,
         migrator-ng,
         idr-pipeline,
+        idr-bfd-validator,
         idr-pipeline-metrics,
         idr-pipeline-alarms,
         server-ng,

apps/bfd-pipeline-idr/.dockerignore

@@ -0,0 +1 @@
+Dockerfile.dockerignore

apps/bfd-pipeline-idr/Dockerfile

@@ -62,4 +62,4 @@ ENV PATH="/app/.venv/bin:$PATH"
 ENV PYTHONDONTWRITEBYTECODE=1
 
 # Run the pipeline in IDR mode by default
-CMD ["uv", "run", "--no-sync", "pipeline.py"]
+CMD ["uv", "run", "--no-sync", "idr-pipeline"]

apps/bfd-pipeline-idr/Dockerfile.dockerignore

@@ -2,6 +2,6 @@
 !pyproject.toml
 !uv.lock
 !.python-version
-!model/*.py
-!matching/*.py
-!*.py
+!src/idr_pipeline/model/*.py
+!src/idr_pipeline/matching/*.py
+!src/idr_pipeline/*.py

apps/bfd-pipeline-idr/README.md

@@ -90,7 +90,7 @@ source ./load-credentials.sh
 Run the app (optionally specify a minimum transaction date)
 
 ```sh
-PIPELINE_MIN_TRANSACTION_DATE=2024-01-01 uv run pipeline.py
+PIPELINE_MIN_TRANSACTION_DATE=2024-01-01 uv run idr-pipeline
 ```
 
 ## Adding data to the model

apps/bfd-pipeline-idr/load-synthetic-env.sh

@@ -3,14 +3,12 @@
 set -e
 
 read -p "Are you sure you want to overwrite the data in ${BFD_ENV}? [yn] " -n 1 -r
-echo    # (optional) move to a new line
-if ! [[ $REPLY =~ ^[Yy]$ ]]
-then
-    echo 'exiting'
-    exit 0
+echo # (optional) move to a new line
+if ! [[ $REPLY =~ ^[Yy]$ ]]; then
+  echo 'exiting'
+  exit 0
 fi
 
-
 DB_CLUSTER="bfd-${BFD_ENV}-aurora-cluster"
 readonly DB_CLUSTER
 BFD_DB_USERNAME="$(aws ssm get-parameter --name "/bfd/${BFD_ENV}/idr-pipeline/sensitive/db/username" --with-decryption --query "Parameter.Value" --output text)"
@@ -43,7 +41,7 @@ export IDR_SCHEMA
 
 args=('--load-type' 'initial' '--source' 'snowflake' '--load-mode' 'synthetic')
 if [[ -n "$1" ]]; then
-    args+=('--seed-from' "$1")
+  args+=('--seed-from' "$1")
 fi
 
-IDR_ENABLE_DATE_PARTITIONS=0 IDR_ENABLE_PRIOR_AUTH=1 uv run pipeline.py "${args[@]}"
+IDR_ENABLE_DATE_PARTITIONS=0 IDR_ENABLE_PRIOR_AUTH=1 uv run idr-pipeline "${args[@]}"

apps/bfd-pipeline-idr/pyproject.toml

@@ -1,8 +1,7 @@
 [project]
-name = "bfd-pipeline-idr"
+name = "idr-pipeline"
 version = "0.1.0"
-description = "Add your description here"
-readme = "README.md"
+description = "ETL Pipeline to load data from the IDR into BFD's database"
 requires-python = ">=3.14.3"
 dependencies = [
     "cryptography>=46.0.5",
@@ -16,7 +15,6 @@ dependencies = [
     "click>=8.3.3",
     "anyio",
     "loguru",
-    "pydantic-partial" # TODO: remove when idr_bfd_compare is moved
 ]
 
 [dependency-groups]
@@ -28,9 +26,19 @@ dev = [
     "pyright>=1.1.407",
 ]
 
+[project.scripts]
+idr-pipeline = "idr_pipeline:main"
+
+[build-system]
+requires = ["uv_build<=0.12"]
+build-backend = "uv_build"
+
 [tool.uv]
 exclude-newer = "7 days"
 
+[tool.uv.build-backend]
+module-name = "idr_pipeline"
+
 [tool.uv.sources]
 loguru = { git = "https://github.com/Delgan/loguru", rev = "2a17be730e111cf874d0bf967c4c5da15b7e4670" }