chore: rename pipeline dir (#1223)

* rename pipeline dir

* add loading pipeline

* fix a test

* Finish off test

Author: bpblanken

.cloudbuild/docker.cloudbuild.yaml

@@ -3,7 +3,7 @@ steps:
   args:
   - --destination=gcr.io/seqr-project/seqr-pipeline-runner:${COMMIT_SHA}
   - --destination=gcr.io/seqr-project/seqr-pipeline-runner:${_CUSTOM_BRANCH_TAG}
-  - --dockerfile=v03_pipeline/deploy/Dockerfile
+  - --dockerfile=loading_pipeline/deploy/Dockerfile
   - --cache=true
   - --cache-ttl=168h
   - --build-arg=PIPELINE_RUNNER_APP_VERSION=${COMMIT_SHA}

.cloudbuild/vep-docker.cloudbuild.yaml

@@ -1,6 +1,6 @@
 # Run locally with:
 #
-# gcloud builds submit --quiet --substitutions='_REFERENCE_GENOME=GRCh38' --config .cloudbuild/vep-docker.cloudbuild.yaml v03_pipeline/deploy
+# gcloud builds submit --quiet --substitutions='_REFERENCE_GENOME=GRCh38' --config .cloudbuild/vep-docker.cloudbuild.yaml loading_pipeline/deploy
 steps:
 - name: 'gcr.io/kaniko-project/executor:v1.3.0'
   args:

.github/workflows/dev-release.yml

@@ -44,9 +44,9 @@ jobs:
       - name: Copy files to release directory
         run: |-
           gcloud storage rm -r gs://seqr-pipeline-runner-builds/dev/latest/ || echo 'No latest release'
-          gcloud storage cp v03_pipeline/bin/* gs://seqr-pipeline-runner-builds/dev/latest/bin/
+          gcloud storage cp loading_pipeline/bin/* gs://seqr-pipeline-runner-builds/dev/latest/bin/
           gcloud storage cp dist/*.whl gs://seqr-pipeline-runner-builds/dev/latest/pyscripts.zip
-          gcloud storage cp v03_pipeline/bin/* gs://seqr-pipeline-runner-builds/dev/${{ github.event.workflow_run.head_sha }}/bin/
+          gcloud storage cp loading_pipeline/bin/* gs://seqr-pipeline-runner-builds/dev/${{ github.event.workflow_run.head_sha }}/bin/
           gcloud storage cp dist/*.whl gs://seqr-pipeline-runner-builds/dev/${{ github.event.workflow_run.head_sha }}/pyscripts.zip
   docker:
     runs-on: ubuntu-latest

.github/workflows/prod-release.yml

@@ -46,11 +46,11 @@ jobs:
         shell: bash
         run: |-
           gcloud storage rm -r gs://seqr-pipeline-runner-builds/prod/latest/ || echo 'No latest release'
-          gcloud storage cp v03_pipeline/bin/* gs://seqr-pipeline-runner-builds/prod/latest/bin/
+          gcloud storage cp loading_pipeline/bin/* gs://seqr-pipeline-runner-builds/prod/latest/bin/
           gcloud storage cp dist/*.whl gs://seqr-pipeline-runner-builds/prod/latest/pyscripts.zip
-          gcloud storage cp v03_pipeline/bin/* gs://seqr-pipeline-runner-builds/prod/${{ github.event.workflow_run.head_sha }}/bin/
+          gcloud storage cp loading_pipeline/bin/* gs://seqr-pipeline-runner-builds/prod/${{ github.event.workflow_run.head_sha }}/bin/
           gcloud storage cp dist/*.whl gs://seqr-pipeline-runner-builds/prod/${{ github.event.workflow_run.head_sha }}/pyscripts.zip
-          gcloud storage cp -r v03_pipeline/var/vep/* gs://seqr-reference-data/vep/
+          gcloud storage cp -r loading_pipeline/var/vep/* gs://seqr-reference-data/vep/
 
   docker:
     runs-on: ubuntu-latest

.github/workflows/unit-tests.yml

@@ -56,18 +56,18 @@ jobs:
       - name: Sync dependencies (prod + dataproc)
         run: uv sync --locked 
       - name: Check Ruff Format
-        run: uv run ruff format --check v03_pipeline --diff
+        run: uv run ruff format --check loading_pipeline --diff
       - name: Check Ruff
         run: uv run ruff check --output-format github
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master
         with:
           severity: error
-          scandir: './v03_pipeline/bin'
+          scandir: './loading_pipeline/bin'
       - name: Unit Tests
         run: |
           export CLICKHOUSE_OPTIMIZE_TABLE_WAIT_S=1
           export PYSPARK_SUBMIT_ARGS='--driver-memory 8G pyspark-shell'
           export CLICKHOUSE_DATABASE=test
-          uv run nosetests --with-coverage --cover-package v03_pipeline v03_pipeline
+          uv run nosetests --with-coverage --cover-package loading_pipeline loading_pipeline
           uv run coverage report --omit '*test*' --fail-under=85

.gitignore

@@ -8,7 +8,7 @@ download_and_create_reference_datasets/*.txt
 download_and_create_reference_datasets/*.gz
 *.ht
 *.mt
-!/v03_pipeline/var/test/**/*
+!/loading_pipeline/var/test/**/*
 
 # Standard python gitignore below
 ######################################################################

README.md

@@ -9,21 +9,21 @@
 
 ## 📁 Repository Structure
 
-### `v03_pipeline/api/`
-Contains the interface layer to the _seqr_ application. 
+### `loading_pipeline/api/`
+Contains the interface layer to the _seqr_ application.
 - `api/model.py` defines pydantic models for the REST interface.
-- `api/app.py` specifies an `aiohttp` webserver that handles load data requests. 
+- `api/app.py` specifies an `aiohttp` webserver that handles load data requests.
 
-### `v03_pipeline/bin/`
+### `loading_pipeline/bin/`
 Scripts or command-line utilities used for setup or task execution.
 - `bin/pipeline_worker.py` — manages asynchronous jobs requested by _seqr_.
 
-### `v03_pipeline/deploy/`
+### `loading_pipeline/deploy/`
 Dockerfiles for the loading pipeline itself & any annotation utilities.
 Kubernetes manifests are managed separately in [seqr-helm](https://github.com/broadinstitute/seqr-helm/tree/main/charts/pipeline-runner)
 
-### `v03_pipeline/lib/`
-Core logic and shared libraries.  
+### `loading_pipeline/lib/`
+Core logic and shared libraries.
 - `annotations` defines hail logic to re-format and standardize fields.
 - `methods` wraps hail-defined genomics methods for QC.
 - `misc` contains single modules with defined utilities.
@@ -37,10 +37,10 @@ the pipeline is defined, effectively, in reverse.
 - `test` holds a few utilities used by the tests, which are dispersed throughout the rest of the repository.
 - `paths.py` defines paths for all intermediate and output files of the pipeline.
 
-### `v03_pipeline/ops/`
+### `loading_pipeline/ops/`
 Manual operations scripts.
 
-### `v03_pipeline/var/`
+### `loading_pipeline/var/`
 Static configuration and test files.
 
 ---
@@ -58,14 +58,14 @@ RUN uv sync --group dev --locked
 ### [Install](https://clickhouse.com/docs/getting-started/quick-start/oss) & start ClickHouse with provided test configuration:
 ```bash
 curl https://clickhouse.com/ | sh
-./clickhouse server --config-file=./seqr-loading-pipelines/v03_pipeline/var/clickhouse_config/test-clickhouse.xml
+./clickhouse server --config-file=./seqr-loading-pipelines/loading_pipeline/var/clickhouse_config/test-clickhouse.xml
 ```
 
 ### [Run the Tests](https://github.com/broadinstitute/seqr-loading-pipelines/blob/main/.github/workflows/unit-tests.yml#L66-L73)
 
 ### Run an Individual Test
 ```bash
-uv run nosetests v03_pipeline/lib/misc/math_test.py
+uv run nosetests loading_pipeline/lib/misc/math_test.py
 ```
 
 ### Formatting and Linting

v03_pipeline/__init__.py loading_pipeline/__init__.pyv03_pipeline/__init__.py renamed to loading_pipeline/__init__.py

@@ -1,7 +1,7 @@
 from aiohttp import web
 
-from v03_pipeline.api.app import init_web_app
-from v03_pipeline.lib.logger import get_logger
+from loading_pipeline.api.app import init_web_app
+from loading_pipeline.lib.logger import get_logger
 
 
 def run():