Replace custom build system with Poetry

.dockerignore

@@ -1,3 +1,4 @@
+# Tell Docker to _exclude_ these directories from the build context.
 .env*
 !.env.example
 !.env.test
@@ -7,3 +8,6 @@ deployment
 .pytest_cache
 build
 dist
+/.venv
+/.git
+/demo/metadata_migration/notebooks

.github/workflows/build-and-release-to-spin.yml

@@ -11,7 +11,7 @@ on:
       - 'Makefile'
       - '**.Dockerfile'
       - '**.py'
-      - 'requirements/main.txt'
+      - 'poetry.lock'
 
 env:
   # We don't want to do certain steps if this is running in a fork

.github/workflows/mkdocs.yml

@@ -18,6 +18,8 @@ jobs:
         with:
           python-version: 3.x
       - run: pip install git+https://${GH_TOKEN}@github.com/squidfunk/mkdocs-material-insiders.git
+      # TODO: Note that the versions of these packages that `pip` installs may differ
+      #       from the versions listed in `poetry.lock`. Was that intentional?
       - run: pip install mkdocs-mermaid2-plugin
       - run: pip install mkdocs-jupyter
       - run: mkdocs gh-deploy --force

.github/workflows/python-app.yml

@@ -11,14 +11,14 @@ on:
       - 'Makefile'
       - '**.Dockerfile'
       - '**.py'
-      - 'requirements/main.txt'      
+      - 'poetry.lock'
   pull_request:
     paths:
       - '.github/workflows/python-app.yml'
       - 'Makefile'
       - '**.Dockerfile'
       - '**.py'
-      - 'requirements/main.txt'
+      - 'poetry.lock'
   # Allow developers to trigger this workflow manually via the "Actions" page on GitHub.
   # Reference: https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch: { }

Makefile

@@ -1,102 +1,100 @@
+# Use Poetry to install Python dependencies.
 init:
-	pip install -r requirements/main.txt
-	pip install -r requirements/dev.txt
-	pip install --editable .[dev]
-
-# Updates Python dependencies based upon the contents of the `requirements/main.in` and `requirements/dev.in` files.
-#
-# To omit the `--upgrade` option (included by default) when running `pip-compile`:
-# ```
-# $ make update-deps UPDATE_DEPS_MAIN_UPGRADE_OPT='' UPDATE_DEPS_DEV_UPGRADE_OPT=''
-# ```
-UPDATE_DEPS_MAIN_UPGRADE_OPT ?= --upgrade
-UPDATE_DEPS_DEV_UPGRADE_OPT ?= --upgrade
-update-deps:
-    # --allow-unsafe pins packages considered unsafe: distribute, pip, setuptools.
-	pip install --upgrade pip-tools pip setuptools
-	pip-compile $(UPDATE_DEPS_MAIN_UPGRADE_OPT) --build-isolation \
-		--allow-unsafe --resolver=backtracking --strip-extras \
-		--output-file requirements/main.txt \
-		requirements/main.in
-	pip-compile --allow-unsafe $(UPDATE_DEPS_DEV_UPGRADE_OPT) --build-isolation \
-		--allow-unsafe --resolver=backtracking --strip-extras \
-		--output-file requirements/dev.txt \
-		requirements/dev.in
-
-update: update-deps init
+	poetry install
 
+# TODO: Document this target.
 up-dev:
 	docker compose up --build --force-recreate --detach --remove-orphans
 
+# TODO: Document this target.
 dev-reset-db:
 	docker compose \
 		exec mongo /bin/bash -c "./app_tests/mongorestore-nmdc-testdb.sh"
 
+# TODO: Document this target.
 up-test:
 	docker compose --file docker-compose.test.yml \
 		up --build --force-recreate --detach --remove-orphans
 
+# TODO: Document this target.
 test-build:
 	docker compose --file docker-compose.test.yml build test
 
+# TODO: Document this target.
 test-dbinit:
 	docker compose --file docker-compose.test.yml \
 		exec mongo /bin/bash -c "/mongorestore-nmdc-testdb.sh"
 
+# TODO: Document this target.
 test-run:
 	docker compose --file docker-compose.test.yml run test
 
+# TODO: Document this target.
 test: test-build test-run
 
+# TODO: Document this target.
 black:
 	black nmdc_runtime
 
+# TODO: Document this target.
 lint:
 	# Python syntax errors or undefined names
 	flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-ignore=F722
 	# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
 	flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 \
 		--statistics --extend-exclude="./build/" --extend-ignore=F722
 
-PIP_PINNED_FLAKE8 := $(shell grep 'flake8==' requirements/dev.txt)
-PIP_PINNED_BLACK := $(shell grep 'black==' requirements/dev.txt)
+# Define variables containing Python package version numbers read from the Poetry lock file.
+POETRY_LOCKED_FLAKE8_VERSION := $(shell grep -A 1 'name = "flake8"' poetry.lock | sed -ne 's/^version = "\(.*\)"/\1/p')
+POETRY_LOCKED_BLACK_VERSION  := $(shell grep -A 1 'name = "black"'  poetry.lock | sed -ne 's/^version = "\(.*\)"/\1/p')
 
+# Install lint and black via pip, without installing other dependencies.
 init-lint-and-black:
-	pip install $(PIP_PINNED_FLAKE8)
-	pip install $(PIP_PINNED_BLACK)
+	pip install flake8==$(POETRY_LOCKED_FLAKE8_VERSION)
+	pip install black==$(POETRY_LOCKED_BLACK_VERSION)
 
+# TODO: Document this target.
 down-dev:
 	docker compose down
 
+# TODO: Document this target.
 down-test:
 	docker compose --file docker-compose.test.yml down --volumes
 
+# TODO: Document this target.
 follow-fastapi:
 	docker compose logs fastapi -f
 
+# TODO: Document this target.
 fastapi-deploy-spin:
 	rancher kubectl rollout restart deployment/runtime-fastapi --namespace=nmdc-dev
 
+# TODO: Document this target.
 dagster-deploy-spin:
 	rancher kubectl rollout restart deployment/dagit --namespace=nmdc-dev
 	rancher kubectl rollout restart deployment/dagster-daemon --namespace=nmdc-dev
 
+# TODO: Document this target.
 publish:
 	invoke publish
 
+# TODO: Document this target.
 docs-dev:
 	mkdocs serve -a localhost:8080
 
+# TODO: Document this target.
 nersc-sshproxy:
 	bash ./nersc-sshproxy.sh -u ${NERSC_USERNAME} # https://docs.nersc.gov/connect/mfa/#sshproxy
 
+# TODO: Document this target.
 nersc-mongo-tunnels:
 	ssh -L27072:mongo-loadbalancer.nmdc.production.svc.spin.nersc.org:27017 \
 		-L28082:mongo-loadbalancer.nmdc-dev.development.svc.spin.nersc.org:27017 \
 		-L27092:mongo-loadbalancer.nmdc-dev.production.svc.spin.nersc.org:27017 \
 		-o ServerAliveInterval=60 \
 		${NERSC_USERNAME}@dtn02.nersc.gov
 
+# TODO: Document this target.
 mongorestore-nmdc-db:
 	mkdir -p /tmp/remote-mongodump/nmdc
 	# SSH into the remote server, stream the dump directory as a gzipped tar archive, and extract it locally.
@@ -106,8 +104,9 @@ mongorestore-nmdc-db:
 	mongorestore -v -h localhost:27018 -u admin -p root --authenticationDatabase=admin \
 		--drop --nsInclude='nmdc.*' --dir /tmp/remote-mongodump
 
+# TODO: Document this target.
 quick-blade:
 	python -c "from nmdc_runtime.api.core.idgen import generate_id; print(f'nmdc:nt-11-{generate_id(length=8, split_every=0)}')"
 
-.PHONY: init update-deps update up-dev down-dev follow-fastapi \
+.PHONY: init up-dev down-dev follow-fastapi \
 	publish docs

docker-compose.yml

@@ -65,7 +65,7 @@ services:
     container_name: fastapi
     ports:
       - "8000:8000"
-    command: ["uvicorn", "nmdc_runtime.api.main:app", "--reload", "--host", "0.0.0.0", "--port", "8000"]
+    command: ["poetry", "run", "python", "-m", "uvicorn", "nmdc_runtime.api.main:app", "--reload", "--host", "0.0.0.0", "--port", "8000"]
     env_file:
       - .env
     depends_on:

docs/admin.md

@@ -96,6 +96,6 @@ If you need to delete objects copy it to `nmdc_deleted` database under the corre
 
 ## nmdc-schema update
 
-1. Set the desired version of `nmdc-schema` in `requirements/main.in`.
-2. `make update-deps`.
+1. Set the desired version of `nmdc-schema` in `pyproject.toml`.
+2. Run `$ poetry install`.
 3. commit and push, start PR and seek approval + merge to `main`, which will trigger GH actions to deploy.

nmdc_runtime/dagster.Dockerfile

@@ -16,15 +16,18 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
   apt-get -y clean && \
   rm -rf /var/lib/apt/lists/*
 
+# Install Poetry.
+RUN pip install poetry
 
-# Install requirements
 WORKDIR /opt/dagster/lib
-COPY requirements/main.txt requirements.txt
-RUN pip install --no-cache-dir -r requirements.txt
+
+# Use Poetry to install production Python dependencies.
+COPY ./pyproject.toml /opt/dagster/lib/pyproject.toml
+COPY ./poetry.lock /opt/dagster/lib/poetry.lock
+RUN poetry install --without dev
 
 # Add repository code
 COPY . .
-RUN pip install --no-cache-dir --editable .
 
 # Set $DAGSTER_HOME and copy dagster instance and workspace YAML there
 ENV DAGSTER_HOME=/opt/dagster/dagster_home/

nmdc_runtime/fastapi.Dockerfile

@@ -16,14 +16,17 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
   apt-get -y clean && \
   rm -rf /var/lib/apt/lists/*
 
-WORKDIR /code
+# Install Poetry.
+RUN pip install poetry
 
-COPY ./requirements/main.txt /code/requirements.txt
+WORKDIR /code
 
-RUN pip install --no-cache-dir -r /code/requirements.txt
+# Use Poetry to install production Python dependencies.
+COPY ./pyproject.toml /code/pyproject.toml
+COPY ./poetry.lock /code/poetry.lock
+RUN poetry install --without dev
 
 # Add repository code
 COPY . /code
-RUN pip install --no-cache-dir --editable .
 
-CMD ["uvicorn", "nmdc_runtime.api.main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["python", "-m", "poetry", "run", "uvicorn", "nmdc_runtime.api.main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "8000"]

nmdc_runtime/site/entrypoint-daemon.sh

@@ -23,4 +23,8 @@ file_env() {
 file_env "MONGO_PASSWORD"
 file_env "DAGSTER_POSTGRES_PASSWORD"
 
-exec dagster-daemon run
+# Note: We specify a directory to Poetry so it knows where to find our `pyproject.toml` file.
+#       Reference: https://python-poetry.org/docs/cli/#global-options
+#
+exec poetry run --directory /opt/dagster/lib \
+  dagster-daemon run

nmdc_runtime/site/entrypoint-dagit.sh

@@ -23,4 +23,10 @@ file_env() {
 file_env "MONGO_PASSWORD"
 file_env "DAGSTER_POSTGRES_PASSWORD"
 
-exec dagit -h 0.0.0.0 -p 3000 -w workspace.yaml
+# Note: We specify a directory to Poetry so it knows where to find our `pyproject.toml` file.
+#       Reference: https://python-poetry.org/docs/cli/#global-options
+#
+# TODO: The `dagit` CLI command is deprecated. Use `dagster-webserver` instead.
+#
+exec poetry run --directory /opt/dagster/lib \
+  dagit -h 0.0.0.0 -p 3000 -w workspace.yaml

nmdc_runtime/site/workspace.yaml

@@ -1,22 +1,31 @@
 load_from:
   - python_package:
+      # Note: The directory to which the following "working_directory"
+      #       is _relative_, is `/opt/dagster/dagster_home`.
+      working_directory: ../lib
       package_name: nmdc_runtime.site.repository
       attribute: repo
   - python_package:
+      working_directory: ../lib
       package_name: nmdc_runtime.site.repository
       attribute: translation
   - python_package:
+      working_directory: ../lib
       package_name: nmdc_runtime.site.repository
       attribute: test_translation
   - python_package:
+      working_directory: ../lib
       package_name: nmdc_runtime.site.repository
       attribute: biosample_submission_ingest
   - python_package:
+      working_directory: ../lib
       package_name: nmdc_runtime.site.repository
       attribute: biosample_export
 #  - python_package:
+#      working_directory: ../lib
 #      package_name: nmdc_runtime.site.repository
 #      attribute: validation
 #  - python_package:
+#      working_directory: ../lib
 #      package_name: nmdc_runtime.site.repository
 #      attribute: test_validation

nmdc_runtime/test.Dockerfile

@@ -16,18 +16,20 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
   apt-get -y clean && \
   rm -rf /var/lib/apt/lists/*
 
-# Install requirements, incl. for tests
+# Install Poetry.
+RUN pip install poetry
+
 WORKDIR /code
-COPY ./requirements/dev.txt /code/requirements.dev.txt
-RUN pip install --no-cache-dir -r /code/requirements.dev.txt
-COPY ./requirements/main.txt /code/requirements.txt
-RUN pip install --no-cache-dir -r /code/requirements.txt
 
+# Use Poetry to install Python dependencies, including development ones.
+COPY ./pyproject.toml /code/pyproject.toml
+COPY ./poetry.lock /code/poetry.lock
+RUN poetry install --with dev
 
 # Add repository code
 COPY . /code
-RUN pip install --no-cache-dir --editable .
 
+# TODO: Add a claim to the sentence below. Example: "Ensure wait-for-it... is executable"
 ## Ensure wait-for-it
 RUN chmod +x wait-for-it.sh
 
@@ -36,8 +38,8 @@ ENV PYTHONFAULTHANDLER=1
 
 
 # uncomment line below to run all tests
-# ENTRYPOINT [ "./wait-for-it.sh", "fastapi:8000" , "--strict" , "--timeout=300" , "--" , "pytest"]
+# ENTRYPOINT [ "./wait-for-it.sh", "fastapi:8000" , "--strict" , "--timeout=300" , "--" , "poetry", "run", "pytest"]
 
 # uncomment line below to stop after first test failure:
 # https://docs.pytest.org/en/6.2.x/usage.html#stopping-after-the-first-or-n-failures
-ENTRYPOINT [ "./wait-for-it.sh", "fastapi:8000" , "--strict" , "--timeout=300" , "--" , "pytest"]
+ENTRYPOINT [ "./wait-for-it.sh", "fastapi:8000" , "--strict" , "--timeout=300" , "--" , "poetry", "run", "pytest"]