Merge pull request #3485 from effigies/uv

chore: Migrate pipenv->uv, Python 3.13, git-annex @ PyPI

Author: nellh

.github/dependabot.yml

@@ -8,11 +8,11 @@ updates:
       actions-infrastructure:
         patterns:
           - "actions/*"
-  - package-ecosystem: "pip"
+  - package-ecosystem: "uv"
     directory: "/services/datalad"
     schedule:
       interval: "monthly"
     groups:
-      pipenv:
+      datalad:
         patterns:
           - "*"

.github/workflows/python.yml

@@ -14,31 +14,30 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v5
+      - name: Set up Python 3.13 (uv)
+        uses: astral-sh/setup-uv@v6
         with:
-          python-version: "3.11"
+          version: "latest"
+          python-version: "3.13"
       - name: Use Node.js 20.x
         uses: actions/setup-node@v4
         with:
           node-version: 20.x
       - name: Setup Deno stable
         uses: denoland/setup-deno@v2
         with:
-          deno-version: v1.x
+          deno-version: v2.x
       - name: Install git-annex
-        run: curl -L http://archive.org/download/git-annex-builds/SHA256E-s53592796--6e0a3c16d50379f89d2064081376a3f7f6efa1a89eb1afc2a2335597e7cf9fac.tar.gz | tar -C /usr/local/bin --strip-components 1 -xvz
-      - name: Install pipenv
-        run: pip install 'pipenv==2023.7.23'
+        run: uv tool install git-annex
       - name: Install dependencies
-        run: pipenv install --dev
+        run: uv sync --locked --dev
       - name: Configure git defaults
         run: |
           git config --global user.name "Git Worker"
           git config --global user.email "[email protected]"
           git config --global receive.denyCurrentBranch updateInstead
       - name: Run tests and collect coverage
-        run: pipenv run pytest -n 4 --cov=./datalad_service --cov-report=xml:coverage.xml ./tests
+        run: uv run pytest -n 4 --cov=./datalad_service --cov-report=xml:coverage.xml ./tests
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
         with:

services/datalad/Dockerfile

@@ -1,32 +1,36 @@
 ARG LERNA_JSON
 FROM docker.io/denoland/deno:bin-2.2.0 AS deno
-FROM docker.io/library/python:3.11-slim as production
+FROM ghcr.io/astral-sh/uv:0.7.11-python3.13-bookworm-slim as production
 
 WORKDIR /srv
 
-COPY hooks /hooks
-COPY Pipfile /srv/Pipfile
-COPY Pipfile.lock /srv/Pipfile.lock
-COPY datalad_service /srv/datalad_service
-COPY get_docker_scale.py /get_docker_scale.py
-COPY ./ssh_config /root/.ssh/config
-COPY tests /srv/tests
-# Install Deno the Dockery-but-hacky-way
-COPY --from=deno /deno /usr/local/bin/deno
-
 RUN apt-get update \
-  && apt-get install -y curl openssh-client \
+  && apt-get install -y git curl openssh-client \
+  && apt-get clean
+
+RUN mkdir -p /root/.ssh \
   && ssh-keyscan github.com >> /root/.ssh/known_hosts \
-  && curl -L https://storage.googleapis.com/openneuro-git-annex/10.20250320/SHA256E-s62052170--58aadf91f6b05a8ed9fb6a351d5e786da9fe1890e202b9b935d18e876caf25a1.tar.gz | tar -C /usr/local/bin --strip-components 1 -xvz \
-  && pip3 install 'pipenv==2020.6.2' \
-  && pipenv install --deploy --system \
-  && chmod 600 /root/.ssh/config \
-  && rm -rf /var/lib/apt/lists/* \
   && git config --global user.name "Git Worker" \
   && git config --global user.email "[email protected]" \
-  && git config --global receive.denyCurrentBranch updateInstead \
+  && git config --global receive.denyCurrentBranch updateInstead
+
+# Install Deno the Dockery-but-hacky-way
+COPY --from=deno /deno /usr/local/bin/deno
+
+RUN uv tool install git-annex==10.20250520b7
+
+COPY . /srv
+
+RUN ln -s /srv/hooks /hooks \
+  && mv /srv/ssh_config /root/.ssh/config \
+  && chmod 600 /root/.ssh/config \
+  && ln -s /srv/get_docker_scale.py /get_docker_scale.py \
   && echo "$LERNA_JSON" > /srv/lerna.json
 
+RUN uv sync --locked
+
+ENV PATH="/srv/.venv/bin:${PATH}"
+
 HEALTHCHECK --interval=5s --timeout=30s --start-period=10s --retries=10 CMD [ "curl", "-f", "http://localhost/heartbeat" ]
 
 ENV LOCPATH=""

services/datalad/Pipfile

@@ -8,10 +8,18 @@ A backend component of OpenNeuro, this service provides HTTP interfaces for crea
 
 # Setup
 
+## uv (recommended)
+
+```bash
+uv sync --locked
+```
+
+## virtualenv
+
 ```bash
 virtualenv --python python3 .venv
 source .venv/bin/activate
-pip install -r requirements.txt
+pip install .
 ```
 
 You will also need [npm](https://www.npmjs.com) or [Yarn](https://yarnpkg.com) to install the [bids-validator](https://github.com/INCF/bids-validator).
@@ -29,11 +37,14 @@ npm install
 # Running
 
 ```bash
-gunicorn --reload "datalad_service.app:create_app('/path-to-repos')"
+source .venv/bin/activate
+export DATALAD_DATASET_PATH=/path/to/repos
+uvicorn --factory "datalad_service.app:create_app"
 ```
 
 # Tests
 
 ```bash
-pytest
+uv sync --dev
+uv run pytest -n auto
 ```

services/datalad/Pipfile.lock

@@ -0,0 +1,42 @@
+[project]
+name = "datalad-service"
+description = "HTTP interfaces for manipulating DataLad datasets"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "aiofiles>=24.1.0",
+    "aioshutil>=1.5",
+    "bidsschematools>=1.0.10",
+    "boto3>=1.38.30",
+    "charset-normalizer>=3.4.2",
+    "falcon>=4.0.2",
+    "pygit2>=1.18.0",
+    "pygithub>=2.6.1",
+    "pyjwt>=2.10.1",
+    "requests>=2.32.3",
+    "sentry-sdk[falcon]>=2.29.1",
+    "uvicorn[standard]>=0.34.3",
+]
+dynamic = ["version"]
+
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "coverage>=7.8.2",
+    "datalad>=1.2.0",
+    "exceptiongroup>=1.3.0",
+    "pytest>=8.4.0",
+    "pytest-asyncio>=1.0.0",
+    "pytest-cov>=6.1.1",
+    "pytest-xdist>=3.7.0",
+]
+
+[tool.hatch.version]
+source = "vcs"
+fallback-version = "0+unknown"
+
+[tool.hatch.version.raw-options]
+root = "../.."