Add baseimage dependencies and enhance Docker images

- Create docker/requirements/baseimage.txt with general utilities:
  miniwdl, udocker, awscli, google-cloud-storage, csvkit, jq,
  parallel, pigz, unzip, zstd
- Move general utilities from core.txt to baseimage.txt
- Add seaborn to core.txt for data visualization
- Update Dockerfile.baseimage to install from baseimage.txt
- Set MINIWDL__SCHEDULER__CONTAINER_BACKEND=udocker env var
- Update Dockerfile.core to install baseimage.txt + core.txt together
- Add Python dependencies to pyproject.toml for pip-based installs
- Update AGENT_CONTEXT.md with learnings (udocker quirks, qsv x86-only)

Note: qsv excluded as it lacks ARM64 support

Co-Authored-By: Claude <noreply@anthropic.com>

Author: dpark01

AGENT_CONTEXT.md

@@ -361,6 +361,16 @@ For tools without ARM64 support (novoalign, mvicuna):
 
 These packages are in bioconda but only have x86_64 builds. Use exact versions (e.g., `novoalign=3.09.04`, `mvicuna=1.0`) since newer versions may not exist.
 
+### udocker Quirks
+
+- udocker refuses to run as root by default. Use `--allow-root` flag for Docker build verification
+- In Dockerfile, use `udocker --allow-root version` to verify installation
+- The `MINIWDL__SCHEDULER__CONTAINER_BACKEND=udocker` environment variable tells miniwdl to use udocker
+
+### qsv is x86-Only
+
+The `qsv` package (fast CSV tool) is only available for linux-64, osx-64, win-64 - no ARM64 build. For multi-arch Docker builds, either skip it or add to an x86-only requirements file.
+
 ### PrexistingUnixCommand is the Only InstallMethod
 
 Since all tools are installed via conda, `PrexistingUnixCommand` is the only `InstallMethod` subclass needed. It just checks if the tool exists in PATH.
@@ -418,7 +428,7 @@ When working on this migration:
 ## Next Steps
 
 1. Read `MONOREPO_IMPLEMENTATION_PLAN.md` for the detailed task list
-2. **Phase 2**: Migrate viral-core with git history preservation using `git filter-repo`
+2. **Phase 3a**: Migrate viral-assemble with git history preservation using `git filter-repo`
 3. Work through phases sequentially
 4. Verify each phase before moving to the next
 
@@ -454,6 +464,15 @@ When working on this migration:
 - Deleted legacy viral-core CI workflow (build.yml) that was accidentally merged with git history
 - Docker build verified with all module imports working
 
+### Baseimage Enhancements (Post-Phase 2)
+- Created `docker/requirements/baseimage.txt` with general utilities
+- Added to baseimage: miniwdl, udocker, awscli, google-cloud-storage, csvkit, jq, parallel, pigz, unzip, zstd
+- Set `MINIWDL__SCHEDULER__CONTAINER_BACKEND=udocker` environment variable
+- Moved general utilities from core.txt to baseimage.txt
+- Added seaborn to core.txt for data visualization
+- Updated pyproject.toml with Python dependencies for pip-based installs
+- Note: qsv is x86-only (no ARM64 build), excluded for multi-arch support
+
 ## Reference Repositories
 
 When implementing Docker patterns, refer to:

docker/Dockerfile.baseimage

@@ -1,10 +1,11 @@
 # viral-ngs baseimage
-# Base image with micromamba and Python for viral-ngs tools
+# Base image with micromamba, Python, and general utilities for viral-ngs tools
 #
-# This provides a minimal foundation with:
+# This provides a foundation with:
 # - micromamba (conda-compatible package manager)
 # - Python 3.12
 # - conda/mamba symlinks for compatibility
+# - General utilities (miniwdl, udocker, cloud CLIs, etc.)
 #
 # Derivative images (core, classify, assemble, phylo) build on this.
 
@@ -56,10 +57,23 @@ ENV VIRAL_NGS_PATH="/opt/viral-ngs/source" \
     PYTHONPATH="/opt/viral-ngs/source" \
     PATH="/opt/conda/bin:/usr/local/bin:$PATH"
 
+# Configure miniwdl to use udocker as container backend
+ENV MINIWDL__SCHEDULER__CONTAINER_BACKEND="udocker"
+
+# Copy requirements and install script
+COPY docker/requirements/baseimage.txt /tmp/requirements/
+COPY docker/install-conda-deps.sh /tmp/
+
+# Install baseimage dependencies (miniwdl, udocker, cloud CLIs, utilities)
+RUN /tmp/install-conda-deps.sh /tmp/requirements/baseimage.txt && \
+    rm -rf /tmp/requirements /tmp/install-conda-deps.sh
+
 WORKDIR /opt/viral-ngs/source
 
 # Verify installation
 RUN python --version && \
-    micromamba --version
+    micromamba --version && \
+    miniwdl --version && \
+    udocker --allow-root version
 
 CMD ["/bin/bash"]

docker/Dockerfile.core

@@ -18,12 +18,13 @@ LABEL org.opencontainers.image.source="https://github.com/broadinstitute/viral-n
 ARG MAMBA_DOCKERFILE_ACTIVATE=1
 
 # Copy requirements and dependency installation script
-COPY docker/requirements/core.txt docker/requirements/core-x86.txt /tmp/requirements/
+COPY docker/requirements/baseimage.txt docker/requirements/core.txt docker/requirements/core-x86.txt /tmp/requirements/
 COPY docker/install-conda-deps.sh /tmp/
 
 # Install conda dependencies (bioinformatics tools)
-# First install all-architecture packages, then x86-only packages (skipped on ARM)
-RUN /tmp/install-conda-deps.sh /tmp/requirements/core.txt && \
+# Install baseimage.txt + core.txt together in single resolver call for proper dependency resolution
+# Then install x86-only packages (skipped on ARM)
+RUN /tmp/install-conda-deps.sh /tmp/requirements/baseimage.txt /tmp/requirements/core.txt && \
     /tmp/install-conda-deps.sh --x86-only /tmp/requirements/core-x86.txt
 
 # Copy source code

docker/requirements/baseimage.txt

@@ -0,0 +1,24 @@
+# Conda dependencies for viral-ngs baseimage
+#
+# These are general-purpose tools installed in the base image that are
+# useful across all derivative images (core, classify, assemble, phylo).
+#
+# All packages are from conda-forge unless noted.
+
+# WDL local execution (miniwdl + udocker for containerized task execution)
+miniwdl>=1.11.1
+udocker>=1.3.16
+
+# Cloud CLIs
+awscli
+google-cloud-storage
+
+# General utilities
+csvkit>=1.0.4
+jq>=1.6
+parallel>=20190922
+pigz>=2.4
+unzip>=6.0
+zstd>=1.5.7
+
+# Note: qsv is x86-only (no linux-aarch64 build), skipping for multi-arch support

docker/requirements/core.txt

@@ -4,6 +4,7 @@
 # installation and better binary compatibility. This includes packages
 # with C extensions that would otherwise require compilation.
 #
+# Note: General utilities (parallel, pigz, jq, etc.) are in baseimage.txt
 # Note: x86-only packages (novoalign, mvicuna) are in core-x86.txt
 
 # Python runtime dependencies
@@ -18,6 +19,7 @@ pybedtools>=0.7.10
 pysam>=0.23.0
 pyyaml
 scipy
+seaborn
 zstandard>=0.23.0
 
 # Bioinformatics tools
@@ -26,22 +28,16 @@ bcftools>=1.10
 bedtools>=2.29.2
 bwa>=0.7.17
 cd-hit>=4.6.8
-csvkit>=1.0.4
 fastqc>=0.11.7
 fgbio>=2.2.1
 gatk=3.8
 illumina-interop=1.5.0
-jq>=1.6
 lbzip2>=2.5
 lz4-c>=1.8.3
 minimap2>=2.17
-parallel>=20190922
 picard=2.25.6
-pigz>=2.4
 prinseq>=0.20.4
 sambamba>=1.0.1
 samtools>=1.21
 splitcode>=0.31.4
 trimmomatic>=0.38
-unzip>=6.0
-zstd>=1.5.7

pyproject.toml

@@ -24,9 +24,24 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Bio-Informatics",
 ]
 keywords = ["bioinformatics", "genomics", "viral", "ngs", "sequencing"]
-# All runtime dependencies are installed via conda for faster installation
-# and better binary compatibility. See docker/requirements/core.txt
-dependencies = []
+# Python dependencies for pip-based installs
+# Note: For Docker-based usage, all deps (including bioinformatics tools) are
+# installed via conda. See docker/requirements/core.txt
+dependencies = [
+    "arrow>=0.12.1",
+    "biopython>=1.72",
+    "jinja2>=2.11.3",
+    "lz4>=2.2.1",
+    "matplotlib>=2.2.4",
+    "pandas",
+    "psutil>=6.1.0",
+    "pybedtools>=0.7.10",
+    "pysam>=0.23.0",
+    "pyyaml",
+    "scipy",
+    "seaborn",
+    "zstandard>=0.23.0",
+]
 
 [project.optional-dependencies]
 dev = [