Merge pull request #352 from getwilds/skip-cellranger

Scaling back CellRanger to a template and private distribution

Author: tefirman

.github/scripts/docker_update.py

@@ -75,6 +75,34 @@ def load_amd64_only_tools():
 AMD64_ONLY_TOOLS = load_amd64_only_tools()
 
 
+def load_ci_skip_tools():
+    """
+    Load the list of tools to skip in CI from ci_skip_tools.txt.
+
+    These tools are excluded from the automated build-and-push pipeline,
+    typically due to licensing restrictions that prevent automated downloads.
+    Their images are maintained manually outside of CI.
+
+    Returns:
+        set: Set of tool names that should be skipped in CI
+    """
+    ci_skip_file = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
+        "ci_skip_tools.txt"
+    )
+    try:
+        with open(ci_skip_file, "r") as f:
+            return {line.strip() for line in f if line.strip()}
+    except FileNotFoundError:
+        logger.warning(f"ci_skip_tools.txt not found at {ci_skip_file}, using empty set")
+        return set()
+
+
+# Tools to skip entirely in CI (e.g., due to licensing restrictions)
+# Loaded from ci_skip_tools.txt
+CI_SKIP_TOOLS = load_ci_skip_tools()
+
+
 def get_image_size(image_name):
     """
     Get the size of a Docker image in bytes using docker manifest inspect.
@@ -521,6 +549,11 @@ def build_and_push_images(docker_files):
         tool_name = os.path.dirname(dockerfile)
         tag = os.path.basename(dockerfile).split("_")[-1]
 
+        # Skip tools that are excluded from CI (e.g., due to licensing restrictions)
+        if tool_name in CI_SKIP_TOOLS:
+            logger.info(f"Skipping {tool_name}:{tag} (listed in ci_skip_tools.txt)")
+            continue
+
         logger.info(f"Building image for {tool_name}:{tag}")
 
         # Check if this image should be AMD64-only
@@ -591,6 +624,12 @@ def update_dockerhub_descriptions(affected_dirs):
             continue
 
         repo_name = os.path.basename(directory)
+
+        # Skip tools excluded from CI (no DockerHub repo to update)
+        if repo_name in CI_SKIP_TOOLS:
+            logger.info(f"Skipping {repo_name}: excluded from CI (ci_skip_tools.txt)")
+            continue
+
         logger.info(f"Processing repository: getwilds/{repo_name}")
 
         # Check for README.md

cellranger/CVEs_10.0.0.md

@@ -2,10 +2,16 @@
 # Using the Ubuntu base image
 FROM ubuntu:24.04
 
+# Build arguments for Cell Ranger version and download URL
+# Users must obtain their own download URL from 10x Genomics after accepting
+# their license agreement: https://www.10xgenomics.com/support/software/cell-ranger/downloads
+ARG CELLRANGER_VERSION
+ARG CELLRANGER_URL
+
 # Adding labels for the GitHub Container Registry
 LABEL org.opencontainers.image.title="cellranger"
 LABEL org.opencontainers.image.description="Container image for the use of Cell Ranger in Fred Hutch OCDO's WILDS"
-LABEL org.opencontainers.image.version="10.0.0"
+LABEL org.opencontainers.image.version="${CELLRANGER_VERSION}"
 LABEL org.opencontainers.image.authors="wilds@fredhutch.org"
 LABEL org.opencontainers.image.url=https://hutchdatascience.org/
 LABEL org.opencontainers.image.documentation=https://getwilds.org/
@@ -15,6 +21,19 @@ LABEL org.opencontainers.image.licenses=MIT
 # Set the shell option to fail if any command in a pipe fails
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
+# Validate that required build arguments were provided
+RUN if [ -z "${CELLRANGER_VERSION}" ]; then \
+  echo "ERROR: CELLRANGER_VERSION build argument is required." && \
+  echo "Example: --build-arg CELLRANGER_VERSION=10.0.0" && \
+  exit 1; \
+  fi
+RUN if [ -z "${CELLRANGER_URL}" ]; then \
+  echo "ERROR: CELLRANGER_URL build argument is required." && \
+  echo "Please obtain a download URL from 10x Genomics:" && \
+  echo "https://www.10xgenomics.com/support/software/cell-ranger/downloads" && \
+  exit 1; \
+  fi
+
 # Installing prerequisites
 RUN apt-get update \
   && BUILD_ESSENTIAL_VERSION=$(apt-cache policy build-essential | grep Candidate | awk '{print $2}') \
@@ -41,19 +60,18 @@ RUN apt-get update \
   && rm -rf /var/lib/apt/lists/*
 
 # Download and install Cell Ranger pre-built binary
-# Note: The download URL contains a signed key that expires.
-# A new download URL should be obtained from 10x Genomics each time this Dockerfile is updated.
-RUN wget -q --no-check-certificate -O cellranger-10.0.0.tar.gz \
-  "https://cf.10xgenomics.com/releases/cell-exp/cellranger-10.0.0.tar.gz?Expires=1766424907&Key-Pair-Id=APKAI7S6A5RYOXBWRPDA&Signature=Se7y~SfN80~5Q61MkUOLIgsO40FlwMj6BuJjAmP6S74H7ZI6PSI8CCZgOHMoz3BUYZKhKwI~oC5sUqUqMbzw3wbmJcmrhPneNagGvnDwRmULTA6Smyn4K7A-k0t7qjVW5W8aiERueW5057JJk4LKPZJHKoV7tqMVA5beAU4Ari2Deu5K5hHKfNK0OWIYl9mxlAP8mg2Go6KzhOVFKAQy-VzoqpGFSOW3jSnHAMrOhHtpX4jdhshm-Kca2NeSAZome4j6uPfUyKYdQRWTwOI1EgXcyRFRBHEQe6TVsOt4IqxtCYKEgjLm3SDt5f5Nzds5plRJ~V2jpcG-ew5ABnUXNg__" \
-  && tar -xzf cellranger-10.0.0.tar.gz \
-  && rm -rf cellranger-10.0.0.tar.gz
+# The download URL must be obtained from 10x Genomics after accepting their license agreement.
+# URLs contain a signed key that expires, so a fresh URL is needed for each build.
+RUN wget -q --no-check-certificate -O "cellranger-${CELLRANGER_VERSION}.tar.gz" \
+  "${CELLRANGER_URL}" \
+  && tar -xzf "cellranger-${CELLRANGER_VERSION}.tar.gz" \
+  && rm -rf "cellranger-${CELLRANGER_VERSION}.tar.gz"
 
 # Set environment variables
-ENV PATH="${PATH}:/cellranger-10.0.0"
+ENV PATH="${PATH}:/cellranger-${CELLRANGER_VERSION}"
 
 # Set working directory
 WORKDIR /data
 
 # Smoke test (NOTE: Cell Ranger only works on x86_64 Linux)
 RUN cellranger --version
-