Merge pull request #3593 from AlexsLemonade/davidsmejia/3574-un-nvidia-compendia

Move compendia off of nvidia cuda image and onto base

Author: davidsmejia

docker-bake.hcl

@@ -44,8 +44,9 @@ variable "PUSH_CACHE" {
   default = false
 }
 
-// Default amd64 because some upstream bases (nvidia/cuda for compendia) and
-// apt packages have no arm64 variant.
+// Default amd64 because the Salmon / SRA Toolkit binaries (Dockerfile.salmon,
+// Dockerfile.transcriptome) are x86_64-only and some apt packages have no
+// arm64 variant.
 variable "PLATFORMS" {
   // description = "Comma-separated platforms to build for."
   default = "linux/amd64"
@@ -240,10 +241,10 @@ target "downloaders" {
   cache-to   = cache_to_for("downloaders")
 }
 
-// Compendia uses nvidia/cuda directly, not the shared base.
 target "compendia" {
   inherits   = ["_common"]
   dockerfile = "workers/dockerfiles/Dockerfile.compendia"
+  contexts   = from_target("base")
   tags       = tags_for("compendia")
   cache-from = cache_from_for("compendia")
   cache-to   = cache_to_for("compendia")

workers/data_refinery_workers/processors/create_compendia.py

@@ -278,10 +278,6 @@ def _filter_rows_and_columns(job_context: Dict) -> Dict:
 
     drop_na_genes_start = log_state("start drop NA genes", job_context["job"].id)
 
-    # # Visualize Prefiltered
-    # output_path = job_context['output_dir'] + "pre_filtered_" + str(time.time()) + ".png"
-    # visualized_prefilter = visualize.visualize(combined_matrix.copy(), output_path)
-
     combined_matrix = job_context.pop("combined_matrix")
 
     # Remove genes (rows) with <=70% present values in combined_matrix
@@ -297,10 +293,6 @@ def _filter_rows_and_columns(job_context: Dict) -> Dict:
     log_state("end drop NA genes", job_context["job"].id, drop_na_genes_start)
     drop_na_samples_start = log_state("start drop NA samples", job_context["job"].id)
 
-    # # Visualize Row Filtered
-    # output_path = job_context['output_dir'] + "row_filtered_" + str(time.time()) + ".png"
-    # visualized_rowfilter = visualize.visualize(row_filtered_matrix.copy(), output_path)
-
     # Remove samples (columns) with <50% present values in combined_matrix
     # XXX: Find better test data for this!
     col_thresh = row_filtered_matrix.shape[0] * 0.5
@@ -342,11 +334,6 @@ def _reset_zero_values(job_context: Dict) -> Dict:
     row_col_filtered_matrix_index = job_context.get("row_col_filtered_matrix_index")
     cached_zeroes = job_context.pop("cached_zeroes")
 
-    # # Visualize Row and Column Filtered
-    # output_path = job_context['output_dir'] + "row_col_filtered_" + str(time.time()) + ".png"
-    # visualized_rowcolfilter = visualize.visualize(row_col_filtered_matrix.copy(),
-    #                                               output_path)
-
     # "Reset" zero values that were set to NA in RNA-seq samples
     # (i.e., make these zero again) in combined_matrix
     for column in cached_zeroes.keys():
@@ -442,9 +429,6 @@ def _run_iterativesvd(job_context: Dict) -> Dict:
     del row_col_filtered_matrix_columns
 
     job_context["merged_no_qn"] = untransposed_imputed_matrix_df
-    # output_path = job_context['output_dir'] + "compendia_no_qn_" + str(time.time()) + ".png"
-    # visualized_merged_no_qn = visualize.visualize(untransposed_imputed_matrix_df.copy(),
-    #                                               output_path)
 
     log_state("end untranspose", job_context["job"].id, untranspose_start)
 
@@ -511,10 +495,6 @@ def _quantile_normalize(job_context: Dict) -> Dict:
 
     log_state("end quantile normalize", job_context["job"].id, quantile_start)
 
-    # Visualize Final Compendia
-    # output_path = job_context['output_dir'] + "compendia_with_qn_" + str(time.time()) + ".png"
-    # visualized_merged_qn = visualize.visualize(job_context['merged_qn'].copy(), output_path)
-
     return job_context
 
 

workers/data_refinery_workers/processors/requirements.in

@@ -18,13 +18,6 @@ simplejson
 sympy
 unicodecsv
 untangle>=1.1.1
-# For visualization
-dask>=1.1.1
-xarray==0.16.2
-datashader==0.14.4
-datashape>=0.5.2
-holoviews>=1.11.2
-selenium==3.141.0
 # For mocking out S3 calls for the smasher.
 vcrpy==4.1.1
 django-computedfields>=0.1.5

workers/data_refinery_workers/processors/requirements.txt

@@ -1,93 +1,61 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile --annotation-style=line requirements.in
 #
 asgiref==3.6.0            # via django
-bleach==6.0.0             # via panel
-bokeh==2.4.3              # via panel
 boto3==1.26.72            # via -r requirements.in
 botocore==1.29.72         # via boto3, s3transfer
 certifi==2023.7.22        # via -r requirements.in, requests
 cffi==1.15.1              # via rpy2
 charset-normalizer==3.0.1  # via requests
-click==8.1.3              # via dask
-cloudpickle==2.2.1        # via dask
-colorcet==3.0.1           # via datashader, holoviews
 contourpy==1.0.7          # via matplotlib
 cycler==0.11.0            # via matplotlib
-dask==2023.2.0            # via -r requirements.in, datashader
-datashader==0.14.4        # via -r requirements.in
-datashape==0.5.2          # via -r requirements.in, datashader
 defusedxml==0.7.1         # via untangle
 django==3.2.18            # via -r requirements.in, django-computedfields, django-fast-update
 django-computedfields==0.2.1  # via -r requirements.in
 django-fast-update==0.2.1  # via django-computedfields
 fonttools==4.38.0         # via matplotlib
-fsspec==2023.1.0          # via dask
-holoviews==1.15.4         # via -r requirements.in
 idna==3.4                 # via requests, yarl
-importlib-metadata==6.0.0  # via markdown, numba
-importlib-resources==5.10.2  # via matplotlib
-jinja2==3.1.2             # via -r requirements.in, bokeh, rpy2
+jinja2==3.1.2             # via -r requirements.in, rpy2
 jmespath==1.0.1           # via boto3, botocore
 joblib==1.2.0             # via scikit-learn
 kiwisolver==1.4.4         # via matplotlib
-llvmlite==0.39.1          # via numba
-locket==1.0.0             # via partd
-markdown==3.4.1           # via panel
 markupsafe==2.1.2         # via jinja2
 matplotlib==3.7.0         # via -r requirements.in
 mpmath==1.2.1             # via sympy
 multidict==6.0.4          # via yarl
-multipledispatch==0.6.0   # via datashape
 nose==1.3.7               # via -r requirements.in
-numba==0.56.4             # via datashader
-numpy==1.23.5             # via -r requirements.in, bokeh, contourpy, datashader, datashape, holoviews, matplotlib, numba, pandas, scikit-learn, scipy, xarray
-packaging==23.0           # via bokeh, dask, holoviews, matplotlib
-pandas==1.5.3             # via -r requirements.in, datashader, holoviews, xarray
-panel==0.14.3             # via holoviews
-param==1.12.3             # via datashader, holoviews, panel, pyct, pyviz-comms
-partd==1.3.0              # via dask
-pillow==9.4.0             # via -r requirements.in, bokeh, datashader, matplotlib
+numpy==1.23.5             # via -r requirements.in, contourpy, matplotlib, pandas, scikit-learn, scipy
+packaging==23.0           # via matplotlib
+pandas==1.5.3             # via -r requirements.in
+pillow==9.4.0             # via -r requirements.in, matplotlib
 psutil==5.9.4             # via -r requirements.in
 psycopg2-binary==2.9.5    # via -r requirements.in
 pycparser==2.21           # via cffi
-pyct==0.5.0               # via colorcet, datashader, panel
 pyparsing==3.0.9          # via matplotlib
-python-dateutil==2.8.2    # via botocore, datashape, matplotlib, pandas
+python-dateutil==2.8.2    # via botocore, matplotlib, pandas
 pytz==2022.7.1            # via django, pandas, rpy2
 pytz-deprecation-shim==0.1.0.post0  # via tzlocal
-pyviz-comms==2.2.1        # via holoviews, panel
-pyyaml==6.0               # via -r requirements.in, bokeh, dask, vcrpy
-requests==2.28.2          # via -r requirements.in, datashader, panel
+pyyaml==6.0               # via -r requirements.in, vcrpy
+requests==2.28.2          # via -r requirements.in
 retrying==1.3.4           # via -r requirements.in
 rpy2===3.4.5              # via -r requirements.in
 s3transfer==0.6.0         # via boto3
 scikit-learn==1.2.1       # via -r requirements.in
-scipy==1.10.0             # via -r requirements.in, datashader, scikit-learn
-selenium==3.141.0         # via -r requirements.in
+scipy==1.10.0             # via -r requirements.in, scikit-learn
 simplejson==3.18.3        # via -r requirements.in
-six==1.16.0               # via bleach, multipledispatch, python-dateutil, retrying, vcrpy
+six==1.16.0               # via python-dateutil, retrying, vcrpy
 sqlparse==0.4.3           # via -r requirements.in, django
 sympy==1.11.1             # via -r requirements.in
 threadpoolctl==3.1.0      # via scikit-learn
-toolz==0.12.0             # via dask, datashader, partd
-tornado==6.2              # via bokeh
-tqdm==4.64.1              # via panel
-typing-extensions==4.5.0  # via bokeh, django-computedfields, panel
+typing-extensions==4.5.0  # via django-computedfields
 tzdata==2022.7            # via pytz-deprecation-shim
 tzlocal==4.2              # via -r requirements.in, rpy2
 unicodecsv==0.14.1        # via -r requirements.in
 untangle==1.2.1           # via -r requirements.in
-urllib3==1.26.14          # via botocore, requests, selenium
+urllib3==1.26.14          # via botocore, requests
 vcrpy==4.1.1              # via -r requirements.in
-webencodings==0.5.1       # via bleach
 wrapt==1.14.1             # via vcrpy
-xarray==0.16.2            # via -r requirements.in, datashader
 yarl==1.8.2               # via vcrpy
-zipp==3.13.0              # via importlib-metadata, importlib-resources
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools

workers/data_refinery_workers/processors/visualize.py

@@ -1,112 +1,38 @@
-# This is very similar to the `smasher` image, but comes with OpenBLAS and some
-# of the other libraries required for fancyimpute.
+# This is very similar to the `smasher` image, but additionally installs
+# fancySVD, which provides the IterativeSVD imputation create_compendia uses.
 
-FROM nvidia/cuda:11.8.0-runtime-ubuntu18.04
+ARG DOCKERHUB_REPO
+ARG SYSTEM_VERSION
+FROM $DOCKERHUB_REPO/dr_base:$SYSTEM_VERSION
+ARG SYSTEM_VERSION
 
 # Fail in case of an error at any stage in the pipe.
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-# Add MIT as a fallback mirror alongside archive.ubuntu.com.
-RUN grep '^deb ' /etc/apt/sources.list \
-    | sed -e 's|http://archive.ubuntu.com/ubuntu|http://mirrors.mit.edu/ubuntu|g' \
-          -e 's|http://security.ubuntu.com/ubuntu|http://mirrors.mit.edu/ubuntu|g' \
-    >> /etc/apt/sources.list
-
-# Prevent tzdata from prompting us for a timezone and hanging the build.
-ENV DEBIAN_FRONTEND=noninteractive
-
-# Source: https://github.com/thisbejim/Pyrebase/issues/87#issuecomment-354452082
-# For whatever reason this worked and 'en_US.UTF-8' did not.
-ENV LANG=C.UTF-8
-
-# RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
-# via https://github.com/ilikenwf/apt-fast/issues/85#issuecomment-261640099
-RUN <<EOF
-echo debconf apt-fast/maxdownloads string 16 | debconf-set-selections
-echo debconf apt-fast/dlflag boolean true | debconf-set-selections
-echo debconf apt-fast/aptmanager string apt-get | debconf-set-selections
-echo 'tzdata tzdata/Areas select Etc' | debconf-set-selections
-echo 'tzdata tzdata/Zones/Etc select UTC' | debconf-set-selections
-apt-get update -qq
-apt-get install --no-install-recommends -y software-properties-common
-add-apt-repository ppa:apt-fast/stable
-add-apt-repository ppa:deadsnakes/ppa
-add-apt-repository ppa:savoury1/llvm-defaults-10
-apt-get update -qq
-apt-get install --no-install-recommends -y apt-fast apt-transport-https gpg-agent tzdata
-apt-fast update -qq
-apt-fast install -y \
-    build-essential \
-    cmake \
-    curl \
-    cython3 \
-    ed \
-    gfortran \
-    git \
-    libblas-dev \
-    libcairo-dev \
-    libcurl4-openssl-dev \
-    libedit-dev \
-    liblapack-dev \
-    libpq-dev \
-    libssl-dev \
-    libxml2-dev \
-    llvm-10-dev \
-    lsb-release \
-    mercurial \
-    pkg-config \
-    python3-pip \
-    python3-pybind11 \
-    python3.8 \
-    python3.8-dev \
-    r-base-core \
-    wget
-apt-get clean
-rm -rf /var/lib/apt/lists/*
-ln -s /usr/bin/llvm-config-10 /usr/bin/llvm-config
-update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
-pip3 install --upgrade pip setuptools
-groupadd user
-useradd --create-home --home-dir /home/user -g user user
-EOF
-
 WORKDIR /home/user
 
-ENV R_LIBS=/usr/local/lib/R/site-library
-
 COPY workers/R/dependencies/compendia/renv.lock .
 COPY common/R/renv_load.R renv_load_compendia.R
 RUN Rscript renv_load_compendia.R
 
-COPY workers/data_refinery_workers/requirements/compendia.txt requirements.txt
-# Pin setuptools<70 (otherwise it tries to import wheel internals that
-# newer wheel releases moved) so fancySVD builds cleanly from sdist.
-RUN pip3 install --upgrade 'setuptools<70' 'wheel<0.42'
-RUN pip3 install --ignore-installed --no-cache-dir -r requirements.txt
-
 COPY workers/data_refinery_workers/processors/requirements.txt requirements.txt
 RUN pip3 install --ignore-installed --no-cache-dir -r requirements.txt
 
-# Install phantomjs.
-RUN <<EOF
-wget -q https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2
-tar xvjf phantomjs-2.1.1-linux-x86_64.tar.bz2 -C /usr/local/share/
-ln -s /usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/local/bin/
-rm phantomjs-2.1.1-linux-x86_64.tar.bz2
-EOF
+# fancySVD provides the IterativeSVD imputation create_compendia imports. It is
+# compendia-only and pure-Python, but its setup.py pins scikit-learn==0.24.2 and
+# an old numpy that have no Python 3.10 wheels (they'd build from sdist). Install
+# it without deps; the scikit-learn and numpy pinned above satisfy it at runtime.
+RUN pip3 install --no-deps --no-cache-dir fancySVD==0.1.3
 
 # Get the latest version from the dist directory.
 COPY common/dist/data?refinery?common-* common/
 RUN pip3 install --ignore-installed --no-cache-dir common/$(ls common -1 | sort --version-sort | tail -1)
 
 COPY .boto .boto
-COPY config/ config/
-COPY setup.cfg .
 COPY workers/ .
 
 RUN rm -rf /root/.cache/*
 
-ARG SYSTEM_VERSION
 ENV SYSTEM_VERSION=$SYSTEM_VERSION
 
 USER user