Dockerfile.25.12-pytorch2.10-py312-cuda13.1

FROM nvcr.io/nvidia/pytorch:25.12-py3
# NVIDIA NGC PyTorch 2.10 with Python 3.12

ARG CODE_SERVER_VERSION=4.107.0 \
    GIT_LFS_VERSION=3.7.1

ENV DEBIAN_FRONTEND=noninteractive \
    MPLBACKEND=Svg \
    PIP_IGNORE_INSTALLED=0 \
    PIP_BREAK_SYSTEM_PACKAGES=1 \
    PYTHONUNBUFFERED=1 \
    LANG=C.UTF-8

RUN dpkgArch="$(dpkg --print-architecture)"; \
    case "${dpkgArch##*-}" in \
        amd64) tarArch='amd64'; dirArch='x86_64';; \
        arm64) tarArch='arm64'; dirArch='aarch64';; \
        *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding binary release"; exit 1 ;; \
    esac; \
    LD_LIBRARY_PATH="/usr/local/lib/python3.12/dist-packages/torch/lib:/usr/local/lib/python3.12/dist-packages/torch_tensorrt/lib:/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda:/usr/local/cuda/include:/usr/include/${dirArch}-linux-gnu:/opt/hpcx/ucc/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH" \
    PATH="/usr/local/lib/python3.12/dist-packages/torch_tensorrt/bin:/usr/local/mpi/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/ucx/bin:/opt/tensorrt/bin" \
    CPLUS_INCLUDE_PATH=/usr/include/gdal \
    C_INCLUDE_PATH=/usr/include/gdal

RUN apt-key adv --refresh-keys --keyserver keyserver.ubuntu.com && \
    apt-get update && \
    apt-get install -y --no-install-recommends \
	automake \
	bison \
	build-essential \
	bzip2 \
	ca-certificates \
	cabextract \
	chrpath \
	devscripts \
	dh-make \
	dirmngr \
	dkms \
	ethtool \
	fakeroot \
	ffmpeg \
	flex \
	fonts-nanum \
	fonts-nanum-coding \
	fonts-nanum-extra \
	g++ \
	gcc \
	gdal-bin \
	gfortran \
	gir1.2-gtk-3.0 \
	git \
	graphviz \
	htop \
	iputils-ping \
	libasound2-dev \
	libatlas-base-dev \
	libavcodec-dev \
	libavformat-dev \
	libboost-dev \
	libboost-filesystem-dev \
	libboost-system-dev \
	libcairo2-dev \
	libcurl4-openssl-dev \
	libdc1394-dev \
	libeigen3-dev \
	libfaac-dev \
	libfftw3-dev \
	libfreetype-dev \
	libfuse2 \
	libgdal-dev \
	libgeos++-dev \
	libgeos-dev \
	libgflags-dev \
	libgirepository1.0-dev \
	libglib2.0-0 \
	libgoogle-glog-dev \
	libgphoto2-dev \
	libgstreamer-plugins-base1.0-dev \
	libgstreamer1.0-dev \
	libgtk-3-dev \
	libhdf5-dev \
	libjemalloc-dev \
	libjpeg-dev \
	liblapack-dev \
	libmnl0 \
	libmp3lame-dev \
	libncurses-dev \
	libnl-3-200 \
	libnl-3-dev \
	libnl-route-3-200 \
	libnl-route-3-dev \
	libnlopt-dev \
	libnuma-dev \
	libopenblas-dev \
	libopencore-amrnb-dev \
	libopencore-amrwb-dev \
	libpci3 \
	libpng-dev \
	libportmidi-dev \
	libproj-dev \
	libprotobuf-dev \
	libsdl2-dev \
	libsdl-image1.2-dev \
	libsdl-mixer1.2-dev \
	libsdl-ttf2.0-dev \
	libsm6 \
	libsmpeg-dev \
	libssl-dev \
	libswscale-dev \
	libswresample-dev \
	libtbb-dev \
	libtheora-dev \
	libtiff-dev \
	libv4l-dev \
	libvorbis-dev \
	libx264-dev \
	libxext6 \
	libxine2-dev \
	libxml2-dev \
	libxrender-dev \
	libxrender1 \
	libxslt1-dev \
	libxvidcore-dev \
	libyaml-dev \
	libzmq3-dev \
	lintian \
	lsof \
	make \
	mercurial \
	media-types \
	mailcap \
	ncurses-term \
	net-tools \
	openjdk-11-jdk \
	openssh-client \
	openssh-server \
	pbzip2 \
	pciutils \
	pdsh \
	pkg-config \
	proj-bin \
	protobuf-compiler \
	pv \
	python-is-python3 \
	python3-xkit \
	python3-gdal \
	screen-resolution-extra \
	sudo \
	subversion \
	swig \
	tk \
	unzip \
	v4l-utils \
	vim \
	x264 \
	xvfb \
	xz-utils \
	sudo \
	yasm \
	zip \
	tcl \
	udev && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/*

RUN dpkgArch="$(dpkg --print-architecture)"; \
    case "${dpkgArch##*-}" in \
        amd64) tarArch='amd64'; dirArch='x64';; \
        arm64) tarArch='arm64'; dirArch='-arm64';; \
        *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding binary release"; exit 1 ;; \
    esac; \
    \
	mkdir -p /opt/oracle && \
    cd /opt/oracle && \
    wget https://download.oracle.com/otn_software/linux/instantclient/instantclient-basiclite-linux${dirArch}.zip && \
    unzip instantclient-basiclite-linux${dirArch}.zip && \
    rm -f instantclient-basiclite-linux${dirArch}.zip && \
    cd /opt/oracle/instantclient* && \
    rm -f *jdbc* *occi* *mysql* *README *jar uidrvci genezi adrci && \
    echo /opt/oracle/instantclient* > /etc/ld.so.conf.d/oracle-instantclient.conf && \
    ldconfig

# Install Node.js (LTS) + yarn
RUN cd /tmp && \
    curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
    apt-get update && apt-get install -y nodejs && \
    npm install -g corepack && \
    corepack enable && \
    corepack prepare yarn@stable --activate && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/*

# Install CUDA + cuDNN
RUN dpkgArch="$(dpkg --print-architecture)"; \
    case "${dpkgArch##*-}" in \
        amd64) tarArch='amd64'; dirArch='x86_64';; \
        arm64) tarArch='arm64'; dirArch='aarch64';; \
        *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding binary release"; exit 1 ;; \
    esac; \
	mkdir -p /usr/local/nvidia/lib && \
    ln -s /usr/lib/${dirArch}-linux-gnu/libcudnn.so.9.17.0 /usr/local/cuda/lib64/libcudnn.so && \
    ldconfig && \
    update-alternatives --install /usr/bin/python python /usr/bin/python3 2

WORKDIR /tmp
RUN dpkgArch="$(dpkg --print-architecture)"; \
    case "${dpkgArch##*-}" in \
        amd64) openblasTarget='CORE2'; crossCompile='0'; EXTRA_FLAGS="DYNAMIC_ARCH=1"; ;; \
        arm64) openblasTarget='ARMV8'; crossCompile='1'; EXTRA_FLAGS=""; ;; \
        *) echo >&2 "Unsupported: $dpkgArch"; exit 1 ;; \
    esac; \
    git clone -q --branch=v0.3.30 https://github.com/OpenMathLib/OpenBLAS.git && \
    cd OpenBLAS && \
    make TARGET=${openblasTarget} CROSS=${crossCompile} ${EXTRA_FLAGS} NO_AFFINITY=1 NUM_THREADS=48 FC=gfortran && \
    make install && \
    rm -rf /tmp/OpenBLAS

# install git-lfs + bashtop
WORKDIR /tmp
RUN dpkgArch="$(dpkg --print-architecture)"; \
    case "${dpkgArch##*-}" in \
        amd64) tarArch='amd64'; dirArch='x64';; \
        arm64) tarArch='arm64'; dirArch='aarch64';; \
        *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding binary release"; exit 1 ;; \
    esac; \
    \
    curl -sLO "https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/git-lfs-linux-${tarArch}-v${GIT_LFS_VERSION}.tar.gz" && \
    tar -zxf "git-lfs-linux-${tarArch}-v${GIT_LFS_VERSION}.tar.gz" && \
    cd /tmp/git-lfs-${GIT_LFS_VERSION} && \
    bash install.sh && \
    cd /tmp && \
    git clone https://github.com/aristocratos/bashtop.git && \
    cd bashtop && \
    make install && \
    rm -rf /tmp/*

RUN dpkgArch="$(dpkg --print-architecture)"; \
    case "${dpkgArch##*-}" in \
        amd64) tarArch='amd64'; dirArch='x64';; \
        arm64) tarArch='arm64'; dirArch='aarch64';; \
        *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding binary release"; exit 1 ;; \
    esac; \
     curl -fL https://github.com/cdr/code-server/releases/download/v${CODE_SERVER_VERSION}/code-server-${CODE_SERVER_VERSION}-linux-${tarArch}.tar.gz \
      | tar -C /usr/local/lib -xz && \
    mv /usr/local/lib/code-server-${CODE_SERVER_VERSION}-linux-${tarArch} /usr/local/lib/code-server-${CODE_SERVER_VERSION} && \
    ln -s /usr/local/lib/code-server-${CODE_SERVER_VERSION}/bin/code-server /usr/local/bin/code-server

# remove hwloc-like packages (ImportError: /opt/hpcx/ucc/lib/libucc.so.1: undefined symbol issue)
#RUN apt-get purge -y hwloc-nox libhwloc-plugins

# Python packages installation (consolidated: requirements + datasets + mpi4py + mlflow)
COPY ./requirements.25.12.*.txt /tmp/
RUN dpkgArch="$(dpkg --print-architecture)"; \
    case "${dpkgArch##*-}" in \
        amd64) tarArch='amd64';; \
        arm64) tarArch='arm64';; \
        *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding binary release"; exit 1 ;; \
    esac; \
    python3 -m pip install --disable-pip-version-check --no-cache-dir \
        -r requirements.25.12.${tarArch}.txt \
        datasets \
        mpi4py==4.1.1 mlflow==3.5.0 && \
    find /usr/local/lib/python3.12/dist-packages -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \
    rm -rf /tmp/*

# PyTorch extensions (requires --no-build-isolation)
RUN python3 -m pip install --no-build-isolation --no-cache-dir \
        pytorch-lightning \
        torch-scatter \
        torch-sparse \
        torch-cluster \
        torch-spline-conv \
        torch-geometric \
        torchao && \
    find /usr/local/lib/python3.12/dist-packages -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \
    rm -rf /tmp/*

WORKDIR /tmp
RUN git clone --recursive  -q https://github.com/bitsandbytes-foundation/bitsandbytes.git && \
    cd /tmp/bitsandbytes && \
    cmake -DCOMPUTE_BACKEND=cuda -DCMAKE_CUDA_COMPILER="/usr/local/cuda-13/bin/nvcc" -DCOMPUTE_CAPABILITY="75;80;86;87;89;90;100;103;110;120;121" -S . && \
	make && \
    python setup.py install && \
    rm -rf /tmp/*

# Install ipython kernelspec
RUN python3 -m ipykernel install --display-name "PyTorch 2.10 (NGC 25.12/Python 3.12) on Backend.AI" && \
    cat /usr/local/share/jupyter/kernels/python3/kernel.json
# Backend.AI specifics
COPY ./service-defs /etc/backend.ai/service-defs
COPY ./runner-scripts/bootstrap.sh runner-scripts/setup_multinode.py /opt/container/

LABEL ai.backend.kernelspec="1" \
      ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
      ai.backend.features="batch query uid-match user-input" \
      ai.backend.base-distro="ubuntu24.04" \
      ai.backend.accelerators="cuda" \
      ai.backend.resource.min.cpu="1" \
      ai.backend.resource.min.mem="1g" \
      ai.backend.resource.min.cuda.device=0 \
      ai.backend.resource.min.cuda.shares=0 \
      ai.backend.runtime-type="python" \
      ai.backend.runtime-path="/usr/bin/python" \
      ai.backend.service-ports="ipython:pty:3000,jupyter:http:8091,jupyterlab:http:8090,vscode:http:8180,tensorboard:http:6006,mlflow-ui:preopen:5000,nniboard:preopen:8080"

# Install Jupyterlab extensions + build (merged with cleanup)
RUN python3 -m pip install --no-cache-dir \
    jupyter_nbextensions_configurator>=0.6.5 \
	jupyter_core \
	jupyter-contrib-core>=0.4.2 \
	jupyter-highlight-selected-word>=0.2.0 \
	jupyter-server>=2.17.0 \
    jupyter_lsp>=2.3.0 \
	jupyter-dash \
	jupyter-js-widgets-nbextension \
	jupyter-latex-envs \
	jupyter-packaging \
	jupyterlab==4.4.10 \
	jupyterlab-code-formatter>=3.0.2 \
	jupyterlab-hdf \
	jupyterlab-launcher \
	jupyterlab-widgets==3.0.16 \
    notebook \
	jupyter-server-mathjax \
	jupyter-server-proxy>=4.4.0 \
	jupyter-telemetry==0.1.0 \
	jupyter>=1.1.1 \
	jupyter-client==8.6.3 \
	jupyter_bokeh==2.0.4 \
	markupsafe>=3.0.2 \
	jsonschema[format,format-nongpl]>=4.23.0 && \
    python3 -m pip install --no-cache-dir jupyter_lsp markupsafe==3.0.2 jupyterlab_widgets && \
    jupyter labextension install --no-build @jupyter-widgets/jupyterlab-manager && \
	jupyter labextension install --no-build @jupyter-widgets/controls && \
    jupyter labextension install --no-build @jupyterlab/toc-extension && \
    jupyter labextension install --no-build @krassowski/jupyterlab-lsp && \
    jupyter labextension install @jupyterlab/toc-extension && \
    jupyter lab build --dev-build=False --minimize=False && \
    find /usr/local/lib/python3.12/dist-packages -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \
    rm -rf /usr/local/share/jupyter/lab/staging && \
    rm -rf /root/.cache /root/.npm && \
    rm -rf /tmp/*

RUN sed -i 's/source \/usr\/local\/nvm\/nvm.sh//' /etc/bash.bashrc && \
    ln -sf /usr/share/terminfo/x/xterm-color /usr/share/terminfo/x/xterm-256color

# change permission
RUN chown root:root /usr/lib

WORKDIR /home/work