Merge pull request #194 from lu-wang-dl/update-gpu-container

Fix the GPU container issues

Author: lu-wang-dl

ubuntu/gpu/cuda-11.8/base/Dockerfile

@@ -8,12 +8,13 @@ RUN cd /etc/apt/sources.list.d && \
 # See https://github.com/databricks/containers/blob/master/ubuntu/minimal/Dockerfile
 RUN apt-get update && \
     apt-get install --yes \
-      openjdk-8-jdk-headless \
+      openjdk-8-jdk \
       iproute2 \
       bash \
       sudo \
       coreutils \
       procps \
+      acl \
       wget && \
     /var/lib/dpkg/info/ca-certificates-java.postinst configure && \
     apt-get clean && \
@@ -35,4 +36,8 @@ RUN apt-get update \
   && add-apt-repository -r "deb [arch=amd64,i386] https://cran.rstudio.com/bin/linux/ubuntu $(lsb_release -cs)-cran40/" \
   && apt-key del E298A3A825C0D65DFD57CBB651716619E084DAB9 \
   && apt-get clean \
-  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Add new user for cluster library installation
+RUN useradd libraries \
+&& usermod -L libraries

ubuntu/gpu/cuda-11.8/venv/Dockerfile

@@ -1,14 +1,20 @@
 
 FROM databricksruntime/gpu-base:cuda11.8
 
+ARG python_version="3.10"
+ARG pip_version="22.3.1"
+ARG setuptools_version="65.6.3"
+ARG wheel_version="0.38.4"
+ARG virtualenv_version="20.16.7"
+
 WORKDIR /databricks
 
 # Install python 3.10 from ubuntu.
 # Install pip via get-pip.py bootstrap script and install versions that match Anaconda distribution.
 RUN apt-get update \
-  && apt-get install curl software-properties-common -y python3.10 python3.10-dev python3.10-distutils \
+  && apt-get install curl software-properties-common -y python${python_version} python${python_version}-dev python${python_version}-distutils \
   && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
-  && /usr/bin/python3.10 get-pip.py pip==22.3.1 setuptools==65.6.3 wheel==0.38.4 \
+  && /usr/bin/python${python_version} get-pip.py pip==${pip_version} setuptools==${setuptools_version} wheel==${wheel_version} \
   && rm get-pip.py
 
 
@@ -17,16 +23,17 @@ RUN apt-get update \
 # with user cleanup and may allow users to inadvertently update pip to newer versions
 # incompatible with Databricks. Instead, we patch virtualenv to disable periodic updates per
 # https://virtualenv.pypa.io/en/latest/user_guide.html#embed-wheels-for-distributions.
-RUN /usr/local/bin/pip3.10 install --no-cache-dir virtualenv==20.16.7 \
-    && sed -i -r 's/^(PERIODIC_UPDATE_ON_BY_DEFAULT) = True$/\1 = False/' /usr/local/lib/python3.10/dist-packages/virtualenv/seed/embed/base_embed.py \
-    && /usr/local/bin/pip3.10 download pip==22.3.1 --dest \
-    /usr/local/lib/python3.10/dist-packages/virtualenv_support/
+RUN /usr/local/bin/pip${python_version} install --no-cache-dir virtualenv==${virtualenv_version} \
+    && sed -i -r 's/^(PERIODIC_UPDATE_ON_BY_DEFAULT) = True$/\1 = False/' /usr/local/lib/python${python_version}/dist-packages/virtualenv/seed/embed/base_embed.py \
+    && /usr/local/bin/pip${python_version} download pip==${pip_version} --dest \
+    /usr/local/lib/python${python_version}/dist-packages/virtualenv_support/
 
 # Create /databricks/python3 environment.
 # We install pip and wheel so their executables show up under /databricks/python3/bin.
 # We use `--system-site-packages` so python will fallback to system site packages.
 # We use `--no-download` so virtualenv will install the bundled pip and wheel.
-RUN virtualenv --python=/usr/bin/python3.10 /databricks/python3 --system-site-packages --no-download
+# Initialize the default environment that Spark and notebooks will use
+RUN virtualenv --python=python${python_version} --system-site-packages /databricks/python3 --no-download --no-setuptools
 
 
 # These python libraries are used by Databricks notebooks and the Python REPL
@@ -53,6 +60,12 @@ RUN /databricks/python3/bin/pip install \
 # Specifies where Spark will look for the python binary
 ENV PYSPARK_PYTHON=/databricks/python3/bin/python3
 
+RUN virtualenv --python=python${python_version} --system-site-packages /databricks/python-lsp --no-download --no-setuptools
+
+COPY python-lsp-requirements.txt /databricks/.
+
+RUN /databricks/python-lsp/bin/pip install -r /databricks/python-lsp-requirements.txt
+
 # Use pip cache purge to cleanup the cache safely
 RUN /databricks/python3/bin/pip cache purge
 

ubuntu/gpu/cuda-11.8/venv/python-lsp-requirements.txt

@@ -0,0 +1,35 @@
+astroid==3.0.1
+attrs==23.1.0
+autopep8==2.0.4
+cattrs==23.2.3
+dill==0.3.7
+docstring-to-markdown==0.13
+exceptiongroup==1.1.3
+flake8==6.1.0
+importlib-metadata==6.8.0
+isort==5.12.0
+jedi==0.19.1
+lsprotocol==2023.0.0
+mccabe==0.7.0
+packaging==23.2
+parso==0.8.3
+platformdirs==4.0.0
+pluggy==1.3.0
+pycodestyle==2.11.1
+pydocstyle==6.3.0
+pyflakes==3.1.0
+pylint==3.0.2
+python-lsp-jsonrpc==1.1.2
+python-lsp-ruff==1.6.0
+python-lsp-server==1.10.0
+pytoolconfig==1.2.6
+rope==1.12.0
+ruff==0.1.6
+snowballstemmer==2.2.0
+tomli==2.0.1
+tomlkit==0.12.3
+typing_extensions==4.8.0
+ujson==5.8.0
+whatthepatch==1.0.5
+yapf==0.40.2
+zipp==3.17.0