diff --git a/.github/scripts/build-and-push.sh b/.github/scripts/build-and-push.sh new file mode 100644 index 0000000..47f2fc2 --- /dev/null +++ b/.github/scripts/build-and-push.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Script to build and push Docker images for the singleuser-release workflow + +# Build the Docker image +docker compose build + +# Get the tag stub from docker-compose.yaml +TAG_STUB=$(eval "echo \"$(cat docker-compose.yaml | grep image: | awk '{ print $2 }' | grep singleuser)\"") + +# Create target and latest tags +TARGET_TAG="${TAG_STUB}-$(git rev-parse --short HEAD)" +LATEST_TAG="${TAG_STUB}-latest" + +# Output tags for GitHub Actions +echo "target_tag=${TARGET_TAG}" >> $GITHUB_OUTPUT +echo "latest_tag=${LATEST_TAG}" >> $GITHUB_OUTPUT + +# Tag and push the images +docker tag $TAG_STUB $TARGET_TAG +docker tag $TAG_STUB $LATEST_TAG +docker push $TARGET_TAG +docker push $LATEST_TAG diff --git a/.github/workflows/singleuser-release.yaml b/.github/workflows/singleuser-release.yaml index 60df0e4..528718f 100644 --- a/.github/workflows/singleuser-release.yaml +++ b/.github/workflows/singleuser-release.yaml @@ -2,6 +2,14 @@ name: singleuser-release on: workflow_dispatch: inputs: + dockerfile_name: + description: 'Dockerfile to use for the build' + required: true + default: 'Dockerfile' + type: choice + options: + - 'Dockerfile' + - 'IDE.Dockerfile' jupyterhub_version: required: true default: '4.0.2' @@ -11,7 +19,8 @@ on: - '4.0.2' jobs: - release-image: + release-image-matrix: + if: github.event.inputs.dockerfile_name == 'Dockerfile' runs-on: ubuntu-latest strategy: matrix: @@ -24,6 +33,7 @@ jobs: env: PYTHON_VERSION: ${{matrix.py_ver}} JUPYTERHUB_VERSION: ${{github.event.inputs.jupyterhub_version}} + DOCKERFILE_NAME: ${{github.event.inputs.dockerfile_name}} steps: - uses: actions/checkout@v4 with: @@ -34,16 +44,31 @@ jobs: with: username: ${{secrets.DOCKERHUB_USERNAME}} password: ${{secrets.DOCKERHUB_TOKEN}} - - run: | + - name: Build and push Docker image + id: docker-build + run: | cd singleuser - docker compose build - export TAG_STUB=$(eval "echo \"$(cat docker-compose.yaml | grep image: | awk '{ print $2 }' | grep singleuser)\"") - export TARGET_TAG="${TAG_STUB}-$(git rev-parse --short HEAD)" - export LATEST_TAG="${TAG_STUB}-latest" - echo "target_tag=${TARGET_TAG}" >> $GITHUB_OUTPUT - echo "latest_tag=${LATEST_TAG}" >> $GITHUB_OUTPUT - docker tag $TAG_STUB $TARGET_TAG - docker tag $TAG_STUB $LATEST_TAG - docker push $TARGET_TAG - docker push $LATEST_TAG + bash ../.github/scripts/build-and-push.sh + + release-image-ide: + if: github.event.inputs.dockerfile_name == 'IDE.Dockerfile' + runs-on: ubuntu-latest + env: + PYTHON_VERSION: '3.11' + JUPYTERHUB_VERSION: ${{github.event.inputs.jupyterhub_version}} + DOCKERFILE_NAME: ${{github.event.inputs.dockerfile_name}} + steps: + - uses: actions/checkout@v4 + with: + ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} + submodules: true + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{secrets.DOCKERHUB_USERNAME}} + password: ${{secrets.DOCKERHUB_TOKEN}} + - name: Build and push Docker image id: docker-build + run: | + cd singleuser + bash ../.github/scripts/build-and-push.sh diff --git a/.gitignore b/.gitignore index 2eea525..e58e5a2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.env \ No newline at end of file +.env +.idea diff --git a/singleuser/IDE.Dockerfile b/singleuser/IDE.Dockerfile new file mode 100644 index 0000000..5769854 --- /dev/null +++ b/singleuser/IDE.Dockerfile @@ -0,0 +1,73 @@ +ARG JUPYTERHUB_VERSION +FROM quay.io/jupyter/minimal-notebook:hub-${JUPYTERHUB_VERSION} + +ARG PYTHON_VERSION +RUN if [ "$(python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))')" != "${PYTHON_VERSION}" ]; then \ + echo "Installing python ${PYTHON_VERSION}.." && conda install --quiet --no-pin --yes python=${PYTHON_VERSION}; \ + else echo "Python version matching"; \ + fi + +USER root +COPY ./config /tmp/config +COPY ./ipython-datajoint-creds-updater /tmp/ipython-datajoint-creds-updater +RUN \ + # Install dependencies: apt + bash /tmp/config/apt_install.sh \ + # Add startup hook + && cp /tmp/config/before_start_hook.sh /usr/local/bin/before-notebook.d/ \ + && chmod +x /usr/local/bin/before-notebook.d/before_start_hook.sh \ + # Add jupyter*config*.py + && cp /tmp/config/jupyter*config*.py /etc/jupyter/ \ + && mkdir /etc/jupyter/labconfig/ \ + && cp /tmp/config/*.json /etc/jupyter/labconfig/ \ + # Autoload extension in IPython kernel config + && mkdir -p /etc/ipython \ + && echo "c.IPKernelApp.extensions = ['ipython_datajoint_creds_updater.extension']" > /etc/ipython/ipython_kernel_config.py + +USER $NB_UID +RUN \ + # remove default work directory + [ -d "/home/jovyan/work" ] && rm -r /home/jovyan/work \ + # Install dependencies: pip + && pip install /tmp/ipython-datajoint-creds-updater -r /tmp/config/pip_requirements.txt + + +# CODE-SERVER INSTALLATION +ARG code_server_proxy_wheel="jupyter_codeserver_proxy-1.0b3-py3-none-any.whl" +ARG JUPYTER_CODESERVER_PROXY_DIR="jupyter_codeserver_proxy_dir" + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# prerequisites ---- +RUN apt-get update && apt-get install -yq --no-install-recommends \ + curl \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# creation of the jupyter_codeserver_proxy package ---- +WORKDIR $HOME/$JUPYTER_CODESERVER_PROXY_DIR +COPY setup.py /${HOME}/${JUPYTER_CODESERVER_PROXY_DIR} +COPY jupyter_codeserver_proxy /${HOME}/${JUPYTER_CODESERVER_PROXY_DIR}/jupyter_codeserver_proxy +RUN python setup.py bdist_wheel +WORKDIR ../.. + +# code-server install ---- +RUN wget -qO- https://code-server.dev/install.sh | sh && \ + rm -rf "${HOME}/.cache" + +# jupyter-server-proxy + code-server proxy install ---- +RUN conda install --quiet --yes \ + 'jupyter-server-proxy' && \ + jupyter labextension install @jupyterlab/server-proxy --version latest && \ + pip install --quiet --no-cache-dir "${HOME}/${JUPYTER_CODESERVER_PROXY_DIR}/dist/${code_server_proxy_wheel}" && \ + rm -rf "${HOME}/${JUPYTER_CODESERVER_PROXY_DIR}" && \ + jupyter lab clean -y && \ + npm cache clean --force && \ + conda clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +WORKDIR $HOME +RUN touch .gitconfig +USER $NB_UID diff --git a/singleuser/config/before_start_hook.sh b/singleuser/config/before_start_hook.sh index b03bc5e..cfdc8c6 100755 --- a/singleuser/config/before_start_hook.sh +++ b/singleuser/config/before_start_hook.sh @@ -11,11 +11,20 @@ yq '.properties.defaultViewers.default = {"markdown":"Markdown Preview"}' \ if [[ ! -z "${DJLABHUB_REPO}" ]]; then REPO_NAME=$(basename $DJLABHUB_REPO | sed 's/.git//') - echo "INFO::Cloning repo $DJLABHUB_REPO" - git clone $DJLABHUB_REPO $HOME/$REPO_NAME || echo "WARNING::Failed to clone ${DJLABHUB_REPO}. Continuing..." - if [[ ! -z "${DJLABHUB_REPO_BRANCH}" ]]; then - echo "INFO::Switch to branch $DJLABHUB_REPO_BRANCH" - git -C $HOME/$REPO_NAME switch $DJLABHUB_REPO_BRANCH || echo "WARNING::Failed to checkout branch ${DJLABHUB_REPO_BRANCH}. Continuing..." + # We only clone if the destination directory is empty. git clone will create the directory if it does not exist. + if [ -z "$(ls -A "$HOME/$REPO_NAME" 2>/dev/null)" ]; then + echo "INFO::Cloning repo $DJLABHUB_REPO" + git clone $DJLABHUB_REPO $HOME/$REPO_NAME || echo "WARNING::Failed to clone ${DJLABHUB_REPO}. Continuing..." + echo "INFO::Changing ownership of $HOME/$REPO_NAME to ${NB_USER}:${NB_GID}" + chown -R "${NB_USER}:${NB_GID}" "$HOME/$REPO_NAME" + if [[ ! -z "${DJLABHUB_REPO_BRANCH}" ]]; then + echo "INFO::Switch to branch $DJLABHUB_REPO_BRANCH" + git -C $HOME/$REPO_NAME switch $DJLABHUB_REPO_BRANCH || echo "WARNING::Failed to checkout branch ${DJLABHUB_REPO_BRANCH}. Continuing..." + fi + else + echo "INFO::Directory $HOME/$REPO_NAME already exists and is not empty. Skipping clone." + echo "INFO::Changing ownership of $HOME/$REPO_NAME to ${NB_USER}:${NB_GID}" + chown -R "${NB_USER}:${NB_GID}" "$HOME/$REPO_NAME" fi if [[ $DJLABHUB_REPO_INSTALL == "TRUE" ]]; then diff --git a/singleuser/docker-compose.yaml b/singleuser/docker-compose.yaml index 75606d6..0ebcf4f 100644 --- a/singleuser/docker-compose.yaml +++ b/singleuser/docker-compose.yaml @@ -2,7 +2,7 @@ services: singleuser: build: context: . - dockerfile: Dockerfile + dockerfile: ${DOCKERFILE_NAME:-Dockerfile} args: - JUPYTERHUB_VERSION - PYTHON_VERSION diff --git a/singleuser/jupyter_codeserver_proxy/__init__.py b/singleuser/jupyter_codeserver_proxy/__init__.py new file mode 100644 index 0000000..cfd9600 --- /dev/null +++ b/singleuser/jupyter_codeserver_proxy/__init__.py @@ -0,0 +1,45 @@ +""" +Return config on servers to start for codeserver + +See https://jupyter-server-proxy.readthedocs.io/en/latest/server-process.html +for more information. +""" +import os +import shutil +import subprocess +from .helpers import setup_database_password + +def setup_codeserver(): + # Make sure codeserver is in $PATH + def _codeserver_command(port): + full_path = shutil.which('code-server') + if not full_path: + raise FileNotFoundError('Can not find code-server in $PATH') + working_dir = os.getenv("CODE_WORKINGDIR", None) + if working_dir is None: + working_dir = os.getenv("JUPYTER_SERVER_ROOT", ".") + + setup_database_password() + + dj_user = os.getenv("DJ_USER", None) + dj_user_email = os.getenv("DJ_USER_EMAIL", None) + + # # Run Git commands if user information is provided + if dj_user and dj_user_email: + try: + subprocess.run(['git', 'config', '--global', 'user.name', dj_user], check=True) + subprocess.run(['git', 'config', '--global', 'user.email', dj_user_email], check=True) + except subprocess.CalledProcessError as e: + print(f"Error setting Git user: {e}") + + return [full_path, f'--port={port}', "--auth", "none", working_dir] + + return { + 'command': _codeserver_command, + 'timeout': 20, + 'launcher_entry': { + 'title': 'VS Code IDE', + 'icon_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'icons', 'vscode.svg') + } + } diff --git a/singleuser/jupyter_codeserver_proxy/helpers.py b/singleuser/jupyter_codeserver_proxy/helpers.py new file mode 100644 index 0000000..00273f7 --- /dev/null +++ b/singleuser/jupyter_codeserver_proxy/helpers.py @@ -0,0 +1,116 @@ +import os +import time +import logging +import requests +import jwt +from functools import lru_cache +from typing import Dict, Tuple, Optional +from packaging import version +from datajoint.settings import config as dj_config +from pydantic import ValidationError +from .settings import settings, JHubConfig + +Token = Optional[str] + +@lru_cache(maxsize=1) +def get_token_from_jhub_auth_state( + api_url: str, token: str, user: str, logger=None, ttl_hash=None +) -> Tuple[Token, Token]: + """ + Get the access and refresh tokens from the `auth_state` object returned + from the JupyterHub API. This function is cached for a configurable number + of seconds, defined by `ttl_hash`. If `ttl_hash` is not provided, the + function cache will never expire. + """ + del ttl_hash + logger = logger or logging.getLogger(__name__) + url = api_url + f"/users/{user}" + try: + resp = requests.get( + url, + headers={ + "Authorization": f"token {token}", + }, + timeout=(0.5, 0.5), + verify=False, + ) + except (requests.RequestException,) as e: + logger.error(f"Request to {url=} failed with {e}") + return "", "" + logger.debug(f"Request to {url=} responded with {resp=}") + if resp.status_code >= 300: + logger.error( + f"Request to {url=} failed returning status code {resp.status_code}" + ) + return "", "" + auth_state = resp.json().get("auth_state", dict()) + return auth_state.get("access_token"), auth_state.get("refresh_token") + +def decode_token(tok: str) -> Dict: + """ + Decode a JWT token using PyJWT. + """ + if version.parse(jwt.__version__) < version.parse("2.0.0"): + kwargs = dict(verify=False) + else: + kwargs = dict(options=dict(verify_signature=False)) + return jwt.decode(tok, algorithms=["RS256"], **kwargs) + +def check_token_expiry(tok: str, logger=None, token_type: str = "access") -> int: + """ + Returns number of seconds until token expiry. + """ + logger = logger or logging.getLogger(__name__) + tok: dict = decode_token(tok) + if "exp" not in tok: + logger.error(f"{token_type.title()} token has no expiry time.") + return 0 + now = int(time.time()) + sec_remaining = tok["exp"] - now + logger.info( + f"{token_type.title()} token has {sec_remaining:0.2f} seconds until expiry" + ) + return sec_remaining + +def get_ttl_hash(ttl_seconds=60) -> int: + """ + Return the same value within `ttl_seconds` time period. + Adapted from https://stackoverflow.com/a/55900800 + """ + return round(time.time() / float(ttl_seconds)) + +def setup_database_password(logger=None): + logger = logger or logging.getLogger(__name__) + try: + cfg = JHubConfig() + except ValidationError as e: + logger.warn(f"Invalid JHubConfig: {e}") + return + access_token, refresh_token = get_token_from_jhub_auth_state( + cfg.api_url, + cfg.token, + cfg.user, + logger=logger, + ttl_hash=get_ttl_hash(ttl_seconds=settings.auth_state_response_ttl_seconds), + ) + if settings.debug: + logger.debug(f"{get_token_from_jhub_auth_state.cache_info()=}") + expiry: int = check_token_expiry( + access_token, logger=logger, token_type="access" + ) + logger.debug( + f"Access token ending with {access_token[-7:]} expires in {expiry} seconds" + ) + refresh_expiry: int = check_token_expiry( + refresh_token, logger=logger, token_type="refresh" + ) + if refresh_expiry > 0: + dj_config["database.password"] = access_token + os.environ['DJ_PASSWORD'] = access_token + if settings.debug: + logger.debug( + f"Refresh token ending with {refresh_token - 7:]} " + f"expires in {refresh_expiry} seconds" + ) + elif settings.warn_on_expired_refresh: + logger.warn(settings.expired_refresh_warn_message) diff --git a/singleuser/jupyter_codeserver_proxy/icons/vscode.svg b/singleuser/jupyter_codeserver_proxy/icons/vscode.svg new file mode 100644 index 0000000..ae05f79 --- /dev/null +++ b/singleuser/jupyter_codeserver_proxy/icons/vscode.svg @@ -0,0 +1,368 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + diff --git a/singleuser/jupyter_codeserver_proxy/settings.py b/singleuser/jupyter_codeserver_proxy/settings.py new file mode 100644 index 0000000..c720513 --- /dev/null +++ b/singleuser/jupyter_codeserver_proxy/settings.py @@ -0,0 +1,37 @@ +import logging +from pydantic import Field +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + log_level: int = Field( + default=logging.DEBUG, validation_alias="creds_updater_log_level" + ) + debug: bool = Field(default=False, validation_alias="creds_updater_debug") + auth_state_response_ttl_seconds: int = Field( + default=60, validation_alias="creds_updater_ttl_seconds" + ) + warn_on_expired_refresh: bool = Field( + default=True, validation_alias="creds_updater_warn_on_expired_refresh" + ) + expired_refresh_warn_message: str = Field( + default=( + "Unable to set DataJoint credentials automatically. Please reload " + "Jupyter (e.g. by refreshing the page and restarting the kernel) " + "and try again, or manually " + "set the value of `datajoint.config['database.password']` " + "in the notebook (see " + "https://datajoint.com/docs/core/datajoint-python/0.14/client/credentials/ " + "for details)." + ), + validation_alias="creds_updater_expired_refresh_warn_message", + ) + + +class JHubConfig(BaseSettings): + api_url: str = Field(validation_alias="jupyterhub_api_url") + token: str = Field(validation_alias="jupyterhub_api_token") + user: str = Field(validation_alias="jupyterhub_user") + + +settings = Settings() diff --git a/singleuser/setup.py b/singleuser/setup.py new file mode 100644 index 0000000..f6c2c57 --- /dev/null +++ b/singleuser/setup.py @@ -0,0 +1,24 @@ +import setuptools + + +setuptools.setup( + name="jupyter-codeserver-proxy", + version="1.0b3", + url="https://github.com/dirkcgrunwald/jupyter-codeserver-proxy.git", + author="Dirk Grunwald based on Project Jupyter Contributors", + description="grunwald@colorado.edu", + packages=setuptools.find_packages(), + keywords=["Jupyter"], + classifiers=["Framework :: Jupyter"], + install_requires=[ + "jupyter-server-proxy" + ], + entry_points={ + "jupyter_serverproxy_servers": [ + "codeserver = jupyter_codeserver_proxy:setup_codeserver", + ] + }, + package_data={ + "jupyter_codeserver_proxy": ["icons/*"], + }, +)