From a0abc3d0ec85236a8ae7968c40cdd22837b354d2 Mon Sep 17 00:00:00 2001 From: Nate Cybulski Date: Thu, 17 Apr 2025 15:41:54 -0700 Subject: [PATCH 1/3] create workbench-jupyter-docker devcontainer to support docker workflows --- .../.devcontainer.json | 49 +++++++++++++++++++ src/workbench-jupyter-docker/README.md | 17 +++++++ .../devcontainer-template.json | 23 +++++++++ .../docker-compose.yaml | 28 +++++++++++ startupscript/setup-docker.sh | 45 +++++++++++++++++ 5 files changed, 162 insertions(+) create mode 100644 src/workbench-jupyter-docker/.devcontainer.json create mode 100644 src/workbench-jupyter-docker/README.md create mode 100644 src/workbench-jupyter-docker/devcontainer-template.json create mode 100644 src/workbench-jupyter-docker/docker-compose.yaml create mode 100755 startupscript/setup-docker.sh diff --git a/src/workbench-jupyter-docker/.devcontainer.json b/src/workbench-jupyter-docker/.devcontainer.json new file mode 100644 index 00000000..52bb29b5 --- /dev/null +++ b/src/workbench-jupyter-docker/.devcontainer.json @@ -0,0 +1,49 @@ +{ + "name": "Workbench JupyterLab with docker support devcontainer template", + "dockerComposeFile": "docker-compose.yaml", + "service": "app", + "shutdownAction": "none", + "workspaceFolder": "/workspace", + "initializeCommand": "DOCKER_GID=`getent group docker | cut -d: -f3` && echo \"DOCKER_GID=${DOCKER_GID}\" > .env", + "postCreateCommand": "./startupscript/post-startup.sh jupyter /home/jupyter ${templateOption:cloud} ${templateOption:login} && ./startupscript/setup-docker.sh", + // re-mount bucket files on container start up + "postStartCommand": [ + "./startupscript/remount-on-restart.sh", + "jupyter", + "/home/jupyter", + "${templateOption:cloud}", + "${templateOption:login}" + ], + "remoteUser": "root", + "customizations": { + "workbench": { + "opens": { + "extensions": [ + // Source + ".ipynb", + ".R", + ".py", + // Documents + ".md", + ".html", + ".latex", + ".pdf", + // Images + ".bmp", + ".gif", + ".jpeg", + ".jpg", + ".png", + ".svg", + // Data + ".csv", + ".tsv", + ".json", + ".vl" + ], + "fileUrlSuffix": "/lab/tree/{path}", + "folderUrlSuffix": "/lab/tree/{path}" + } + } + } +} diff --git a/src/workbench-jupyter-docker/README.md b/src/workbench-jupyter-docker/README.md new file mode 100644 index 00000000..7b192102 --- /dev/null +++ b/src/workbench-jupyter-docker/README.md @@ -0,0 +1,17 @@ + +# Custom Workbench JupyterLab App Template (workbench-jupyter-docker) + +A template used to serve the Workbench custom JupyterLab app. + +## Options + +| Options Id | Description | Type | Default Value | +|-----|-----|-----|-----| +| cloud | VM cloud environment | string | gcp | +| login | Whether to log in to workbench CLI | string | false | + + + +--- + +_Note: This file was auto-generated from the [devcontainer-template.json](devcontainer-template.json). Add additional notes to a `NOTES.md`._ diff --git a/src/workbench-jupyter-docker/devcontainer-template.json b/src/workbench-jupyter-docker/devcontainer-template.json new file mode 100644 index 00000000..8897453d --- /dev/null +++ b/src/workbench-jupyter-docker/devcontainer-template.json @@ -0,0 +1,23 @@ +{ + "id": "custom-workbench-jupyter-template", + "description": "A template used to serve the Workbench JupyterLab container image", + "version": "0.0.1", + "name": "Workbench Prebuilt JupyterLab Template", + "documentationURL": "https://github.com/verily-src/workbench-app-devcontainers/tree/master/src/custom-workbench-jupyter-template", + "licenseURL": "https://github.com/verily-src/workbench-app-devcontainers/blob/master/LICENSE", + "options": { + "cloud": { + "type": "string", + "description": "VM cloud environment", + "proposals": ["gcp", "aws"], + "default": "gcp" + }, + "login": { + "type": "string", + "description": "Whether to log in to workbench CLI", + "proposals": ["true", "false"], + "default": "false" + } + }, + "platforms": ["Any"] +} diff --git a/src/workbench-jupyter-docker/docker-compose.yaml b/src/workbench-jupyter-docker/docker-compose.yaml new file mode 100644 index 00000000..b491fc9a --- /dev/null +++ b/src/workbench-jupyter-docker/docker-compose.yaml @@ -0,0 +1,28 @@ +version: "2.4" +services: + app: + container_name: "application-server" + image: "us-central1-docker.pkg.dev/verily-workbench-public/apps/workbench-jupyter:latest" + user: "jupyter:${DOCKER_GID}" + restart: always + volumes: + - .:/workspace:cached + # mount Host machine's docker.sock to container's docker.sock + - /var/run/docker.sock:/var/run/docker.sock + # mount Host machine's /etc/group to container's /etc/host-group + - /etc/group:/etc/host-group + # mount Host machine's default docker config dir to container's jupyter user docker config dir + - /etc/docker:/home/jupyter/.docker + ports: + - "8888:8888" + networks: + - app-network + cap_add: + - SYS_ADMIN + devices: + - /dev/fuse + security_opt: + - apparmor:unconfined +networks: + app-network: + external: true diff --git a/startupscript/setup-docker.sh b/startupscript/setup-docker.sh new file mode 100755 index 00000000..16a0f085 --- /dev/null +++ b/startupscript/setup-docker.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# setup-docker.sh +# Installs the docker CLI and configure the host machine's docker group to +# include the app container user. + +set -o errexit +set -o nounset +set -o pipefail +set -o xtrace + +###################### +# Install Docker CLI # +###################### + +mkdir -p /etc/apt/keyrings && \ +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \ +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \ + +apt-get update && \ +apt-get install -y docker-ce-cli && \ + +########################################### +# Add container user to host docker group # +########################################### + +# Add jupyter user to the host's docker group +sed "/^docker:/ s/$/,jupyter/" /etc/host-group > /tmp/host-group.modified +tee /etc/host-group < /tmp/host-group.modified > /dev/null + +# create a matching docker group in the container and add the user to it +DOCKER_GID=$(grep '^docker:' "/etc/host-group" | cut -d: -f3) +if ! getent group docker; then + groupadd -g $DOCKER_GID docker +fi +usermod -aG docker jupyter + +########################### +# Configuring docker auth # +########################### + +# Give user write permissions to the mounted docker config directory +chown -R jupyter /home/jupyter/.docker + +# Login to docker with gcloud credentials (needs to be re-run every 30 min if needed) +sudo -u jupyter /bin/bash -c "docker login -u oauth2accesstoken -p $(gcloud auth print-access-token) https://us-central1-docker.pkg.dev" From 9eaa8baea7556155388022def1adbc1ca8456350 Mon Sep 17 00:00:00 2001 From: Nate Cybulski Date: Thu, 17 Apr 2025 15:45:50 -0700 Subject: [PATCH 2/3] lint --- startupscript/setup-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/startupscript/setup-docker.sh b/startupscript/setup-docker.sh index 16a0f085..5db7eaf6 100755 --- a/startupscript/setup-docker.sh +++ b/startupscript/setup-docker.sh @@ -30,7 +30,7 @@ tee /etc/host-group < /tmp/host-group.modified > /dev/null # create a matching docker group in the container and add the user to it DOCKER_GID=$(grep '^docker:' "/etc/host-group" | cut -d: -f3) if ! getent group docker; then - groupadd -g $DOCKER_GID docker + groupadd -g "$DOCKER_GID" docker fi usermod -aG docker jupyter From 2633e413600a72523b03d1c92791d94c9c949d13 Mon Sep 17 00:00:00 2001 From: Nate Cybulski Date: Tue, 22 Apr 2025 10:25:07 -0700 Subject: [PATCH 3/3] small comment --- src/workbench-jupyter-docker/.devcontainer.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/workbench-jupyter-docker/.devcontainer.json b/src/workbench-jupyter-docker/.devcontainer.json index 52bb29b5..07d618b6 100644 --- a/src/workbench-jupyter-docker/.devcontainer.json +++ b/src/workbench-jupyter-docker/.devcontainer.json @@ -4,6 +4,8 @@ "service": "app", "shutdownAction": "none", "workspaceFolder": "/workspace", + // Get the host's docker group ID and propagate it into the .env file, which + // allows it to be used within docker-compose.yaml. "initializeCommand": "DOCKER_GID=`getent group docker | cut -d: -f3` && echo \"DOCKER_GID=${DOCKER_GID}\" > .env", "postCreateCommand": "./startupscript/post-startup.sh jupyter /home/jupyter ${templateOption:cloud} ${templateOption:login} && ./startupscript/setup-docker.sh", // re-mount bucket files on container start up