[WIP] feat: add xblocks at runtime without rebuilding image

changelog.d/20250806_160115_muhammad.labeeb_install_xblocks_at_runtime.md

@@ -0,0 +1 @@
+- [Feature] Add Xblocks at runtime without rebuilding image. (by @mlabeeb03)

tutor/commands/cli.py

@@ -16,6 +16,7 @@
 from tutor.commands.local import local
 from tutor.commands.mounts import mounts_command
 from tutor.commands.plugins import plugins_command
+from tutor.commands.packages import packages_command
 
 
 def main() -> None:
@@ -129,6 +130,7 @@ def help_command(context: click.Context) -> None:
         local,
         mounts_command,
         plugins_command,
+        packages_command,
     ]
 )
 

tutor/commands/jobs.py

@@ -13,6 +13,7 @@
 
 from tutor import config as tutor_config
 from tutor import env, fmt, hooks
+from tutor.commands.config import save as config_save_command
 from tutor.commands.context import Context
 from tutor.commands.jobs_utils import (
     create_user_template,
@@ -540,6 +541,85 @@ def do_callback(service_commands: t.Iterable[tuple[str, str]]) -> None:
         runner.run_task_from_str(service, command)
 
 
+@click.command(help="Install a pip package at runtime")
+@click.pass_context
+@click.argument("package")
+def pip_install(context: click.Context, package: str) -> t.Iterable[tuple[str, str]]:
+    """
+    Installs a pip package persistently in the lms and cms container and
+    restarts with uwsgi server in both containers.
+    """
+
+    # TODO Only add package to config if pip install is successful
+    fmt.echo_info(f"Adding {package} to config...")
+    context.invoke(
+        config_save_command,
+        interactive=False,
+        set_vars=[],
+        append_vars=[("PERSISTENT_PIP_PACKAGES", package)],
+        remove_vars=[],
+        unset_vars=[],
+        env_only=False,
+        clean_env=False,
+    )
+
+    script = f"""
+    pip install \
+    --prefix=/mnt/persistent-python-packages \
+    {package} \
+    && echo \"$(date)\" > /mnt/persistent-python-packages/.uwsgi_trigger
+    """
+
+    yield ("lms", script)
+
+
+@click.command(help="Remove a pip package at runtime")
+@click.pass_obj
+@click.pass_context
+@click.argument("package")
+def pip_uninstall(
+    click_context: click.Context, context: Context, package: str
+) -> t.Iterable[tuple[str, str]]:
+    """
+    Deletes the persistently installed pip package along with its dependencies
+    """
+
+    fmt.echo_info(f"Removing {package} from config...")
+    click_context.invoke(
+        config_save_command,
+        interactive=False,
+        set_vars=[],
+        append_vars=[],
+        remove_vars=[("PERSISTENT_PIP_PACKAGES", package)],
+        unset_vars=[],
+        env_only=False,
+        clean_env=False,
+    )
+
+    script = "rm -rf /mnt/persistent-python-packages/lib/"
+    config = tutor_config.load(context.root)
+    values = t.cast(list[str], config["PERSISTENT_PIP_PACKAGES"])
+    remaining_packages = " ".join(values)
+    if len(values) > 0:
+        script += f" && pip install --prefix=/mnt/persistent-python-packages {remaining_packages}"
+    script += ' && echo "$(date)" > /mnt/persistent-python-packages/.uwsgi_trigger'
+
+    yield ("lms", script)
+
+
+@click.command(help="Run migrations for an app")
+@click.argument("package", required=False)
+def run_migrations(package: str) -> t.Iterable[tuple[str, str]]:
+    """
+    Run migrations for a specific app or all apps.
+    """
+    script = "./manage.py lms migrate "
+    if package:
+        script += package
+
+    yield ("lms", script)
+
+
 hooks.Filters.CLI_DO_COMMANDS.add_items(
     [
         convert_mysql_utf8mb4_charset,
@@ -551,5 +631,8 @@ def do_callback(service_commands: t.Iterable[tuple[str, str]]) -> None:
         settheme,
         sqlshell,
         update_mysql_authentication_plugin,
+        pip_install,
+        pip_uninstall,
+        run_migrations,
     ]
 )

tutor/commands/packages.py

@@ -0,0 +1,133 @@
+import click
+import os
+import shlex
+import shutil
+import subprocess
+from typing import cast
+
+from tutor import config as tutor_config
+from tutor.commands.context import Context
+from tutor.commands.config import save as config_save_command
+
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+DEPS_ZIP_PATH = os.path.join(HERE, "deps.zip")
+DEPS_PATH = os.path.join(HERE, "deps/")
+
+
+@click.group(
+    name="packages",
+    short_help="Manage packages",
+)
+def packages_command() -> None:
+    """Custom persistent package manager for Tutor."""
+
+
+@click.command(name="list")
+@click.pass_obj
+def list_command(context: Context) -> None:
+    """
+    List installed persistent packages.
+
+    Entries will be fetched from the `PERSISTENT_PIP_PACKAGES` config setting.
+    """
+    config = tutor_config.load(context.root)
+    packages = [
+        package for package in cast(list[str], config["PERSISTENT_PIP_PACKAGES"])
+    ]
+    print(packages)
+
+
+@click.command(name="build")
+@click.pass_obj
+def build(context: Context) -> None:
+    """Rebuild dependencies from scratch."""
+    config = tutor_config.load(context.root)
+    DEPS = cast(list[str], config["PERSISTENT_PIP_PACKAGES"])
+
+    build_dir = f"{context.root}/data/persistent-python-packages"
+
+    lib_path = os.path.join(build_dir, "lib")
+    bin_path = os.path.join(build_dir, "bin")
+
+    if os.path.exists(lib_path):
+        shutil.rmtree(lib_path)
+    if os.path.exists(bin_path):
+        shutil.rmtree(bin_path)
+
+    _pip_install(DEPS, build_dir)
+
+
+@click.command(name="append")
+@click.pass_obj
+@click.pass_context
+@click.argument("package")
+def append(click_context: click.Context, context: Context, package: str) -> None:
+    """Append a new package to the list."""
+    click_context.invoke(
+        config_save_command,
+        interactive=False,
+        set_vars=[],
+        append_vars=[("PERSISTENT_PIP_PACKAGES", package)],
+        remove_vars=[],
+        unset_vars=[],
+        env_only=False,
+        clean_env=False,
+    )
+
+    build_dir = f"{context.root}/data/persistent-python-packages"
+
+    if os.path.exists(DEPS_ZIP_PATH):
+        shutil.unpack_archive(DEPS_ZIP_PATH, extract_dir=build_dir)
+
+    _pip_install([package], build_dir)
+
+
+@click.command(name="remove")
+@click.pass_context
+@click.argument("package")
+def remove(context: click.Context, package: str) -> None:
+    """Remove a package and rebuild archive from scratch."""
+    context.invoke(
+        config_save_command,
+        interactive=False,
+        set_vars=[],
+        append_vars=[],
+        remove_vars=[("PERSISTENT_PIP_PACKAGES", package)],
+        unset_vars=[],
+        env_only=False,
+        clean_env=False,
+    )
+
+    context.invoke(build)
+
+
+def _pip_install(deps: list[str], prefix_dir: str) -> None:
+    for dep in deps:
+        # We use python3.11 because that's whats used in the Dockerfile
+        check_call(
+            "python3.11",
+            "-m",
+            "pip",
+            "install",
+            "--no-deps",
+            f"--prefix={prefix_dir}",
+            dep,
+        )
+    check_call(f'touch "{prefix_dir}/.uwsgi_trigger"', shell=True)
+
+
+def check_call(*args: str, shell: bool = False) -> None:
+    if shell:
+        command = " ".join(args)
+        print(command)
+        subprocess.check_call(command, shell=True)
+    else:
+        print(shlex.join(args))
+        subprocess.check_call(args)
+
+
+packages_command.add_command(list_command)
+packages_command.add_command(build)
+packages_command.add_command(append)
+packages_command.add_command(remove)

tutor/templates/build/openedx/Dockerfile

@@ -179,6 +179,7 @@ ENV VIRTUAL_ENV=/openedx/venv/
 ENV COMPREHENSIVE_THEME_DIRS=/openedx/themes
 ENV STATIC_ROOT_LMS=/openedx/staticfiles
 ENV STATIC_ROOT_CMS=/openedx/staticfiles/studio
+ENV PYTHONPATH=/mnt/persistent-python-packages/lib/python3.11/site-packages
 
 WORKDIR /openedx/edx-platform
 

tutor/templates/build/openedx/settings/uwsgi.ini

@@ -21,3 +21,5 @@ master = true
 # Clean up settings
 py-call-osafterfork = true
 vacuum = true
+# Touch this file to initiate a reload
+touch-reload = /mnt/persistent-python-packages/.uwsgi_trigger

tutor/templates/dev/docker-compose.jobs.yml

@@ -14,6 +14,8 @@ x-openedx-job-service:
     - ../apps/openedx/config:/openedx/config:ro
     # theme files
     - ../build/openedx/themes:/openedx/themes
+    # third party xblocks and their dependencies
+    - ../../data/persistent-python-packages:/mnt/persistent-python-packages
 
 services:
 

tutor/templates/dev/docker-compose.yml

@@ -6,6 +6,8 @@ x-openedx-service:
   volumes:
     # theme files
     - ../build/openedx/themes:/openedx/themes
+    # third party xblocks and their dependencies
+    - ../../data/persistent-python-packages:/mnt/persistent-python-packages
 
 services:
   permissions:

tutor/templates/local/docker-compose.jobs.yml

@@ -26,6 +26,7 @@ services:
         - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro
         - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro
         - ../apps/openedx/config:/openedx/config:ro
+        - ../../data/persistent-python-packages:/mnt/persistent-python-packages
         {%- for mount in iter_mounts(MOUNTS, "openedx", "lms-job") %}
         - {{ mount }}
         {%- endfor %}
@@ -40,6 +41,7 @@ services:
         - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro
         - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro
         - ../apps/openedx/config:/openedx/config:ro
+        - ../../data/persistent-python-packages:/mnt/persistent-python-packages
         {%- for mount in iter_mounts(MOUNTS, "openedx", "cms-job") %}
         - {{ mount }}
         {%- endfor %}

tutor/templates/local/docker-compose.prod.yml

@@ -41,6 +41,7 @@ services:
       - ../../data/lms:/openedx/data
       - ../../data/openedx-media:/openedx/media
       - ../../data/openedx-media-private:/openedx/media-private
+      - ../../data/persistent-python-packages:/mnt/persistent-python-packages
       {%- for mount in iter_mounts(MOUNTS, "openedx", "lms-worker") %}
       - {{ mount }}
       {%- endfor %}
@@ -62,6 +63,7 @@ services:
       - ../../data/cms:/openedx/data
       - ../../data/openedx-media:/openedx/media
       - ../../data/openedx-media-private:/openedx/media-private
+      - ../../data/persistent-python-packages:/mnt/persistent-python-packages
       {%- for mount in iter_mounts(MOUNTS, "openedx", "cms-worker") %}
       - {{ mount }}
       {%- endfor %}

tutor/templates/local/docker-compose.yml

@@ -107,6 +107,7 @@ services:
       - ../../data/lms:/openedx/data
       - ../../data/openedx-media:/openedx/media
       - ../../data/openedx-media-private:/openedx/media-private
+      - ../../data/persistent-python-packages:/mnt/persistent-python-packages
       {%- for mount in iter_mounts(MOUNTS, "openedx", "lms") %}
       - {{ mount }}
       {%- endfor %}
@@ -134,6 +135,7 @@ services:
       - ../../data/cms:/openedx/data
       - ../../data/openedx-media:/openedx/media
       - ../../data/openedx-media-private:/openedx/media-private
+      - ../../data/persistent-python-packages:/mnt/persistent-python-packages
       {%- for mount in iter_mounts(MOUNTS, "openedx", "cms") %}
       - {{ mount }}
       {%- endfor %}