[WIP] feat: add xblocks at runtime without rebuilding image

changelog.d/20250806_160115_muhammad.labeeb_install_xblocks_at_runtime.md

@@ -0,0 +1,2 @@
+- [Feature] Install python packages at runtime without rebuilding image. (by @mlabeeb03)
+- [Improvement] Add a `do` command to manually run lms/cms migrations. (by @mlabeeb03)

tutor/commands/cli.py

@@ -65,7 +65,7 @@ def ensure_plugins_enabled(self, ctx: click.Context) -> None:
         """
         We enable plugins as soon as possible to have access to commands.
         """
-        if not "root" in ctx.params:
+        if "root" not in ctx.params:
             # When generating docs, this function is called with empty args.
             # That's ok, we just ignore it.
             return

tutor/commands/jobs.py

@@ -540,6 +540,58 @@ def do_callback(service_commands: t.Iterable[tuple[str, str]]) -> None:
         runner.run_task_from_str(service, command)
 
 
+@click.command(
+    help="Build all live dependencies, zip them and upload to storage backend"
+)
+@click.pass_obj
+def build_live_dependencies(context: Context) -> t.Iterable[tuple[str, str]]:
+    """
+    Build the live dependencies and upload using Django's storage API.
+    You need to update the `LIVE_DEPENDENCIES` variable in the config file to add/remove packages.
+    """
+    config = tutor_config.load(context.root)
+    all_packages = " ".join(
+        package for package in t.cast(list[str], config["LIVE_DEPENDENCIES"])
+    )
+
+    script = f"""
+    pip install \
+    --prefix=/openedx/live-dependencies/deps \
+    {all_packages} \
+    && python3 -c '
+import os, shutil, tempfile
+from django.core.files.storage import storages
+from django.core.files.base import File
+
+DEPS_DIR = "/openedx/live-dependencies/deps"
+DEPS_KEY = "deps.zip"
+
+with tempfile.TemporaryDirectory(prefix="tutor-livedeps-") as zip_dir:
+    base = os.path.join(zip_dir, DEPS_KEY)
+    archive_path = shutil.make_archive(base[:-4], format="zip", root_dir=DEPS_DIR)
+
+    with open(archive_path, "rb") as f:
+        # TODO Use a separate storage for live dependencies
+        storages["default"].save(DEPS_KEY, File(f))
+'
+    """
+
+    yield ("lms", script)
+
+
+@click.command(help="Run migrations for an app")
+@click.argument("package", required=False)
+def run_migrations(package: str) -> t.Iterable[tuple[str, str]]:
+    """
+    Run migrations for a specific app or all apps.
+    """
+    script = "./manage.py lms migrate "
+    if package:
+        script += package
+
+    yield ("lms", script)
+
+
 hooks.Filters.CLI_DO_COMMANDS.add_items(
     [
         convert_mysql_utf8mb4_charset,
@@ -551,5 +603,7 @@ def do_callback(service_commands: t.Iterable[tuple[str, str]]) -> None:
         settheme,
         sqlshell,
         update_mysql_authentication_plugin,
+        build_live_dependencies,
+        run_migrations,
     ]
 )

tutor/templates/build/openedx/Dockerfile

@@ -179,6 +179,7 @@ ENV VIRTUAL_ENV=/openedx/venv/
 ENV COMPREHENSIVE_THEME_DIRS=/openedx/themes
 ENV STATIC_ROOT_LMS=/openedx/staticfiles
 ENV STATIC_ROOT_CMS=/openedx/staticfiles/studio
+ENV PYTHONPATH=/openedx/live-dependencies/deps/lib/python3.11/site-packages
 
 WORKDIR /openedx/edx-platform
 
@@ -313,8 +314,14 @@ ENV UWSGI_WORKERS=2
 # Copy the default uWSGI configuration
 COPY --chown=app:app settings/uwsgi.ini /openedx
 
+# Copy live dependencies scripts and trigger file
+RUN mkdir -p /openedx/live-dependencies
+RUN touch /openedx/live-dependencies/uwsgi_trigger
+COPY --chown=app:app settings/monitor_livedeps.py /openedx/live-dependencies/
+COPY --chown=app:app settings/update_livedeps.py /openedx/live-dependencies/
+
 # Run server
 CMD ["uwsgi", "/openedx/uwsgi.ini"]
 
-{{ patch("openedx-dockerfile-final") }}
 
+{{ patch("openedx-dockerfile-final") }}

tutor/templates/build/openedx/settings/monitor_livedeps.py

@@ -0,0 +1,28 @@
+import datetime
+import os
+import time
+from django.core.files.storage import storages
+
+
+DEPS_DIR = "/openedx/live-dependencies/deps"
+DEPS_KEY = "deps.zip"
+DEPS_ZIP_PATH = DEPS_DIR[:-4] + DEPS_KEY
+TRIGGER_FILE = "/openedx/live-dependencies/uwsgi_trigger"
+
+# TODO Use a separate storage for live dependencies
+storage = storages["default"]
+
+while True:
+    if storage.exists(DEPS_KEY):
+        remote_ts = storage.get_modified_time(DEPS_KEY)
+
+        if os.path.exists(DEPS_ZIP_PATH):
+            local_ts = os.path.getmtime(DEPS_ZIP_PATH)
+            local_ts = datetime.datetime.fromtimestamp(local_ts, tz=datetime.timezone.utc)
+
+            if local_ts < remote_ts:
+                with open(TRIGGER_FILE, "a"):
+                    os.utime(TRIGGER_FILE, None)
+    # TODO What happens if download takes more than 10 seconds? 
+    # This could keep initiating a reload in a loop.
+    time.sleep(10)

tutor/templates/build/openedx/settings/update_livedeps.py

@@ -0,0 +1,23 @@
+import os
+import shutil
+import zipfile
+
+from django.core.files.storage import storages
+
+DEPS_DIR = "/openedx/live-dependencies/deps"
+DEPS_KEY = "deps.zip"
+DEPS_ZIP_PATH = DEPS_DIR[:-4] + DEPS_KEY
+
+# TODO Use a separate storage for live dependencies
+storage = storages["default"]
+
+if storage.exists(DEPS_KEY):
+    if os.path.exists(DEPS_DIR):
+        shutil.rmtree(DEPS_DIR)
+    os.makedirs(DEPS_DIR, exist_ok=True)
+
+    with storage.open(DEPS_KEY, "rb") as remote_f, open(DEPS_ZIP_PATH, "wb") as local_f:
+        shutil.copyfileobj(remote_f, local_f)
+
+    with zipfile.ZipFile(DEPS_ZIP_PATH, "r") as zip_ref:
+        zip_ref.extractall(DEPS_DIR)

tutor/templates/build/openedx/settings/uwsgi.ini

@@ -21,3 +21,9 @@ master = true
 # Clean up settings
 py-call-osafterfork = true
 vacuum = true
+# Run a daemon to detects changes in live dependencies zip file and triggers reloads
+attach-daemon = python /openedx/live-dependencies/monitor_livedeps.py
+# This file is touched by the monitor_livedeps.py script to initiate a reload
+touch-reload = /openedx/live-dependencies/uwsgi_trigger
+# This script downloads persistent packages on server starts/reload
+exec-asap = python /openedx/live-dependencies/update_livedeps.py