use django storages api

Author: mlabeeb03

changelog.d/20250806_160115_muhammad.labeeb_install_xblocks_at_runtime.md

@@ -1 +1,2 @@
-- [Feature] Install python packages at runtime without rebuilding image. (by @mlabeeb03)
+- [Feature] Install python packages at runtime without rebuilding image. (by @mlabeeb03)
+- [Improvement] Add a `do` command to manually run lms/cms migrations. (by @mlabeeb03)

tutor/commands/jobs.py

@@ -540,55 +540,39 @@ def do_callback(service_commands: t.Iterable[tuple[str, str]]) -> None:
         runner.run_task_from_str(service, command)
 
 
-@click.command(help="Build all persistent pip packages and upload to MinIO")
+@click.command(
+    help="Build all live dependencies, zip them and upload to storage backend"
+)
 @click.pass_obj
-def build_packages(context: Context) -> t.Iterable[tuple[str, str]]:
+def build_live_dependencies(context: Context) -> t.Iterable[tuple[str, str]]:
     """
-    Build the persistent pip packages and upload to MinIO.
-    You need to update the `PERSISTENT_PIP_PACKAGES` variable
-    in the config file to add/remove packages.
+    Build the live dependencies and upload using Django's storage API.
+    You need to update the `LIVE_DEPENDENCIES` variable in the config file to add/remove packages.
     """
     config = tutor_config.load(context.root)
     all_packages = " ".join(
-        package for package in t.cast(list[str], config["PERSISTENT_PIP_PACKAGES"])
+        package for package in t.cast(list[str], config["LIVE_DEPENDENCIES"])
     )
 
     script = f"""
     pip install \
-    --prefix=/openedx/persistent-python-packages/deps \
+    --prefix=/openedx/live-dependencies/deps \
     {all_packages} \
     && python3 -c '
-import os, shutil, tempfile, boto3, botocore, datetime, zipfile
-
-DEPS_DIR = "/openedx/persistent-python-packages/deps"
-MINIO_KEY = "deps.zip"
-DEPS_ZIP_PATH = DEPS_DIR[:-4] + MINIO_KEY
-MINIO_BUCKET = "tutor-deps"
-
-s3 = boto3.client(
-    "s3",
-    endpoint_url="http://" + os.environ.get("MINIO_HOST"),
-    aws_access_key_id=os.environ.get("OPENEDX_AWS_ACCESS_KEY"),
-    aws_secret_access_key=os.environ.get("OPENEDX_AWS_SECRET_ACCESS_KEY"),
-)
+import os, shutil, tempfile
+from django.core.files.storage import storages
+from django.core.files.base import File
+
+DEPS_DIR = "/openedx/live-dependencies/deps"
+DEPS_KEY = "deps.zip"
+
+with tempfile.TemporaryDirectory(prefix="tutor-livedeps-") as zip_dir:
+    base = os.path.join(zip_dir, DEPS_KEY)
+    archive_path = shutil.make_archive(base[:-4], format="zip", root_dir=DEPS_DIR)
 
-def _upload_to_minio(local_path):
-    try:
-        s3.head_bucket(Bucket=MINIO_BUCKET)
-    except botocore.exceptions.ClientError:
-        s3.create_bucket(Bucket=MINIO_BUCKET)
-    s3.upload_file(local_path, MINIO_BUCKET, MINIO_KEY)
-    print(f"Uploaded {{local_path}} → MinIO:{{MINIO_BUCKET}}/{{MINIO_KEY}}")
-    os.remove(local_path)
-
-def _make_zip_archive(src_dir):
-    with tempfile.TemporaryDirectory(prefix="tutor-depszip-") as zip_dir:
-        path = os.path.join(zip_dir, "deps.zip")
-        shutil.make_archive(path[:-4], format="zip", root_dir=src_dir)
-        shutil.move(path, DEPS_ZIP_PATH)
-
-_make_zip_archive(DEPS_DIR)
-_upload_to_minio(DEPS_ZIP_PATH)
+    with open(archive_path, "rb") as f:
+        # TODO Use a separate storage for live dependencies
+        storages["default"].save(DEPS_KEY, File(f))
 '
     """
 
@@ -619,7 +603,7 @@ def run_migrations(package: str) -> t.Iterable[tuple[str, str]]:
         settheme,
         sqlshell,
         update_mysql_authentication_plugin,
-        build_packages,
+        build_live_dependencies,
         run_migrations,
     ]
 )

tutor/templates/build/openedx/Dockerfile

@@ -179,7 +179,7 @@ ENV VIRTUAL_ENV=/openedx/venv/
 ENV COMPREHENSIVE_THEME_DIRS=/openedx/themes
 ENV STATIC_ROOT_LMS=/openedx/staticfiles
 ENV STATIC_ROOT_CMS=/openedx/staticfiles/studio
-ENV PYTHONPATH=/openedx/persistent-python-packages/deps/lib/python3.11/site-packages
+ENV PYTHONPATH=/openedx/live-dependencies/deps/lib/python3.11/site-packages
 
 WORKDIR /openedx/edx-platform
 
@@ -314,14 +314,14 @@ ENV UWSGI_WORKERS=2
 # Copy the default uWSGI configuration
 COPY --chown=app:app settings/uwsgi.ini /openedx
 
-# Copy the download script that fetches dependencies from MinIO
-COPY --chown=app:app settings/download_packages_from_minio.py /openedx
+# Copy live dependencies scripts and trigger file
+RUN mkdir -p /openedx/live-dependencies
+RUN touch /openedx/live-dependencies/uwsgi_trigger
+COPY --chown=app:app settings/monitor_livedeps.py /openedx/live-dependencies/
+COPY --chown=app:app settings/update_livedeps.py /openedx/live-dependencies/
 
 # Run server
-CMD sh -c "python /openedx/download_packages_from_minio.py & \
-           while true; do sleep 30; python /openedx/download_packages_from_minio.py; done & \
-           uwsgi /openedx/uwsgi.ini"
+CMD ["uwsgi", "/openedx/uwsgi.ini"]
 
 
 {{ patch("openedx-dockerfile-final") }}
-

tutor/templates/build/openedx/settings/download_packages_from_minio.py

@@ -0,0 +1,28 @@
+import datetime
+import os
+import time
+from django.core.files.storage import storages
+
+
+DEPS_DIR = "/openedx/live-dependencies/deps"
+DEPS_KEY = "deps.zip"
+DEPS_ZIP_PATH = DEPS_DIR[:-4] + DEPS_KEY
+TRIGGER_FILE = "/openedx/live-dependencies/uwsgi_trigger"
+
+# TODO Use a separate storage for live dependencies
+storage = storages["default"]
+
+while True:
+    if storage.exists(DEPS_KEY):
+        remote_ts = storage.get_modified_time(DEPS_KEY)
+
+        if os.path.exists(DEPS_ZIP_PATH):
+            local_ts = os.path.getmtime(DEPS_ZIP_PATH)
+            local_ts = datetime.datetime.fromtimestamp(local_ts, tz=datetime.timezone.utc)
+
+            if local_ts < remote_ts:
+                with open(TRIGGER_FILE, "a"):
+                    os.utime(TRIGGER_FILE, None)
+    # TODO What happens if download takes more than 10 seconds? 
+    # This could keep initiating a reload in a loop.
+    time.sleep(10)

tutor/templates/build/openedx/settings/monitor_livedeps.py

@@ -0,0 +1,23 @@
+import os
+import shutil
+import zipfile
+
+from django.core.files.storage import storages
+
+DEPS_DIR = "/openedx/live-dependencies/deps"
+DEPS_KEY = "deps.zip"
+DEPS_ZIP_PATH = DEPS_DIR[:-4] + DEPS_KEY
+
+# TODO Use a separate storage for live dependencies
+storage = storages["default"]
+
+if storage.exists(DEPS_KEY):
+    if os.path.exists(DEPS_DIR):
+        shutil.rmtree(DEPS_DIR)
+    os.makedirs(DEPS_DIR, exist_ok=True)
+
+    with storage.open(DEPS_KEY, "rb") as remote_f, open(DEPS_ZIP_PATH, "wb") as local_f:
+        shutil.copyfileobj(remote_f, local_f)
+
+    with zipfile.ZipFile(DEPS_ZIP_PATH, "r") as zip_ref:
+        zip_ref.extractall(DEPS_DIR)

tutor/templates/build/openedx/settings/update_livedeps.py

@@ -21,5 +21,9 @@ master = true
 # Clean up settings
 py-call-osafterfork = true
 vacuum = true
-# Touch this file to initiate a reload
-touch-reload = /openedx/persistent-python-packages/.uwsgi_trigger
+# Run a daemon to detects changes in live dependencies zip file and triggers reloads
+attach-daemon = python /openedx/live-dependencies/monitor_livedeps.py
+# This file is touched by the monitor_livedeps.py script to initiate a reload
+touch-reload = /openedx/live-dependencies/uwsgi_trigger
+# This script downloads persistent packages on server starts/reload
+exec-asap = python /openedx/live-dependencies/update_livedeps.py

tutor/templates/build/openedx/settings/uwsgi.ini

@@ -21,17 +21,11 @@ services:
     <<: *openedx-job-service
     environment:
       DJANGO_SETTINGS_MODULE: lms.envs.tutor.development
-      MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
-      OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
-      OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
 
 
   cms-job:
     <<: *openedx-job-service
     environment:
       DJANGO_SETTINGS_MODULE: cms.envs.tutor.development
-      MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
-      OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
-      OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
 
   {{ patch("dev-docker-compose-jobs-services")|indent(2) }}

tutor/templates/dev/docker-compose.jobs.yml

@@ -17,9 +17,6 @@ services:
     command: ./manage.py lms runserver 0.0.0.0:8000
     environment:
         DJANGO_SETTINGS_MODULE: lms.envs.tutor.development
-        MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
-        OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
-        OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
     ports:
         - "8000:8000"
     networks:
@@ -32,9 +29,6 @@ services:
     command: ./manage.py cms runserver 0.0.0.0:8000
     environment:
         DJANGO_SETTINGS_MODULE: cms.envs.tutor.development
-        MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
-        OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
-        OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
     ports:
         - "8001:8000"
 

tutor/templates/dev/docker-compose.yml

@@ -22,9 +22,6 @@ services:
       environment:
         SERVICE_VARIANT: lms
         DJANGO_SETTINGS_MODULE: lms.envs.tutor.production
-        MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
-        OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
-        OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
       volumes:
         - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro
         - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro
@@ -39,9 +36,6 @@ services:
       environment:
         SERVICE_VARIANT: cms
         DJANGO_SETTINGS_MODULE: cms.envs.tutor.production
-        MINIO_HOST: {{ MINIO_HOST | default("minio:9000") }}
-        OPENEDX_AWS_ACCESS_KEY: {{ OPENEDX_AWS_ACCESS_KEY}}
-        OPENEDX_AWS_SECRET_ACCESS_KEY: {{ OPENEDX_AWS_SECRET_ACCESS_KEY }}
       volumes:
         - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro
         - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro