Feature/wget integration

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
-        exclude: traefik/*.yml|helm/templates/.*
+        exclude: traefik/*.yml|helm/templates/.*|helm/deploy/.*|helm/values.yaml
 
   # Back-end
   # ------------------------------------------------------------------------------

backend/Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.11-slim
 
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONUNBUFFERED=1
 
 RUN apt-get update \
     # dependencies for building Python packages
@@ -13,8 +13,8 @@ RUN apt-get update \
     && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
     && rm -rf /var/lib/apt/lists/*
 
-RUN addgroup --system django \
-    && adduser --system --ingroup django django
+RUN groupadd --system --gid 1000 django && \
+    useradd --system --create-home --uid 1000 --gid django django
 
 # Requirements are installed here to ensure they will be cached.
 COPY requirements /requirements

backend/config/settings/site_specific.py

@@ -24,9 +24,37 @@ class MetagridBackendSettings(BaseSettings):
         examples=["https://api.stac.esgf-west.org/"],
     )
 
-    WGET_URL: str = Field(
-        description="The URL at which the ESG-Search wget endpoint can be reached.",
-        examples=["https://esgf-node.llnl.gov/esg-search/wget"],
+    # Expand the number of fields allowed for wget API payloads (Django's DATA_UPLOAD_MAX_NUMBER_FIELDS)
+    DATA_UPLOAD_MAX_NUMBER_FIELDS: int = Field(
+        default=1024,
+        description="Maximum number of form fields allowed in a single upload. Useful for large wget payloads.",
+        examples=[1024],
+    )
+
+    # === wget related settings ===
+    GLOBUS_PUBLIC_INDEX_ENDPOINT_ID: str = Field(
+        default="a8ef4320-9e5a-4793-837b-c45161ca1845",
+        description="The Globus index ID for the public ESGF2 data.",
+        examples=["a8ef4320-9e5a-4793-837b-c45161ca1845"],
+    )
+
+    WGET_SCRIPT_FILE_DEFAULT_LIMIT: int = Field(
+        default=1000,
+        description="Default limit on the number of files allowed in a generated wget script.",
+        examples=[1000],
+    )
+
+    WGET_SCRIPT_FILE_MAX_LIMIT: int = Field(
+        default=100000,
+        description="Maximum number of files allowed in a generated wget script.",
+        examples=[100000],
+    )
+
+    # Maximum length for facet values used in the wget directory structure
+    WGET_MAX_DIR_LENGTH: int = Field(
+        default=50,
+        description="Maximum character length for facet values when creating directory names for wget downloads.",
+        examples=[50],
     )
 
     KEYCLOAK_CLIENT_ID: str = Field(
@@ -57,6 +85,12 @@ class MetagridBackendSettings(BaseSettings):
         "Reference: <https://docs.djangoproject.com/en/5.1/ref/settings/#admins>",
     )
 
+    DATABASE_URL: str = Field(
+        default="postgresql://postgres:postgres@postgres:5432/postgres",
+        examples=["postgresql://postgres:postgres@postgres:5432/postgres"],
+        description="The database connection URL for the Metagrid backend database.",
+    )
+
     SOCIAL_AUTH_GLOBUS_KEY: str = Field(
         examples=["94c44808-9efd-4236-bffd-1185b1071736"],
         description="The `Client UUID` obtained by registering a `portal, science gateway, or other application you host` with Globus at <https://app.globus.org/settings/developers>",

backend/config/settings/static.py

@@ -2,19 +2,24 @@
 from typing import Any, Optional, Sequence
 
 import environ
-from pydantic import BaseModel, Field
+from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 env = environ.Env()
 
-ROOT_DIR = (
-    environ.Path(__file__) - 3
-)  # (config/settings/django.py - 3 = metagrid/)
+# project root (config/settings/static.py - 3 = metagrid/)
+ROOT_DIR = environ.Path(__file__) - 3
 
-# parses DATABASE_URL environment variable
-DATABASES = env.db_url()
+TEMPLATE_DIR = ROOT_DIR("metagrid", "wget", "templates")
+
+# Parse DATABASE_URL environment variable; default to an in-memory sqlite DB for tests/local runs.
+# This prevents "Set the DATABASE_URL environment variable" errors when none is provided.
+DATABASES = env.db_url(
+    default="postgresql://postgres:postgres@postgres:5432/postgres"
+)
 DATABASES.update(ATOMIC_REQUESTS=True)
 
+
 class DjangoStaticSettings(BaseSettings):
     """Django settings that do not vary by site"""
 
@@ -101,7 +106,7 @@ class DjangoStaticSettings(BaseSettings):
     # https://docs.djangoproject.com/en/dev/ref/settings/#static-url
     STATIC_URL: str = "/static/"
     # https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS
-    STATICFILES_DIRS: Sequence[str] = []
+    STATICFILES_DIRS: Sequence[str] = [TEMPLATE_DIR]
     # https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-finders
     STATICFILES_FINDERS: Sequence[str] = [
         "django.contrib.staticfiles.finders.FileSystemFinder",
@@ -122,7 +127,7 @@ class DjangoStaticSettings(BaseSettings):
         {
             "BACKEND": "django.template.backends.django.DjangoTemplates",
             "DIRS": STATICFILES_DIRS,
-            "APP_DIRS": False,
+            "APP_DIRS": True,
             "OPTIONS": {
                 "context_processors": [
                     "django.template.context_processors.debug",

backend/config/urls.py

@@ -24,7 +24,6 @@
     do_search,
     do_stac_search,
     do_status,
-    do_wget,
     fetch_stac_aggregations,
     get_frontend_config,
     get_temp_storage,
@@ -34,6 +33,7 @@
 from metagrid.observability.views import liveness, readiness
 from metagrid.projects.views import ProjectsViewSet
 from metagrid.users.views import UserCreateViewSet, UserViewSet
+from metagrid.wget.views import do_wget
 
 router = DefaultRouter()
 router.register(r"users", UserViewSet)

backend/metagrid/api_proxy/tests/test_views.py

@@ -79,18 +79,6 @@ def test_globus_auth_logout(self):
         response = self.client.get(url)
         self.assertEqual(response.status_code, status.HTTP_302_FOUND)
 
-    @responses.activate
-    def test_wget(self):
-        url = reverse("do-wget")
-        responses.get(settings.WGET_URL)
-        response = self.client.get(
-            url,
-            {
-                "dataset_id": "CMIP6.CMIP.IPSL.IPSL-CM6A-LR.abrupt-4xCO2.r12i1p1f1.Amon.n2oglobal.gr.v20191003|esgf-data1.llnl.gov"
-            },
-        )
-        assert response.status_code == status.HTTP_200_OK
-
     @responses.activate
     def test_search(self):
         url = reverse("do-search")

backend/metagrid/api_proxy/views.py

@@ -152,12 +152,6 @@ def do_status(request):
         return HttpResponseBadRequest(resp.text)
 
 
-@require_http_methods(["GET", "POST"])
-@csrf_exempt
-def do_wget(request):
-    return do_request(request, settings.WGET_URL, True)
-
-
 def do_post(request, urlbase):
     """Helper function to handle POST requests."""
     if request.method != "POST":  # pragma: no cover