V1.5.3 release

.github/workflows/backend.yml

@@ -14,20 +14,20 @@ env:
   PGHOST: localhost
   PGPASSWORD: postgres
   PGUSER: postgres
+  DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
   METAGRID_SEARCH_URL: https://esgf-node.llnl.gov/esg-search/search
   METAGRID_WGET_URL: https://esgf-node.llnl.gov/esg-search/wget
   METAGRID_STATUS_URL: https://esgf-node.llnl.gov/proxy/status
   METAGRID_SOCIAL_AUTH_GLOBUS_KEY: ${{ secrets.GLOBUS_KEY }}
   METAGRID_SOCIAL_AUTH_GLOBUS_SECRET: ${{ secrets.GLOBUS_SECRET }}
-  METAGRID_GLOBUS_CLIENT_ID: 21982de0-2b7a-4ba8-bef5-5606ae098201
 
 jobs:
   build:
     runs-on: ubuntu-latest
 
     services:
       postgres:
-        image: postgres:12.6
+        image: postgres:16.10
         env:
           POSTGRES_PASSWORD: postgres
         ports:

.github/workflows/containers.yml

@@ -77,11 +77,24 @@ jobs:
           push: true
           tags: ${{ steps.metadata.outputs.tags }}
           labels: ${{ steps.metadata.outputs.labels }}
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+    - uses: actions/checkout@v4
+    - uses: azure/[email protected]
+    - name: Chart | Lint
+      run: |
+        helm repo add bitnami https://charts.bitnami.com/bitnami
+        helm dep build helm/
+        helm lint helm/
   test:
     name: Test Helm Chart
     runs-on: ubuntu-latest
     needs: [frontend, backend]
     if: github.event_name == 'pull_request'
+    continue-on-error: true
     steps:
     - uses: actions/checkout@v4
     - name: Start minikube
@@ -93,13 +106,57 @@ jobs:
       run: |
         PR_NUMBER=$(echo ${GITHUB_REF} | awk -F'/' '{print $3}')
 
-        helm repo add bitnami https://charts.bitnami.com/bitnami
+        helm install test . \
+          --set frontend.image.tag=pr-${PR_NUMBER} \
+          --set backend.image.tag=pr-${PR_NUMBER} \
+          --wait
+
+        kubectl get pods
+  test-upgrade:
+    name: Test Helm Chart upgrade
+    runs-on: ubuntu-latest
+    needs: [frontend, backend]
+    if: github.event_name == 'pull_request'
+    continue-on-error: true
+    steps:
+    - uses: actions/checkout@v4
+    - name: Start minikube
+      uses: medyagh/setup-minikube@latest
+    - name: Set up Helm
+      uses: azure/[email protected]
+    - name: Install Helm Chart
+      working-directory: helm/
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        helm install test oci://ghcr.io/esgf2-us/metagrid --version `gh release list --json isLatest,tagName -q ".[] | select(.isLatest).tagName"` --timeout 1m || echo
 
-        helm dependency build
+        kubectl get pods
 
-        helm install test . \
+        PR_NUMBER=$(echo ${GITHUB_REF} | awk -F'/' '{print $3}')
+
+        helm upgrade test . \
           --set frontend.image.tag=pr-${PR_NUMBER} \
           --set backend.image.tag=pr-${PR_NUMBER} \
           --wait
 
         kubectl get pods
+  publish:
+    name: Publish
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    if: github.event_name != 'pull_request'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Chart | Push
+      uses: appany/[email protected]
+      with:
+        name: metagrid
+        repository: esgf2-us
+        tag: ${{ github.ref_name }}
+        path: helm
+        registry: ghcr.io
+        registry_username: ${{ github.actor }}
+        registry_password: ${{ secrets.GITHUB_TOKEN }}
+        update_dependencies: 'true'

.github/workflows/pre-commit.yml

@@ -3,7 +3,6 @@ on:
   push:
     branches: [master]
   pull_request:
-    branches: [master]
 
 jobs:
   pre-commit:

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
-        exclude: traefik/*.yml|helm/templates/.*
+        exclude: traefik/*.yml|helm/templates/.*|helm/deploy/.*|helm/values.yaml
 
   # Back-end
   # ------------------------------------------------------------------------------

backend/Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.11-slim
 
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONUNBUFFERED=1
 
 RUN apt-get update \
     # dependencies for building Python packages

backend/config/settings/site_specific.py

@@ -42,6 +42,11 @@ class MetagridBackendSettings(BaseSettings):
         description="A list of all the people who get code error notifications. When `DEBUG=False` and `AdminEmailHandler` is configured in `LOGGING` (done by default), Django emails these people the details of exceptions raised in the request/response cycle. Each item in the list should be a tuple of (Full name, email address). "
         "Reference: <https://docs.djangoproject.com/en/5.1/ref/settings/#admins>",
     )
+    DATABASE_URL: str = Field(
+        default="postgresql://postgres:postgres@postgres:5432/postgres",
+        examples=["postgresql://postgres:postgres@postgres:5432/postgres"],
+        description="The database connection URL for the Metagrid backend database.",
+    )
     SOCIAL_AUTH_GLOBUS_KEY: str = Field(
         examples=["94c44808-9efd-4236-bffd-1185b1071736"],
         description="The `Client UUID` obtained by registering a `portal, science gateway, or other application you host` with Globus at <https://app.globus.org/settings/developers>",
@@ -82,10 +87,6 @@ class MetagridFrontendSettings(BaseSettings):
         default="",
         description="Text to display in the footer of the frontend. Useful for adding a link to the terms of service or other legal information. The string should be formatted as MarkDown and will be rendered as such.",
     )
-    GLOBUS_CLIENT_ID: str = Field(
-        examples=["536321f7-c0e9-462c-b5c6-34d4a3672076"],
-        description="The `Client UUID` obtained by registering a `a thick client or script that will be installed and run by users on their devices` with Globus at <https://app.globus.org/settings/developers>  This is required even if signing in with Globus is not enabled. It is used for browsing Globus Collections to which files may be sent.",
-    )
     STATUS_URL: Optional[str] = Field(
         default=None,
         description="The URL at which the backend can reach the Node Status API.",

backend/config/settings/static.py

@@ -5,16 +5,25 @@
 from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
-ROOT_DIR = (
-    environ.Path(__file__) - 3
-)  # (config/settings/django.py - 3 = metagrid/)
+env = environ.Env()
+
+# project root (config/settings/static.py - 3 = metagrid/)
+ROOT_DIR = environ.Path(__file__) - 3
+
+# Parse DATABASE_URL environment variable; default to an in-memory sqlite DB for tests/local runs.
+# This prevents "Set the DATABASE_URL environment variable" errors when none is provided.
+DATABASES = env.db_url(
+    default="postgresql://postgres:postgres@postgres:5432/postgres"
+)
+DATABASES.update(ATOMIC_REQUESTS=True)
 
 
 class DjangoStaticSettings(BaseSettings):
     """Django settings that do not vary by site"""
 
     model_config = SettingsConfigDict(
         env_prefix="DJANGO_",
+        env_nested_delimiter="__",
         case_sensitive=True,
     )
 
@@ -88,14 +97,9 @@ class DjangoStaticSettings(BaseSettings):
     EMAIL_TIMEOUT: int = 5
 
     CORS_ORIGIN_ALLOW_ALL: bool = True
-    DATABASES: dict[str, dict[str, Any]] = {
-        "default": {
-            "ENGINE": "django.db.backends.postgresql",
-            "NAME": "postgres",
-            "ATOMIC_REQUESTS": True,
-            "DATABASE_URL": "pgsql:///postgres",
-        }
-    }
+
+    DATABASES: dict[str, dict[str, Any]] = {"default": DATABASES}
+
     STATIC_ROOT: str = ROOT_DIR("staticfiles")
     # https://docs.djangoproject.com/en/dev/ref/settings/#static-url
     STATIC_URL: str = "/static/"
@@ -267,7 +271,7 @@ class DjangoStaticSettings(BaseSettings):
     # social_django
     # -------------------------------------------------------------------------------
     # This is a general Django setting if views need to redirect to login
-    # https://docs.djangoproject.com/en/3.2/ref/settings/#login-url
+    # https://docs.djangoproject.com/en/5.0/ref/settings/#login-url
     LOGIN_URL: str = "/login/globus/"
 
     LOGIN_REDIRECT_URL: str = "/search"

backend/config/urls.py

@@ -12,7 +12,10 @@
 from rest_framework import permissions
 from rest_framework.routers import DefaultRouter
 
-from metagrid.api_globus.views import create_globus_transfer, get_access_token
+from metagrid.api_globus.views import (
+    do_globus_reset_tokens,
+    globus_download_request,
+)
 from metagrid.api_proxy.views import (
     do_citation,
     do_globus_auth,
@@ -59,6 +62,11 @@ class KeycloakLogin(SocialLoginView):
     path("", include("social_django.urls", namespace="social")),
     path("proxy/globus-logout/", do_globus_logout, name="globus-logout"),
     path("proxy/globus-auth/", do_globus_auth, name="globus-auth"),
+    path(
+        "proxy/globus-reset-tokens/",
+        do_globus_reset_tokens,
+        name="globus_reset_tokens",
+    ),
     path(
         "proxy/globus-search-endpoints/",
         do_globus_search_endpoints,
@@ -77,8 +85,7 @@ class KeycloakLogin(SocialLoginView):
     ),
     path("tempStorage/get", get_temp_storage, name="temp_storage_get"),
     path("tempStorage/set", set_temp_storage, name="temp_storage_set"),
-    path("globus/auth", get_access_token, name="globus_auth"),
-    path("globus/transfer", create_globus_transfer, name="globus_transfer"),
+    path("globus/transfer", globus_download_request, name="globus_transfer"),
     path("frontend-config.js", get_frontend_config, name="frontend_config"),
     path("liveness", liveness, name="liveness"),
     path("readiness", readiness, name="readiness"),