Draft: Wget simple, bearer token and Globus script API integrations

.github/workflows/backend.yml

@@ -19,16 +19,17 @@ env:
   DATABASE_URL: pgsql://postgres:postgres@localhost:5432/postgres
   REACT_APP_ESGF_NODE_URL: https://esgf-node.llnl.gov/esg-search/search
   REACT_APP_WGET_API_URL: https://greyworm1-rh7.llnl.gov/wget
-  REACT_APP_ESGF_NODE_STATUS_URL: https://aims4.llnl.gov/prometheus/api/v1/query?query=probe_success%7Bjob%3D%22http_2xx%22%2C+target%3D~%22.%2Athredds.%2A%22%7D
-
+  REACT_APP_GLOBUS_SCRIPT_URL: https://greyworm1-rh7.llnl.gov/globusscript
+  REACT_APP_ESGF_NODE_STATUS_URL: https://aims2.llnl.gov/metagrid-backend/proxy/status
+  REACT_APP_ESGF_SOLR_URL: https://esgf-fedtest.llnl.gov/solr
 
 jobs:
   build:
     runs-on: ubuntu-latest
 
     services:
       postgres:
-        image: postgres:11.6
+        image: postgres:12.6
         env:
           POSTGRES_PASSWORD: postgres
         ports:

.gitignore

@@ -5,6 +5,7 @@
 .pytest_cache/
 
 # Config backups
+metagrid_configs/metagrid_config
 metagrid_configs/backups
 
 ### Linux template

.vscode/settings.json

@@ -59,6 +59,7 @@
     "typescript",
     "typescriptreact"
   ],
+  "eslint.workingDirectories": ["./frontend/"],
   // Jest
   // -----------------------------
   "jest.pathToJest": "yarn test:watch",
@@ -76,7 +77,5 @@
   "[jsonc]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
-  "python.analysis.extraPaths": [
-    "backend/venv/bin/python"
-  ]
+  "python.analysis.extraPaths": ["backend/venv/bin/python"]
 }

backend/.envs/.django

@@ -7,20 +7,26 @@ DOMAIN_NAME=http://localhost:8000
 # ------------------------------------------------------------------------------
 CORS_ORIGIN_WHITELIST=http://localhost:3000
 
-# django-all-auth
-# ------------------------------------------------------------------------------
-KEYCLOAK_URL=http://keycloak:8080/auth
-KEYCLOAK_REALM=metagrid
-KEYCLOAK_CLIENT_ID=backend
+
+REACT_APP_GLOBUS_SCRIPT_URL=https://greyworm1-rh7.llnl.gov/globusscript
+
+KEYCLOAK_URL=https://login.esgf.io/
+KEYCLOAK_REALM=esgf
+KEYCLOAK_CLIENT_ID=metagrid-localhost
 
 # ESGF wget API
 # https://github.com/ESGF/esgf-wget
-REACT_APP_WGET_API_URL=https://nimbus3.llnl.gov/wget
+#REACT_APP_WGET_API_URL=https://greyworm1-rh7.llnl.gov/wget
+REACT_APP_WGET_API_URL=https://esgf-node.llnl.gov/esg-search/wget
+
 
 # ESGF Search API
 # https://esgf.github.io/esg-search/ESGF_Search_RESTful_API.html
-REACT_APP_ESGF_NODE_URL=https://esgf-node.llnl.gov/esg-search/search
+REACT_APP_SEARCH_URL=https://esgf-node.llnl.gov/esg-search/search
 
 # ESGF Node Status API
 # https://github.com/ESGF/esgf-utils/blob/master/node_status/query_prom.py
 REACT_APP_ESGF_NODE_STATUS_URL=https://aims2.llnl.gov/metagrid-backend/proxy/status
+
+# ESGF Solr URL
+REACT_APP_ESGF_SOLR_URL=https://esgf-fedtest.llnl.gov/solr

backend/UpdateProjectData_README.txt

@@ -0,0 +1,45 @@
+STEPS TO UPDATE PROJECTS, FACETS OR CATEGORIES
+
+1. Edit the initial data file with the desired changes: metagrid/backend/metagrid/initial_projects_data.py
+2. Change to the backend directory:
+cd metagrid/backend/
+3. Make sure the docker traefik and backend containers are up and running.
+If not running, you can run the containers by going to the traefik directory first and running this command:
+sudo docker compose -f docker-compose.prod.yml up --build -d
+Then do the same in the backend directory (production backend depends on traefik)
+
+RUN UPDATE AND CLEAR TABLES
+If you need to clear tables to remove existing facets/projects or change their order then do the steps below:
+
+Option 1:
+4. Use the updateProjects.sh script. Just run the script using clear option:
+sudo ./updateProject.sh --clear
+
+DONE!
+
+Option 2: Manually update without the script and clear tables:
+4. sudo docker compose -f docker-compose.prod.yml build django # Build the container
+5. sudo docker compose -f docker-compose.prod.yml run --rm django python manage.py migrate projects zero
+6. sudo docker compose -f docker-compose.prod.yml run --rm django python manage.py migrate projects
+
+DONE!
+
+RUN UPDATE WITHOUT CLEARING TABLES
+If your update is small and only involves minor modifications or additions to existing projects then you don't need to clear the tables.
+If there are deletions or more significant changes, run steps above which include clearing tables otherwise do steps below:
+
+Option 1:
+4. Use the updateProjects.sh script. Just run the default script:
+sudo ./updateProject.sh
+
+DONE!
+
+Option 2: Manually update without the script and don't clear tables:
+4. sudo docker compose -f docker-compose.prod.yml build django # Build the container
+5. sudo docker compose -f docker-compose.prod.yml run --rm django python manage.py migrate --fake projects 0001_initial
+6. sudo docker compose -f docker-compose.prod.yml run --rm django python manage.py migrate projects
+
+DONE!
+
+MIGRATION FILE ISSUES:
+If there are issues when attempting the steps above, make sure you don't have obsolete or modified migration files that don't match what's in the latest repository.

backend/config/settings/base.py

@@ -82,6 +82,8 @@
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
 )
 
+SESSION_SAVE_EVERY_REQUEST = True
+
 # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
 ALLOWED_HOSTS = ["localhost", "0.0.0.0", "127.0.0.1"]
 ROOT_URLCONF = "config.urls"
@@ -272,7 +274,7 @@
         "KEYCLOAK_REALM": env(
             "KEYCLOAK_REALM",
         ),
-    }
+    },
 }
 # Used in data migration to register Keycloak social app
 KEYCLOAK_CLIENT_ID = env("KEYCLOAK_CLIENT_ID")
@@ -295,9 +297,11 @@
 # -------------------------------------------------------------------------------
 # https://github.com/adamchainz/django-cors-headers#setup
 CORS_ORIGIN_ALLOW_ALL = False
+CORS_ALLOW_CREDENTIALS = True
 CORS_ORIGIN_WHITELIST = env.list("CORS_ORIGIN_WHITELIST")
 
-
-SEARCH_URL = env("REACT_APP_ESGF_NODE_URL")
+SEARCH_URL = env("REACT_APP_SEARCH_URL")
 WGET_URL = env("REACT_APP_WGET_API_URL")
 STATUS_URL = env("REACT_APP_ESGF_NODE_STATUS_URL")
+SOLR_URL = env("REACT_APP_ESGF_SOLR_URL")
+GLOBUS_SCRIPT_URL = env("REACT_APP_GLOBUS_SCRIPT_URL")

backend/config/urls.py

@@ -12,7 +12,16 @@
 from rest_framework import permissions
 from rest_framework.routers import DefaultRouter
 
-from metagrid.api_proxy.views import do_citation, do_search, do_status, do_wget
+from metagrid.api_globus.views import do_globus_transfer, get_access_token
+from metagrid.api_proxy.views import (
+    do_citation,
+    do_globus_script,
+    do_search,
+    do_status,
+    do_wget,
+    get_temp_storage,
+    set_temp_storage,
+)
 from metagrid.cart.views import CartViewSet, SearchViewSet
 from metagrid.projects.views import ProjectsViewSet
 from metagrid.users.views import UserCreateViewSet, UserViewSet
@@ -50,9 +59,14 @@ class KeycloakLogin(SocialLoginView):
     path("proxy/citation", do_citation, name="do-citation"),
     path("proxy/wget", do_wget, name="do-wget"),
     path("proxy/status", do_status, name="do-status"),
+    path("proxy/globus_script", do_globus_script, name="do-globusscript"),
     path(
         "dj-rest-auth/keycloak", KeycloakLogin.as_view(), name="keycloak_login"
     ),
+    path("tempStorage/get", get_temp_storage, name="temp_storage_get"),
+    path("tempStorage/set", set_temp_storage, name="temp_storage_set"),
+    path("globus/auth", get_access_token, name="globus_auth"),
+    path("globus/transfer", do_globus_transfer, name="globus_transfer"),
     re_path(
         r"^account-confirm-email/",
         VerifyEmailView.as_view(),

backend/docker-compose.yml

@@ -22,22 +22,6 @@ services:
     ports:
       - "5433:5432"
 
-  keycloak:
-    # Source: https://github.com/keycloak/keycloak-containers/blob/master/docker-compose-examples/keycloak-postgres.yml
-    image: jboss/keycloak
-    container_name: keycloak
-    # https://github.com/jhipster/generator-jhipster/issues/7157#issuecomment-367813386
-    depends_on:
-      - postgres
-    volumes:
-      - ./docker/local/keycloak:/opt/jboss/keycloak/imports
-    env_file:
-      - ./.envs/.keycloak
-    ports:
-      - "8080:8080"
-    command: -Dkeycloak.import=/opt/jboss/keycloak/imports/realm-export.json
-      -Dkeycloak.migration.strategy=IGNORE_EXISTING
-
   django:
     build:
       context: .
@@ -46,7 +30,6 @@ services:
     container_name: django
     depends_on:
       - postgres
-      - keycloak
     volumes:
       - .:/app
     env_file:

backend/docker/local/django/Dockerfile

@@ -16,7 +16,7 @@ RUN apt-get update \
 
 # Requirements are installed here to ensure they will be cached.
 COPY ./requirements /requirements
-RUN pip install -r /requirements/local.txt
+RUN pip3 install -r /requirements/local.txt
 
 COPY ./docker/production/django/entrypoint /entrypoint
 RUN sed -i 's/\r$//g' /entrypoint

backend/docker/production/django/Dockerfile

@@ -18,7 +18,7 @@ RUN addgroup --system django \
 
 # Requirements are installed here to ensure they will be cached.
 COPY ./requirements /requirements
-RUN pip install --no-cache-dir -r /requirements/production.txt \
+RUN pip3 install --no-cache-dir -r /requirements/production.txt \
     && rm -rf /requirements
 
 COPY ./docker/production/django/entrypoint /entrypoint

backend/docker/production/postgres/maintenance/backup

@@ -4,7 +4,7 @@
 ### Create a database backup.
 ###
 ### Usage:
-###     $ docker-compose -f <environment>.yml (exec |run --rm) postgres backup
+###     $ docker compose -f <environment>.yml (exec |run --rm) postgres backup
 
 
 set -o errexit

backend/docker/production/postgres/maintenance/restore

@@ -7,7 +7,7 @@
 ###     <1> filename of an existing backup.
 ###
 ### Usage:
-###     $ docker-compose -f <environment>.yml (exec |run --rm) postgres restore <1>
+###     $ docker compose -f <environment>.yml (exec |run --rm) postgres restore <1>
 
 
 set -o errexit

backend/metagrid/api_globus/tests/test_views.py

@@ -0,0 +1,56 @@
+from django.urls import reverse
+from rest_framework import status
+from rest_framework.test import APITestCase
+
+from metagrid.api_globus.views import split_value, truncate_urls
+
+
+class TestGlobusViewSet(APITestCase):
+    def test_truncate(self):
+        lst = [{"url": ["test_url:globus_value|Globus"]}]
+        results = []
+        for value in truncate_urls(lst):
+            results.append(value)
+        assert results == ["globus_value"]
+
+    def test_split_value(self):
+        result = split_value(1)
+        result2 = split_value(1.0)
+        result3 = split_value("noSplitting")
+        result4 = split_value("splitValues,values")
+        result5 = split_value("splitValues,values,CESM1(CAM5.1,FV2)")
+        result6 = split_value(
+            "{splitValues,test},[test,test2],CESM1(CAM5.1,FV2)"
+        )
+
+        assert result == 1
+        assert result2 == 1.0
+        assert result3 == ["noSplitting"]
+        assert result4 == ["splitValues", "values"]
+        assert result5 == ["splitValues", "values", "CESM1(CAM5.1,FV2)"]
+        assert result6 == [
+            "{splitValues,test}",
+            "[test,test2]",
+            "CESM1(CAM5.1,FV2)",
+        ]
+
+    def test_get_access_token(self):
+        url = reverse("globus_auth")
+        getdata = {}
+        response = self.client.post(url, getdata)
+        assert response.status_code == status.HTTP_404_NOT_FOUND
+
+    def test_globus_transfer(self):
+        url = reverse("globus_transfer")
+        getdata = {}
+        response = self.client.get(url, getdata)
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
+
+        postdata = {
+            "access_token": "",
+            "refresh_token": "",
+            "endpointId": "test",
+            "path": "bad/path",
+        }
+        response = self.client.post(url, postdata)
+        assert response.status_code == status.HTTP_400_BAD_REQUEST