First version of the release process, with the pages for the list of releases and dealing with records. It includes user login, communities and a bunch of new dependencies.

    Co-authored-by: Diana Rand <54885108+dianarand@users.noreply.github.com>

Author: psaiz

cernopendata/config.py

@@ -29,6 +29,11 @@
 
 from celery.schedules import crontab, timedelta
 from flask import request
+from invenio_accounts.views import rest as accounts_rest
+from invenio_cern_sync.sso import cern_keycloak, cern_remote_app_name, handlers
+from invenio_cern_sync.sso.api import confirm_registration_form
+from invenio_cern_sync.users.profile import CERNUserProfileSchema
+from invenio_oauthclient.views.client import auto_redirect_login
 from invenio_records_files.api import _Record
 from invenio_records_rest.config import RECORDS_REST_ENDPOINTS
 from invenio_records_rest.facets import nested_filter, range_filter, terms_filter
@@ -43,8 +48,18 @@
 from urllib3.exceptions import InsecureRequestWarning
 
 from cernopendata.cold_storage.tasks import CheckTransfersTask
-from cernopendata.modules.pages.config import *
+
+# noinspection PyUnresolvedReferences
+# from cernopendata.modules.pages.config import *
+# noinspection PyUnresolvedReferences
+from cernopendata.modules.releases import models
+from cernopendata.modules.releases.utils import (
+    user_info_with_cern_roles,
+    user_payload_with_cern_roles,
+)
 from cernopendata.modules.search_ui.helpers import CODSearchAppInvenioRestConfigHelper
+
+# noinspection PyUnresolvedReferences
 from cernopendata.modules.theme.config import *
 from cernopendata.tasks import ProcessEosDumpTask
 
@@ -68,7 +83,23 @@
 
 # Piwik tracking code: set None to disabled it
 THEME_PIWIK_ID = os.environ.get("PIWIK_ID", None)
-ACCOUNTS_SESSION_ACTIVITY_ENABLED = None
+# ACCOUNTS_SESSION_ACTIVITY_ENABLED = True
+# ACCOUNTS_REGISTER = True           # allow registration
+# ACCOUNTS_CONFIRM_EMAIL = None# require email confirmation (recommended)
+# ACCOUNTS_SESSION_RESTORATION = True
+# ACCOUNTS_REST_AUTH_VIEWS= True
+# SECURITY_REGISTERABLE = True
+# SECURITY_RECOVERABLE = True
+# SECURITY_CHANGEABLE = True
+
+# SECURITY_LOGIN_URL = "/login"
+# SECURITY_LOGOUT_URL = "/logout"
+# SECURITY_REGISTER_URL = "/signup"
+
+# Required for sessions
+SECRET_KEY = "change-me-pleaaaseeeee!!!"
+# SECURITY_PASSWORD_SALT = "change-me-too"
+
 SITE_URL = os.environ.get("CERNOPENDATA_SITE_URL", "opendata.cern.ch")
 
 # Logging - Set up Sentry for Invenio-Logging
@@ -163,7 +194,6 @@
     "params": {"preprocessors": [flag_robots, anonymize_user, build_record_unique_id]},
 }
 
-
 STATS_AGGREGATIONS = {
     "file-download-agg": {
         "templates": "invenio_stats.contrib.aggregations.aggr_file_download",
@@ -253,7 +283,6 @@
     },
 }
 
-
 CELERY_BEAT_SCHEDULE = {
     # indexing of statistics events & aggregations
     "stats-process-events": {
@@ -276,11 +305,17 @@
 # JSONSchemas
 JSONSCHEMAS_ENDPOINT = "/schema"
 JSONSCHEMAS_HOST = "opendata.cern.ch"
-JSONSCHEMAS_URL_SCHEME = "http"
+# JSONSCHEMAS_URL_SCHEME = "http"
+JSONSCHEMAS_REPLACE_REFS = True
 
 # HOST_URI
-HOST_URI = "{}://{}".format(JSONSCHEMAS_URL_SCHEME, JSONSCHEMAS_HOST)
-
+# HOST_URI = "{}://{}".format(JSONSCHEMAS_URL_SCHEME, JSONSCHEMAS_HOST)
+#
+# TODO: This is a hack! There is an issue with the resolver. Setting these two things make the test fail
+# If we don't set them, the curation process can't validate the json schema
+if "CERNOPENDATA_CERN_APP_CREDENTIALS" in os.environ:
+    RECORDS_REFRESOLVER_STORE = "invenio_jsonschemas.proxies.current_refresolver_store"
+    RECORDS_REFRESOLVER_CLS = "invenio_records.resolver.InvenioRefResolver"
 # Records
 # Add tuple as array type on record validation
 # http://python-jsonschema.readthedocs.org/en/latest/validate/#validating-types
@@ -501,6 +536,15 @@ def _query_parser_and(qstr=None):
         ),
     }
 }
+COVER_TEMPLATE = "invenio_theme/page_cover.html"
+ACCOUNTS_COVER_TEMPLATE = "invenio_accounts/base_cover.html"
+ACCOUNTS_BASE_TEMPLATE = "invenio_accounts/base.html"
+# After 'Create app'
+# MAIL_SERVER='smtp.example.com'
+# MAIL_PORT = 465
+# MAIL_USE_SSL= True
+# MAIL_USERNAME = 'username'
+COMMUNITIES_IDENTITIES_CACHE_REDIS_URL = "redis://cache:6379/0"
 
 # TODO: based on invenio-records-rest default config
 RECORDS_REST_DEFAULT_SORT = dict(
@@ -634,7 +678,6 @@ def _query_parser_and(qstr=None):
     "invenio_records_rest": CODSearchAppInvenioRestConfigHelper,
 }
 
-
 SEARCH_UI_SEARCH_VIEW = search_legacy
 # OAI-PMH
 # =======
@@ -718,3 +761,52 @@ def _query_parser_and(qstr=None):
 
 # THIS ONE IS ONLY FOR THE DEVELOPMENT
 RATELIMIT_PER_ENDPOINT = {"static": "600 per minute"}
+
+# Checking communities
+THEME_FRONTPAGE = False
+# Enable communities
+COMMUNITIES_ENABLED = True
+
+ACCOUNTS_LOCAL_LOGIN_ENABLED = False
+ACCOUNTS_LOGIN_VIEW_FUNCTION = auto_redirect_login
+
+ACCOUNTS_USER_PROFILE_SCHEMA = CERNUserProfileSchema()
+
+USERPROFILES_EXTEND_SECURITY_FORMS = True
+CERN_SYNC_KEYCLOAK_BASE_URL = "https://auth.cern.ch/"
+
+OAUTHCLIENT_SIGNUP_FORM = confirm_registration_form
+OAUTHCLIENT_CERN_REALM_URL = "https://auth.cern.ch/auth/realms/cern"
+OAUTHCLIENT_CERN_USER_INFO_URL = (
+    "https://auth.cern.ch/auth/realms/cern/protocol/openid-connect/userinfo"
+)
+OAUTHCLIENT_SETTINGS_TEMPLATE = "invenio_oauthclient/settings/base.html"
+OAUTHCLIENT_CERN_VERIFY_EXP = True
+OAUTHCLIENT_CERN_VERIFY_AUD = False
+OAUTHCLIENT_CERN_USER_INFO_FROM_ENDPOINT = True
+OAUTHCLIENT_CERN_OPENID_ALLOWED_ROLES = [
+    "cms-curator",
+    "atlas-curator",
+    "delphi-curator",
+    "alice-curator",
+    "default-role",
+]
+OAUTHCLIENT_AUTO_REDIRECT_TO_EXTERNAL_LOGIN = True
+
+OAUTH_REMOTE_APP_NAME = "cern_openid"
+OAUTHCLIENT_REMOTE_APPS = {
+    cern_remote_app_name: cern_keycloak.remote_app,
+}
+CERN_APP_CREDENTIALS = {
+    "consumer_key": "opendata-dev",
+    "consumer_secret": os.environ.get(
+        "CERNOPENDATA_CERN_APP_CREDENTIALS", "<CHANGEME>"
+    ),
+}
+
+accounts_rest.default_user_payload = user_payload_with_cern_roles
+handlers["signup_handler"]["info"] = user_info_with_cern_roles
+
+# OAUTHCLIENT_SETTINGS_TEMPLATE = 'invenio_theme/page_settings.html'
+COMMUNITIES_CUSTOM_FIELDS = None
+MAX_CONTENT_LENGTH = 1000 * 1024 * 1024

cernopendata/modules/fixtures/cli.py

@@ -147,19 +147,6 @@ def _handle_record_files(record, data, logger=None):
     record.check_availability()
 
 
-def delete_record(pid, logger=None):
-    """Deletes a record."""
-    logger.info("Ready to delete the object {pid}")
-    record = RecordFilesWithIndex.get_record(pid.object_uuid)
-
-    for o in ObjectVersion.get_by_bucket(record.bucket).all():
-        o.remove()
-        FileInstance.query.filter_by(id=o.file_id).delete()
-    FileIndexMetadata.delete_by_record(record=record)
-    record.delete()
-    return None
-
-
 def create_record(data, skip_files, logger=None):
     """Creates a new record."""
     id = uuid.uuid4()

cernopendata/modules/globals/ext.py

@@ -3,10 +3,10 @@
 import json
 import logging
 import os
-
 from counter_robots import is_robot_or_machine
 from flask import Flask, request
 
+from cernopendata.modules.releases.utils import curator_experiments
 from cernopendata.version import __version__
 
 logger = logging.getLogger(__name__)
@@ -51,6 +51,7 @@ def __init__(self, app):
         if not isinstance(app, Flask):
             return
 
+        app.context_processor(curator_experiments)
         self.set_experiments(app)
 
     @staticmethod

cernopendata/modules/releases/__init__.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of CERN Open Data Portal.
+# Copyright (C) 2024 CERN.
+#
+# CERN Open Data Portal is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# CERN Open Data Portal is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with CERN Open Data Portal; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+
+"""CERN Open Data Releases."""