First version of the cold storage interface. New actions to request files, see the requests and subscribe

Author: psaiz

Dockerfile

@@ -29,6 +29,11 @@ FROM --platform=$BUILDPLATFORM registry.cern.ch/inveniosoftware/almalinux:1
 # Use XRootD 5.8.1
 ENV XROOTD_VERSION=5.8.1
 
+# Install the CERN CA
+COPY docker/carepo.repo /etc/yum.repos.d/
+
+RUN yum install -y ca_CERN-Root-2 && yum clean -y all
+
 # Install CERN Open Data Portal web node pre-requisites
 # hadolint ignore=DL3033
 RUN yum install -y \
@@ -42,8 +47,9 @@ RUN yum install -y \
     yum groupinstall -y "Development Tools" && \
     yum clean -y all
 
+# hadolint ignore=DL3033
 RUN echo "Will install xrootd version: $XROOTD_VERSION (latest if empty)" && \
-    yum install -y xrootd-"$XROOTD_VERSION" python3-xrootd-"$XROOTD_VERSION" && \
+    yum install -y xrootd-"$XROOTD_VERSION" python3-xrootd-"$XROOTD_VERSION" swig python3-gfal2-util gfal2-plugin-http python3-gfal2 && \
     yum clean -y all
 
 RUN pip uninstall pipenv -y && pip install --no-cache-dir --upgrade pip==24.3.1 setuptools==70.0.0 wheel==0.45.1 && \
@@ -72,13 +78,16 @@ ENV PATH=$PATH:${INVENIO_INSTANCE_PATH}/python/bin
 
 # Add CERN Open Data Portal sources to `code` and work there
 WORKDIR ${CODE_DIR}
+COPY . ${CODE_DIR}
+USER root
+RUN chown -R "${INVENIO_USER_ID}":root "${CODE_DIR}"
+USER ${INVENIO_USER_ID}
 
 # Debug off by default
 ARG DEBUG=""
 ENV DEBUG=${DEBUG:-""}
 
 # Install CERN Open Data Portal sources
-COPY . ${CODE_DIR}
 # hadolint ignore=DL3013,SC2086
 RUN git config --global url.https://github.com/.insteadOf git://github.com/ && if [ "$DEBUG" ]; then FLAGS="-e"; fi && \
     pip install --no-cache-dir --user ${FLAGS} ".[all]" && pip check

MANIFEST.in

@@ -45,3 +45,4 @@ recursive-include sentry *.json
 recursive-include sentry *.py
 recursive-include sentry *.sh
 recursive-include sentry *.yml
+recursive-include docker *.repo

cernopendata/api.py

@@ -45,7 +45,7 @@ def __getitem__(self, key):
         """Get a specific file."""
         obj = FileIndexMetadata.get(self.record, key)
         if obj:
-            return self.file_cls(obj, self.filesmap.get(obj.key, {}))
+            return self.file_cls(obj, self.file_indices.get(obj.key, {}))
         raise KeyError(key)
 
     def flush(self):
@@ -96,7 +96,18 @@ def check_availability(self):
         if len(self._avl.keys()) == 1:
             self["availability"] = list(self._avl.keys())[0]
         else:
-            self["availability"] = "sample files"
+            self["availability"] = "partially"
+
+    def flush_indices(self):
+        """Updates the _file_indices information based on what exists on the database."""
+        print("Updating the record with file indices")
+        self["_file_indices"] = []
+        # First, let's get all the file indices that this record has
+        for elem in BucketTag.query.filter_by(value=str(self.id), key="record").all():
+            self["_file_indices"].append(
+                FileIndexMetadata.get(None, str(elem.bucket)).dumps()
+            )
+        self.check_availability()
 
 
 class FileIndexMetadata:
@@ -109,6 +120,7 @@ def __init__(self):
         self._size = 0
         self._files = []
         self._description = ""
+        self._bucket = ""
 
     def __repr__(self):
         """Representation of the object."""
@@ -131,6 +143,7 @@ def create(cls, record, file_object, description=""):
         rb._description = description
         BucketTag.create(rb._bucket, "index_name", index_file_name)
         BucketTag.create(rb._bucket, "record", record.model.id)
+        BucketTag.create(rb._bucket, "description", description)
         print(f"The file index contains {len(index_content)} entries.")
         for entry in index_content:
             entry_file = FileInstance.create()
@@ -162,9 +175,13 @@ def get(cls, record_id, bucket_id):
             .one()
             .value
         )
-        bucket = Bucket.get(bucket_id)
-        for o in ObjectVersion.get_by_bucket(bucket).all():
-            f = FileObject(o, {})
+        tag = BucketTag.query.filter_by(
+            bucket_id=str(bucket_id), key="description"
+        ).first()
+        obj._description = tag.value if tag else obj._index_file_name
+        obj._bucket = Bucket.get(bucket_id)
+        for o in ObjectVersion.get_by_bucket(obj._bucket).all():
+            f = MultiURIFileObject(o, {})
             # Let's put also the uri
             f["uri"] = FileInstance.get(str(o.file_id)).uri
             f["filename"] = f["uri"].split("/")[-1]
@@ -196,6 +213,7 @@ def dumps(self):
             "size": self._size,
             "files": files,
             "description": self._description,
+            "bucket": str(self._bucket),
         }
 
     def flush(self):

cernopendata/cold_storage/__init__.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of CERN Open Data Portal.
+# Copyright (C) 2017-2025 CERN.
+#
+# CERN Open Data Portal is free software; you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# CERN Open Data Portal is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with CERN Open Data Portal; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+"""Cold Storage."""