Merge pull request #8263 from fstagni/90_scitags

[9.0] feat: add scitag support

Author: fstagni

src/DIRAC/DataManagementSystem/Client/FTS3Job.py

@@ -3,9 +3,13 @@
 import datetime
 import errno
 import os
+import requests
 from packaging.version import Version
 
-from cachetools import cachedmethod, LRUCache
+from cachetools import cachedmethod, LRUCache, TTLCache, cached
+from threading import Lock
+from typing import Optional
+
 
 # Requires at least version 3.3.3
 from fts3 import __version__ as fts3_version
@@ -44,6 +48,50 @@
 IDP_CACHE_SIZE = 8
 
 
+_scitag_cache = TTLCache(maxsize=10, ttl=3600)
+_scitag_lock = Lock()
+_scitag_json_cache = TTLCache(maxsize=1, ttl=86400)
+_scitag_json_lock = Lock()
+
+
+@cached(_scitag_cache, lock=_scitag_lock)
+def get_scitag(vo: str, activity: Optional[str] = None) -> int:
+    """
+    Get the scitag based on the VO and activity.
+    If the VO is not found in the scitag.json, it defaults to 1.
+    If no specific activity is provided, it defaults to the "default" activityName.
+
+    :param vo: The VO for which to get the scitag
+    :param activity: The activity for which to get the scitag
+    :return: The scitag value
+    """
+
+    @cached(_scitag_json_cache, lock=_scitag_json_lock)
+    def get_remote_json():
+        gLogger.verbose("Fetching https://scitags.org/api.json from the network")
+        response = requests.get("https://scitags.org/api.json")
+        response.raise_for_status()
+        return response.json()
+
+    vo_id = 1  # Default VO ID
+    activity_id = 1  # Default activity ID
+
+    try:
+        # Load the JSON data from the cache or network
+        sj = get_remote_json()
+
+        for experiment in sj.get("experiments", []):
+            if experiment.get("expName") == vo.lower():
+                vo_id = experiment.get("expId")
+                for act in experiment.get("activities", []):
+                    if act.get("activityName") == activity:
+                        activity_id = act.get("activityId")
+    except Exception as e:
+        gLogger.error(f"Error fetching or parsing scitag.json. Using default scitag values.", repr(e))
+    # Logic to determine the scitag based on vo and activity (this is what FTS wants)
+    return vo_id << 6 | activity_id  # Example logic, replace with actual implementation
+
+
 class FTS3Job(JSerializable):
     """Abstract class to represent a job to be executed by FTS. It belongs
     to an FTS3Operation
@@ -470,6 +518,9 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N
 
                 ftsFileID = getattr(ftsFile, "fileID")
 
+                # scitag 65 is 1 << 6 | 1 (default experiment, default activity)
+                scitag = get_scitag(vo=self.vo, activity=self.activity)
+
                 # Under normal circumstances, we simply submit an fts transfer as such:
                 # * srcProto://myFile -> destProto://myFile
                 #
@@ -499,6 +550,7 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N
                         filesize=ftsFile.size,
                         metadata=stageTrans_metadata,
                         activity=self.activity,
+                        scitag=scitag,
                     )
                     transfers.append(stageTrans)
 
@@ -572,6 +624,7 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N
                     activity=self.activity,
                     source_token=srcToken,
                     destination_token=dstToken,
+                    scitag=scitag,
                 )
 
                 transfers.append(trans)

src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py

@@ -204,6 +204,7 @@ def generateFTS3Job(sourceSE, targetSE, lfns, multiHopSE=None):
     newJob.sourceSE = sourceSE
     newJob.targetSE = targetSE
     newJob.multiHopSE = multiHopSE
+    newJob.vo = "lhcb"
     filesToSubmit = []
 
     for i, lfn in enumerate(lfns, start=1):

src/DIRAC/DataManagementSystem/Client/test/Test_scitag.py

@@ -0,0 +1,69 @@
+from unittest.mock import Mock, patch
+
+import pytest
+
+from DIRAC.DataManagementSystem.Client.FTS3Job import get_scitag
+
+
+class TestGetScitag:
+    def test_valid_vo_and_activity(self):
+        """Test get_scitag with valid VO and activity."""
+        result = get_scitag("atlas", "Analysis Input")
+        expected = 2 << 6 | 17  # atlas expId=2, analysis activityId=17
+        assert result == expected
+
+    def test_valid_vo_no_activity(self):
+        """Test get_scitag with valid VO but no specific activity (should use default)."""
+        result = get_scitag("cms")
+        expected = 3 << 6 | 1  # cms expId=200, default activityId=1
+        assert result == expected
+
+    def test_invalid_vo(self):
+        """Test get_scitag with invalid VO (should use default vo_id=1)."""
+        result = get_scitag("nonexistent")
+        expected = 1 << 6 | 1  # default vo_id=1, default activity_id=1
+        assert result == expected
+
+    def test_valid_vo_invalid_activity(self):
+        """Test get_scitag with valid VO but invalid activity."""
+        result = get_scitag("atlas", "nonexistent_activity")
+        expected = 2 << 6 | 1  # atlas expId=2, default activity_id=1
+        assert result == expected
+
+    def test_case_insensitive_vo(self):
+        """Test that VO matching is case insensitive."""
+        result = get_scitag("ATLAS", "Data Brokering")
+        expected = 2 << 6 | 3  # atlas expId=2, production activityId=3
+        assert result == expected
+
+
+@pytest.mark.parametrize(
+    "vo,activity,expected_vo_id,expected_activity_id",
+    [
+        ("atlas", "Analysis Output", 2, 18),
+        ("atlas", "Debug", 2, 9),
+        ("cms", "Cache", 3, 3),
+        ("cms", "default", 3, 1),
+        ("nonexistent", "any", 1, 1),  # defaults
+        ("atlas", "nonexistent", 2, 1),  # valid vo, invalid activity
+    ],
+)
+def test_parametrized_scenarios(vo, activity, expected_vo_id, expected_activity_id):
+    """Parametrized test for various VO and activity combinations."""
+    result = get_scitag(vo, activity)
+    expected = expected_vo_id << 6 | expected_activity_id
+    assert result == expected
+
+
+@pytest.mark.parametrize(
+    "vo,expected_result",
+    [
+        ("atlas", 2 << 6 | 1),  # Should use default activity
+        ("cms", 3 << 6 | 1),  # Should use default activity
+        ("unknown", 1 << 6 | 1),  # Should use all defaults
+    ],
+)
+def test_no_activity_parameter(vo, expected_result):
+    """Test behavior when no activity parameter is provided."""
+    result = get_scitag(vo)
+    assert result == expected_result