ENH: add data release reprocessing to the release command as another release type (#325)

* ENH: add data release reprocessing to the release command as another release type
* adding sleep between upload and release API call to ensure to not have race cases

Author: tech3371

imap_data_access/cli.py

@@ -734,7 +734,9 @@ def main():
             "  files to release early.\n"
             "- 'unrelease': Unrelease previously released files due to\n"
             "  various causes and reasons. Use --manifest-file to specify\n"
-            "  files to unrelease."
+            "  files to unrelease.\n"
+            "- 'reprocess': Trigger reprocessing for the specified "
+            "--release-number."
         ),
         choices=[e.value for e in ReleaseType],
     )
@@ -743,7 +745,10 @@ def main():
         type=int,
         required=False,
         metavar="NUMBER",
-        help="Release number (required only when --release-type is 'release'). ",
+        help=(
+            "Release number (required only when --release-type is "
+            "'release' or 'reprocess')."
+        ),
     )
     parser_release.add_argument(
         "--exclude-file",

imap_data_access/io.py

@@ -4,6 +4,7 @@
 import logging
 import os
 from pathlib import Path
+from time import sleep
 from typing import Optional, Union
 
 import requests
@@ -565,7 +566,7 @@ def release(
         End date in YYYYMMDD format
     release_number : int, optional
         Release number. Defaults to ``None``. Required if release_type is
-        'release' and should be an integer value.
+        'release' or 'reprocess' and should be an integer value.
     exclude_file : str, optional
         Path to exclude file containing list of files to exclude from public release.
     manifest_file : str, optional
@@ -624,16 +625,26 @@ def release(
             f"'{release_type}' release type."
         )
 
+    if release_type == ReleaseType.REPROCESS.value and release_number is None:
+        raise ValueError(
+            "The 'release_number' parameter is required for 'reprocess' release type."
+        )
     # Handle exclude file upload if provided
     if exclude_file is not None:
         # Upload the exclude file using the standard upload function
         upload(exclude_file)
+        # Sleep few seconds to ensure file is uploaded and indexed
+        # before the release API tries to access it.
+        sleep(10)
         logger.info("Exclude file uploaded successfully")
 
     # Handle manifest file upload if provided
     if manifest_file is not None:
         # Upload the manifest file using the standard upload function
         upload(manifest_file)
+        # Sleep few seconds to ensure file is uploaded and indexed
+        # before the release API tries to access it.
+        sleep(10)
         logger.info("Manifest file uploaded successfully")
 
     # Build release parameters
@@ -644,8 +655,8 @@ def release(
         "end_date": end_date,
     }
 
-    # Add release_number only if release_type is 'release'
-    if release_type == ReleaseType.RELEASE.value:
+    # Add release_number only if release_type is 'release' or 'reprocess'
+    if release_type in {ReleaseType.RELEASE.value, ReleaseType.REPROCESS.value}:
         release_params["release_number"] = release_number
 
     # Add optional parameters if provided

imap_data_access/utils.py

@@ -9,3 +9,4 @@ class ReleaseType(Enum):
     RELEASE = "release"
     EARLY_RELEASE = "early-release"
     UNRELEASE = "unrelease"
+    REPROCESS = "reprocess"

tests/test_io.py

@@ -16,6 +16,12 @@
 test_science_path = "imap/swe/l1/2010/01/" + test_science_filename
 
 
+@pytest.fixture
+def api_key(monkeypatch):
+    """Set a dummy API key so release() doesn't raise on missing key."""
+    monkeypatch.setitem(imap_data_access.config, "API_KEY", "test-api-key")
+
+
 @pytest.mark.parametrize(
     ("url", "api_key", "access_token", "expected"),
     [
@@ -794,3 +800,162 @@ def test_reprocess_bad_instrument(mock_send_request):
         )
     # Should not have made any calls to urlopen
     assert mock_send_request.call_count == 0
+
+
+@pytest.mark.parametrize(
+    "release_params",
+    [
+        {
+            "instrument": "hit",
+            "release_type": "release",
+            "start_date": "20260401",
+            "end_date": "20260430",
+            "release_number": 1,
+        },
+        {
+            "instrument": "mag",
+            "release_type": "release",
+            "start_date": "20260101",
+            "end_date": "20260131",
+            "release_number": 0,
+        },
+    ],
+)
+def test_release(mock_send_request, api_key, release_params):
+    """Test a successful call to the release API with release type.
+
+    Parameters
+    ----------
+    mock_send_request : unittest.mock.MagicMock
+        Mock object for requests.Session
+    api_key : fixture
+        Sets a dummy API key in the config.
+    release_params : dict
+        Parameters passed to release().
+    """
+    mock_response = MagicMock()
+    mock_response.json.return_value = {}
+    mock_send_request.return_value = mock_response
+
+    imap_data_access.release(**release_params)
+
+    mock_send_request.assert_called_once()
+    sent_request = mock_send_request.call_args[0][0]
+    assert "/release" in sent_request.url
+    assert sent_request.method == "GET"
+    for key, value in release_params.items():
+        assert f"{key}={value}" in sent_request.url
+
+
+def test_release_no_api_key(mock_send_request):
+    """Test that release() raises when no API key is set."""
+    with pytest.raises(ValueError, match="API key is required"):
+        imap_data_access.release(
+            instrument="hit",
+            release_type="release",
+            start_date="20260401",
+            end_date="20260430",
+            release_number=1,
+        )
+    assert mock_send_request.call_count == 0
+
+
+def test_release_invalid_release_type(mock_send_request, api_key):
+    """Test that release() raises on an unrecognised release_type."""
+    with pytest.raises(ValueError, match="Not a valid release type"):
+        imap_data_access.release(
+            instrument="hit",
+            release_type="bad-type",
+            start_date="20260401",
+            end_date="20260430",
+        )
+    assert mock_send_request.call_count == 0
+
+
+def test_release_invalid_instrument(mock_send_request, api_key):
+    """Test that release() raises on an invalid instrument for release type."""
+    with pytest.raises(ValueError, match="Not a valid instrument"):
+        imap_data_access.release(
+            instrument="bad_instrument",
+            release_type="release",
+            start_date="20260401",
+            end_date="20260430",
+            release_number=1,
+        )
+    assert mock_send_request.call_count == 0
+
+
+def test_release_missing_release_number(mock_send_request, api_key):
+    """Test that release() raises when release_number is missing for release type."""
+    with pytest.raises(ValueError, match=r"release_number.*required"):
+        imap_data_access.release(
+            instrument="hit",
+            release_type="release",
+            start_date="20260401",
+            end_date="20260430",
+        )
+    assert mock_send_request.call_count == 0
+
+
+def test_release_invalid_start_date(mock_send_request, api_key):
+    """Test that release() raises on a bad start_date."""
+    with pytest.raises(ValueError, match="Not a valid start date"):
+        imap_data_access.release(
+            instrument="hit",
+            release_type="release",
+            start_date="bad-date",
+            end_date="20260430",
+            release_number=1,
+        )
+    assert mock_send_request.call_count == 0
+
+
+def test_release_invalid_end_date(mock_send_request, api_key):
+    """Test that release() raises on a bad end_date."""
+    with pytest.raises(ValueError, match="Not a valid end date"):
+        imap_data_access.release(
+            instrument="hit",
+            release_type="release",
+            start_date="20260401",
+            end_date="bad-date",
+            release_number=1,
+        )
+    assert mock_send_request.call_count == 0
+
+
+@pytest.mark.parametrize("release_type", ["early-release", "unrelease"])
+def test_release_missing_manifest_file(mock_send_request, api_key, release_type):
+    """Test that early-release and unrelease require a manifest_file."""
+    with pytest.raises(ValueError, match=r"manifest_file.*required"):
+        imap_data_access.release(release_type=release_type)
+    assert mock_send_request.call_count == 0
+
+
+def test_release_reprocess_missing_release_number(mock_send_request, api_key):
+    """Test that reprocess release type requires release_number."""
+    with pytest.raises(ValueError, match=r"release_number.*required"):
+        imap_data_access.release(release_type="reprocess")
+    assert mock_send_request.call_count == 0
+
+
+def test_release_early_release_with_manifest(mock_send_request, api_key, tmp_path):
+    """Test early-release type uploads manifest and calls the release endpoint."""
+    manifest = tmp_path / "imap_hit_unrelease_20260101_20260131_v001.txt"
+    manifest.write_text("imap_hit_l2_standard-intensity_20260101_v001.cdf\n")
+
+    # upload() makes two requests: first returns an S3 presigned URL string,
+    # second (PUT to S3) and the final release GET each return an empty dict.
+    s3_response = MagicMock()
+    s3_response.json.return_value = "https://s3-test-bucket.com/manifest"
+    put_response = MagicMock()
+    put_response.json.return_value = {}
+    release_response = MagicMock()
+    release_response.json.return_value = {}
+    mock_send_request.side_effect = [s3_response, put_response, release_response]
+
+    imap_data_access.release(release_type="early-release", manifest_file=manifest)
+
+    assert mock_send_request.call_count == 3
+    last_request = mock_send_request.call_args[0][0]
+    assert "/release" in last_request.url
+    assert "manifest_file=" in last_request.url