Fixing all tests!

Signed-off-by: Cédric Foellmi <cedric@onekiloparsec.dev>

Author: onekiloparsec

arcsecond/api/main.py

@@ -57,7 +57,7 @@ def login(self, username, access_key=None, upload_key=None):
 
         endpoint = ArcsecondAPIEndpoint(self.config, API_AUTH_PATH_VERIFY)
         _, error = endpoint.create(
-            {"username": username, "key": access_key or upload_key}
+            json={"username": username, "key": access_key or upload_key}
         )
         if error:
             click.echo(click.style(error, fg="red"))

arcsecond/cloud/uploader/allskycameraimages/uploader.py

@@ -11,7 +11,7 @@ def _prepare_upload(self):
         # Camera validation was already done in the context
         pass
 
-    def _get_upload_data_fields(self, **kwargs):
+    def _get_upload_data(self, **kwargs):
         fields = {
             "camera": self._context.camera_uuid
         }

arcsecond/cloud/uploader/context.py

@@ -5,7 +5,6 @@
 from arcsecond.api.constants import API_AUTH_PATH_VERIFY_PORTAL
 from arcsecond.api.endpoint import ArcsecondAPIEndpoint
 from arcsecond.api.main import ArcsecondAPI
-
 from .errors import (
     InvalidAstronomerError,
     InvalidOrgMembershipError,
@@ -92,7 +91,7 @@ def _validate_remote_organisation(self):
     def _validate_astronomer_role_in_remote_organisation(self):
         endpoint = ArcsecondAPIEndpoint(self.config, API_AUTH_PATH_VERIFY_PORTAL)
         _, error = endpoint.create(
-            {
+            json={
                 "username": self._config.username,
                 "key": self._config.access_key or self._config.upload_key,
                 "organisation": self._subdomain,

arcsecond/cloud/uploader/datafiles/uploader.py

@@ -23,7 +23,7 @@ def _prepare_upload(self):
                 f"{self.log_prefix} Creating dataset with name {dataset_name}..."
             )
             endpoint = ArcsecondAPIEndpoint(self._context.config, 'datasets', self._context.subdomain)
-            dataset, error = endpoint.create(data=data)
+            dataset, error = endpoint.create(json=data)
 
             if error:
                 error_msg = f"Dataset {dataset_name} could not be created: {error}"
@@ -40,8 +40,11 @@ def _get_upload_data(self, **kwargs):
             "dataset": self._context.dataset_uuid,
             "is_raw": str(self._context.is_raw_data),
         }
-        if self._context.custom_tags:
-            fields["tags"] = ",".join(self._context.custom_tags or [])  # will be split back in backend
+        if self._context.custom_tags and len(self._context.custom_tags) > 0:
+            raw_tags = self._context.custom_tags if isinstance(self._context.custom_tags, list) \
+                else self._context.custom_tags.split(',')
+            tags = [t.strip() for t in raw_tags if len(t.strip()) > 0]
+            fields["tags"] = ",".join(tags)
         clean_kwargs = {k: kwargs[k] for k in ('is_raw', 'tags', 'dataset') if k in kwargs}
         if 'tags' in clean_kwargs and not clean_kwargs['tags']:
             # Tags must really be provided only when non-blank/null/empty

tests/api/test_api.py

@@ -1,4 +1,4 @@
-import responses
+import respx
 
 from arcsecond import ArcsecondAPI, ArcsecondConfig
 from arcsecond.api.constants import ARCSECOND_API_URL_DEV
@@ -10,24 +10,24 @@
 )
 
 
-@responses.activate
+@respx.mock
 def test_login_basic():
     random_api_name = random_string()
     config = ArcsecondConfig(api_name=random_api_name)
     config.api_server = ARCSECOND_API_URL_DEV
     assert config.access_key == ""
-    prepare_successful_login(random_api_name)
+    prepare_successful_login(config)
     ArcsecondAPI(config).login(TEST_LOGIN_USERNAME, access_key=TEST_API_KEY)
     assert config.access_key == TEST_API_KEY
     assert config.upload_key == ""
 
 
-@responses.activate
+@respx.mock
 def test_login_upload_key():
     random_api_name = random_string()
     config = ArcsecondConfig(api_name=random_api_name)
     config.api_server = ARCSECOND_API_URL_DEV
-    prepare_successful_login(random_api_name)
+    prepare_successful_login(config)
     ArcsecondAPI(config).login(username=TEST_LOGIN_USERNAME, upload_key=TEST_UPLOAD_KEY)
     assert config.access_key == ""
     assert config.upload_key == TEST_UPLOAD_KEY

tests/cloud/uploader/allskycameraimages/test_uploader_full_process.py

@@ -0,0 +1,76 @@
+import random
+import shutil
+import tempfile
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+import respx
+from httpx import Response
+
+from arcsecond import (
+    AllSkyCameraImageFileUploader,
+    AllSkyCameraImageUploadContext,
+)
+from arcsecond.cloud.uploader.constants import Status, Substatus
+from tests.utils import (
+    prepare_successful_login,
+    prepare_upload_allskyimages,
+    random_string,
+)
+
+
+@pytest.fixture
+def mock_config():
+    """Create a mock ArcsecondConfig."""
+    config = MagicMock()
+    config.username = "test_user"
+    config.upload_key = "test_key"
+    config.api_name = random_string()
+    config.api_server = "http://mock.example.com"
+    config.access_key = None  # very important, because access_key takes precedence on upload_key in headers setting.
+    config.upload_key = '1234567890'
+    return config
+
+
+@respx.mock
+def test_full_upload_process_allskyimages(mock_config):
+    camera_uuid = str(uuid.uuid4())
+    org_subdomain = "test-portal"
+
+    prepare_successful_login(mock_config, org_subdomain)
+    prepare_upload_allskyimages(mock_config, camera_uuid, org_subdomain)
+
+    # file upload
+    image_id = random.randint(1, 1000)
+    respx.post(
+        "/".join([mock_config.api_server, org_subdomain, "allskycameras", camera_uuid, "images"]) + "/"
+    ).mock(Response(201, json={"status": "success", "id": image_id}))
+
+    context = AllSkyCameraImageUploadContext(
+        mock_config,
+        input_camera_uuid=camera_uuid,
+        org_subdomain=org_subdomain
+    )
+
+    context.validate()  # important step to perform before uploading.
+    fixtures_dir = Path(__file__).parent.parent.parent.parent / "fixtures"
+    fixture_files = list(fixtures_dir.glob("*.jpeg"))
+
+    for fixture_file in fixture_files:
+        # Create a temporary directory and copy the fixture file there
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_path = Path(temp_dir) / fixture_file.name
+            shutil.copy(fixture_file, temp_path)
+
+            # Use the actual file for uploading
+            uploader = AllSkyCameraImageFileUploader(
+                context, str(temp_path), display_progress=False
+            )
+
+            status, substatus, error = uploader.upload_file(datetime.now(timezone.utc).timestamp())
+            assert status.value == Status.OK.value
+            assert substatus.value == Substatus.DONE.value
+            assert error is None

tests/cloud/uploader/datafiles/test_uploader_errors.py

@@ -28,7 +28,7 @@ def test_upload_file_skipped(file_uploader):
         mock_perform.assert_called_once()
 
         # Verify result
-        assert result == [Status.OK, Substatus.DONE, None]
+        assert result == [Status.SKIPPED, Substatus.ALREADY_SYNCED, None]
 
 
 def test_upload_file_invalid_context(file_uploader):
@@ -102,6 +102,5 @@ def test_upload_file_retry_on_error(file_uploader):
         # Verify each method was called twice (initial + retry)
         assert prepare_mock.call_count == 2
         assert perform_mock.call_count == 2
-        assert update_mock.call_count == 2
 
 

tests/cloud/uploader/datafiles/test_uploader_full_process.py

@@ -3,62 +3,56 @@
 import tempfile
 import uuid
 from pathlib import Path
+from unittest.mock import MagicMock
 
-import responses
+import pytest
+import respx
+from httpx import Response
 
-from api.constants import ARCSECOND_API_URL_DEV
 from arcsecond import (
-    AllSkyCameraImageFileUploader,
-    AllSkyCameraImageUploadContext,
-    ArcsecondConfig,
     DatasetFileUploader,
     DatasetUploadContext,
 )
 from arcsecond.cloud.uploader.constants import Status, Substatus
-from arcsecond.options import State
 from tests.utils import (
     prepare_successful_login,
-    prepare_upload_allskyimage,
-    prepare_upload_files,
+    prepare_upload_files, random_string,
 )
 
 
-@responses.activate
-def test_full_upload_process_datafiles():
+@pytest.fixture
+def mock_config():
+    """Create a mock ArcsecondConfig."""
+    config = MagicMock()
+    config.username = "test_user"
+    config.upload_key = "test_key"
+    config.api_name = random_string()
+    config.api_server = "http://mock.example.com"
+    config.access_key = None  # very important, because access_key takes precedence on upload_key in headers setting.
+    config.upload_key = '1234567890'
+    return config
+
+
+@respx.mock
+def test_full_upload_process_datafiles(mock_config):
     dataset_uuid = str(uuid.uuid4())
     telescope_uuid = str(uuid.uuid4())
     org_subdomain = "test-portal"
 
-    prepare_successful_login(org_subdomain)
-    prepare_upload_files(dataset_uuid, telescope_uuid, org_subdomain)
+    prepare_successful_login(mock_config, org_subdomain)
+    prepare_upload_files(mock_config, dataset_uuid, telescope_uuid, org_subdomain)
 
-    # file upload
-    datafile_id = random.randint(1, 1000)
-    responses.post(
-        "/".join([ARCSECOND_API_URL_DEV, org_subdomain, "datafiles"]) + "/",
-        status=201,
-        json={"status": "success", "id": datafile_id},
-    )
-    # update metadata
-    responses.patch(
-        "/".join([ARCSECOND_API_URL_DEV, org_subdomain, "datafiles", str(datafile_id)])
-        + "/",
-        status=200,
-        json={"id": datafile_id},
-    )
+    respx.get(
+        "/".join([mock_config.api_server, org_subdomain, "datasets", dataset_uuid]) + "/"
+    ).mock(Response(201, json={"name": "dummy", "uuid": dataset_uuid}))
 
-    state = State(verbose=False, api_name="cloud")
-    config = {
-        "cloud": {
-            "username": "dummy",
-            "upload_key": "1234",
-            "api_server": ARCSECOND_API_URL_DEV,
-        }
-    }
-    config = ArcsecondConfig(state, config)  # it will read your config file.
+    datafile_id = random.randint(1, 1000)
+    respx.post(
+        "/".join([mock_config.api_server, org_subdomain, "datafiles"]) + "/"
+    ).mock(Response(201, json={"status": "success", "id": datafile_id}))
 
     context = DatasetUploadContext(
-        config,
+        mock_config,
         input_dataset_uuid_or_name=dataset_uuid,
         input_telescope_uuid=telescope_uuid,
         is_raw_data=True,
@@ -84,54 +78,3 @@ def test_full_upload_process_datafiles():
             assert status.value == Status.OK.value
             assert substatus.value == Substatus.DONE.value
             assert error is None
-
-
-@responses.activate
-def test_full_upload_process_allskyimages():
-    camera_uuid = str(uuid.uuid4())
-    org_subdomain = "test-portal"
-
-    prepare_successful_login(org_subdomain)
-    prepare_upload_allskyimage(camera_uuid, org_subdomain)
-
-    # file upload
-    image_id = random.randint(1, 1000)
-    responses.post(
-        "/".join([ARCSECOND_API_URL_DEV, org_subdomain, "allskycameras", camera_uuid, "images"]) + "/",
-        status=201,
-        json={"status": "success", "id": image_id},
-    )
-
-    state = State(verbose=False, api_name="cloud")
-    config = {
-        "cloud": {
-            "username": "dummy",
-            "upload_key": "1234",
-            "api_server": ARCSECOND_API_URL_DEV,
-        }
-    }
-    config = ArcsecondConfig(state, config)  # it will read your config file.
-
-    context = AllSkyCameraImageUploadContext(
-        config, input_camera_uuid=camera_uuid, org_subdomain=org_subdomain
-    )
-
-    context.validate()  # important step to perform before uploading.
-    fixtures_dir = Path(__file__).parent.parent.parent.parent / "fixtures"
-    fixture_files = list(fixtures_dir.glob("*.fits"))
-
-    for fixture_file in fixture_files:
-        # Create a temporary directory and copy the fixture file there
-        with tempfile.TemporaryDirectory() as temp_dir:
-            temp_path = Path(temp_dir) / fixture_file.name
-            shutil.copy(fixture_file, temp_path)
-
-            # Use the actual file for uploading
-            uploader = AllSkyCameraImageFileUploader(
-                context, str(temp_path), display_progress=False
-            )
-
-            status, substatus, error = uploader.upload_file()
-            assert status.value == Status.OK.value
-            assert substatus.value == Substatus.DONE.value
-            assert error is None

tests/cloud/uploader/datafiles/test_uploader_prepare.py

@@ -34,7 +34,7 @@ def test_prepare_upload_with_new_dataset(file_uploader):
             "name": file_uploader._context.dataset_name,
             "telescope": file_uploader._context.telescope_uuid,
         }
-        mock_func_create.assert_called_once_with(data=expected_data)
+        mock_func_create.assert_called_once_with(json=expected_data)
         mock_func_update.assert_not_called()