Made osc_project a configurable parameter

Author: TejasMorbagal

deep_code/tests/tools/test_publish.py

@@ -285,6 +285,80 @@ def test_publish_writes_zarr_stac_to_s3_when_configured(
         # Two S3 files written: catalog.json + item.json
         self.assertEqual(mock_fsspec_open.call_count, 2)
 
+    # ------------------------------------------------------------------
+    # Project collection create-vs-update branching
+    # ------------------------------------------------------------------
+
+    @patch("deep_code.tools.publish.OscDatasetStacGenerator")
+    def test_publish_dataset_creates_project_collection_when_missing(
+        self, MockGenerator
+    ):
+        """When the project collection does not exist, build_project_collection is
+        called and projects/catalog.json is updated via _update_and_add_to_file_dict."""
+        mock_gen = MagicMock()
+        mock_gen.osc_project = "test-project"
+        mock_gen.get_variable_ids.return_value = []
+        mock_gen.build_dataset_stac_collection.return_value.to_dict.return_value = {}
+        mock_gen.build_project_collection.return_value = {
+            "type": "Collection",
+            "id": "test-project",
+        }
+        MockGenerator.return_value = mock_gen
+
+        self.publisher.dataset_config = {
+            "dataset_id": "test-dataset",
+            "collection_id": "test-collection",
+            "license_type": "CC-BY-4.0",
+        }
+        self.publisher.collection_id = "test-collection"
+
+        # Project collection is missing; all other file_exists calls return True
+        self.publisher.gh_publisher.github_automation.file_exists.return_value = False
+
+        with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \
+                patch.object(self.publisher, "_update_variable_catalogs"):
+            file_dict = self.publisher.publish_dataset(write_to_file=False)
+
+        mock_gen.build_project_collection.assert_called_once()
+        self.assertIn("projects/test-project/collection.json", file_dict)
+        mock_gen.update_deepesdl_collection.assert_not_called()
+
+        # projects/catalog.json must be updated
+        updated_paths = [call.args[1] for call in mock_update.call_args_list]
+        self.assertIn("projects/catalog.json", updated_paths)
+
+    @patch("deep_code.tools.publish.OscDatasetStacGenerator")
+    def test_publish_dataset_updates_project_collection_when_exists(
+        self, MockGenerator
+    ):
+        """When the project collection exists, update_deepesdl_collection is called
+        via _update_and_add_to_file_dict and build_project_collection is not called."""
+        mock_gen = MagicMock()
+        mock_gen.osc_project = "test-project"
+        mock_gen.get_variable_ids.return_value = []
+        mock_gen.build_dataset_stac_collection.return_value.to_dict.return_value = {}
+        MockGenerator.return_value = mock_gen
+
+        self.publisher.dataset_config = {
+            "dataset_id": "test-dataset",
+            "collection_id": "test-collection",
+            "license_type": "CC-BY-4.0",
+        }
+        self.publisher.collection_id = "test-collection"
+
+        # Project collection already exists
+        self.publisher.gh_publisher.github_automation.file_exists.return_value = True
+
+        with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \
+                patch.object(self.publisher, "_update_variable_catalogs"):
+            self.publisher.publish_dataset(write_to_file=False)
+
+        mock_gen.build_project_collection.assert_not_called()
+
+        # update_deepesdl_collection passed to _update_and_add_to_file_dict
+        update_methods = [call.args[2] for call in mock_update.call_args_list]
+        self.assertIn(mock_gen.update_deepesdl_collection, update_methods)
+
     @patch.object(Publisher, "publish_dataset", return_value={"github_file.json": {}})
     def test_publish_skips_zarr_stac_when_not_configured(self, mock_publish_ds):
         # No stac_catalog_s3_root in config

deep_code/tests/utils/test_dataset_stac_generator.py

@@ -214,6 +214,145 @@ def test_update_variable_base_catalog(self):
         # self link must remain in place
         self.assertEqual(result["links"][0]["rel"], "self")
 
+    # ------------------------------------------------------------------
+    # osc_project parameter
+    # ------------------------------------------------------------------
+
+    def test_osc_project_default(self):
+        """Default osc_project is 'deep-earth-system-data-lab'."""
+        self.assertEqual(self.generator.osc_project, "deep-earth-system-data-lab")
+
+    @patch("deep_code.utils.dataset_stac_generator.open_dataset")
+    def test_osc_project_custom(self, mock_open_ds):
+        """A custom osc_project is stored on the generator."""
+        mock_open_ds.return_value = self.mock_dataset
+        gen = OscDatasetStacGenerator(
+            dataset_id="mock-dataset-id",
+            collection_id="mock-collection-id",
+            workflow_id="dummy",
+            workflow_title="test",
+            license_type="proprietary",
+            osc_project="my-custom-project",
+        )
+        self.assertEqual(gen.osc_project, "my-custom-project")
+
+    def test_build_dataset_stac_collection_osc_project_in_related_link(self):
+        """The project-related link in the collection uses the configured osc_project."""
+        collection = self.generator.build_dataset_stac_collection(mode="dataset")
+        project_links = [
+            lnk
+            for lnk in collection.links
+            if lnk.rel == "related" and "projects" in str(lnk.target)
+        ]
+        self.assertEqual(len(project_links), 1)
+        self.assertIn("deep-earth-system-data-lab", project_links[0].target)
+
+    # ------------------------------------------------------------------
+    # build_project_collection
+    # ------------------------------------------------------------------
+
+    def test_build_project_collection_structure(self):
+        """build_project_collection returns a minimal valid STAC Collection dict."""
+        result = self.generator.build_project_collection()
+
+        self.assertIsInstance(result, dict)
+        self.assertEqual(result["type"], "Collection")
+        self.assertEqual(result["id"], "deep-earth-system-data-lab")
+        self.assertEqual(result["stac_version"], "1.0.0")
+        self.assertIn("extent", result)
+
+        rels = [lnk["rel"] for lnk in result["links"]]
+        self.assertIn("self", rels)
+        self.assertIn("root", rels)
+        self.assertIn("parent", rels)
+
+        self_link = next(lnk for lnk in result["links"] if lnk["rel"] == "self")
+        self.assertIn("deep-earth-system-data-lab", self_link["href"])
+        self.assertTrue(self_link["href"].endswith("collection.json"))
+
+    @patch("deep_code.utils.dataset_stac_generator.open_dataset")
+    def test_build_project_collection_custom_project(self, mock_open_ds):
+        """build_project_collection reflects a custom osc_project."""
+        mock_open_ds.return_value = self.mock_dataset
+        gen = OscDatasetStacGenerator(
+            dataset_id="mock-dataset-id",
+            collection_id="mock-collection-id",
+            workflow_id="dummy",
+            workflow_title="test",
+            license_type="proprietary",
+            osc_project="my-project",
+        )
+        result = gen.build_project_collection()
+
+        self.assertEqual(result["id"], "my-project")
+        self_link = next(lnk for lnk in result["links"] if lnk["rel"] == "self")
+        self.assertIn("my-project", self_link["href"])
+
+    # ------------------------------------------------------------------
+    # update_project_base_catalog
+    # ------------------------------------------------------------------
+
+    def test_update_project_base_catalog(self):
+        """Child link for the project is appended to the projects base catalog."""
+        import json as _json, os, tempfile
+
+        base = {
+            "type": "Catalog",
+            "id": "projects",
+            "stac_version": "1.0.0",
+            "description": "Projects",
+            "links": [
+                {
+                    "rel": "self",
+                    "href": "https://esa-earthcode.github.io/open-science-catalog-metadata/projects/catalog.json",
+                    "type": "application/json",
+                }
+            ],
+        }
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp:
+            _json.dump(base, tmp)
+            tmp_path = tmp.name
+
+        result = self.generator.update_project_base_catalog(tmp_path)
+        os.unlink(tmp_path)
+
+        self.assertIsInstance(result, dict)
+        child_links = [lnk for lnk in result["links"] if lnk["rel"] == "child"]
+        self.assertEqual(len(child_links), 1)
+        self.assertIn("deep-earth-system-data-lab", child_links[0]["href"])
+        self.assertTrue(child_links[0]["href"].endswith("collection.json"))
+        # existing self link is preserved
+        self.assertEqual(result["links"][0]["rel"], "self")
+
+    def test_update_project_base_catalog_no_duplicate(self):
+        """Calling update_project_base_catalog when the child link already exists
+        does not produce a duplicate."""
+        import json as _json, os, tempfile
+
+        base = {
+            "type": "Catalog",
+            "id": "projects",
+            "stac_version": "1.0.0",
+            "description": "Projects",
+            "links": [
+                {
+                    "rel": "child",
+                    "href": "./deep-earth-system-data-lab/collection.json",
+                    "type": "application/json",
+                    "title": "Deep Earth System Data Lab",
+                }
+            ],
+        }
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp:
+            _json.dump(base, tmp)
+            tmp_path = tmp.name
+
+        result = self.generator.update_project_base_catalog(tmp_path)
+        os.unlink(tmp_path)
+
+        child_links = [lnk for lnk in result["links"] if lnk["rel"] == "child"]
+        self.assertEqual(len(child_links), 1)
+
     def test_update_deepesdl_collection(self):
         """Child and theme-related links are appended; existing links kept."""
         base = {

deep_code/tools/publish.py

@@ -318,11 +318,21 @@ def publish_dataset(
             file_dict, product_catalog_path, generator.update_product_base_catalog
         )
 
-        # Update DeepESDL collection
-        deepesdl_collection_path = "projects/deep-earth-system-data-lab/collection.json"
-        self._update_and_add_to_file_dict(
-            file_dict, deepesdl_collection_path, generator.update_deepesdl_collection
-        )
+        # Update or create project collection
+        project_collection_path = f"projects/{generator.osc_project}/collection.json"
+        if not self.gh_publisher.github_automation.file_exists(project_collection_path):
+            logger.info(
+                f"Project collection for {generator.osc_project} does not exist. Creating..."
+            )
+            file_dict[project_collection_path] = generator.build_project_collection()
+            # Add child link in the projects base catalog
+            self._update_and_add_to_file_dict(
+                file_dict, "projects/catalog.json", generator.update_project_base_catalog
+            )
+        else:
+            self._update_and_add_to_file_dict(
+                file_dict, project_collection_path, generator.update_deepesdl_collection
+            )
 
         # Write to files if testing
         if write_to_file:

deep_code/utils/dataset_stac_generator.py

@@ -11,10 +11,7 @@
 from pystac import Catalog, Collection, Extent, Item, Asset, Link, SpatialExtent, TemporalExtent
 
 from deep_code.constants import (
-    DEEPESDL_COLLECTION_SELF_HREF,
     OSC_THEME_SCHEME,
-    PRODUCT_BASE_CATALOG_SELF_HREF,
-    VARIABLE_BASE_CATALOG_SELF_HREF,
     ZARR_MEDIA_TYPE,
 )
 from deep_code.utils.helper import open_dataset
@@ -35,6 +32,7 @@ class OscDatasetStacGenerator:
         osc_themes: List of themes related to the dataset (e.g., ["climate"]).
         osc_missions: List of satellite missions associated with the dataset.
         cf_params: CF metadata parameters for the dataset.
+        osc_project: OSC project identifier (default: "deep-earth-system-data-lab").
     """
 
     def __init__(
@@ -51,12 +49,14 @@ def __init__(
         osc_themes: list[str] | None = None,
         osc_missions: list[str] | None = None,
         cf_params: list[dict[str]] | None = None,
+        osc_project: str = "deep-earth-system-data-lab",
     ):
         self.dataset_id = dataset_id
         self.collection_id = collection_id
         self.workflow_id = workflow_id
         self.workflow_title = workflow_title
         self.license_type = license_type
+        self.osc_project = osc_project
         self.access_link = access_link or f"s3://deep-esdl-public/{dataset_id}"
         self.documentation_link = documentation_link
         self.osc_status = osc_status
@@ -330,6 +330,70 @@ def add_themes_as_related_links_var_catalog(self, var_catalog):
                 )
             )
 
+    def build_project_collection(self) -> dict:
+        """Build a minimal STAC Collection JSON dict for the OSC project.
+
+        Used when the project collection does not yet exist in the catalog.
+
+        Returns:
+            A plain dict representing the STAC Collection.
+        """
+        now_iso = datetime.now(timezone.utc).isoformat()
+        self_href = (
+            "https://esa-earthcode.github.io/open-science-catalog-metadata"
+            f"/projects/{self.osc_project}/collection.json"
+        )
+        return {
+            "type": "Collection",
+            "id": self.osc_project,
+            "stac_version": "1.0.0",
+            "stac_extensions": [],
+            "title": self.format_string(self.osc_project),
+            "description": self.format_string(self.osc_project),
+            "keywords": [],
+            "license": "various",
+            "extent": {
+                "spatial": {"bbox": [[-180, -90, 180, 90]]},
+                "temporal": {"interval": [[None, None]]},
+            },
+            "created": now_iso,
+            "updated": now_iso,
+            "links": [
+                {
+                    "rel": "self",
+                    "href": self_href,
+                    "type": "application/json",
+                },
+                {
+                    "rel": "root",
+                    "href": "../../catalog.json",
+                    "type": "application/json",
+                    "title": "Open Science Catalog",
+                },
+                {
+                    "rel": "parent",
+                    "href": "../catalog.json",
+                    "type": "application/json",
+                    "title": "Projects",
+                },
+            ],
+        }
+
+    def update_project_base_catalog(self, project_base_catalog_path) -> dict:
+        """Append a child link for the project to the projects base catalog."""
+        with open(project_base_catalog_path, encoding="utf-8") as f:
+            data = json.load(f)
+        self._append_link_if_absent(
+            data.setdefault("links", []),
+            {
+                "rel": "child",
+                "href": f"./{self.osc_project}/collection.json",
+                "type": "application/json",
+                "title": self.format_string(self.osc_project),
+            },
+        )
+        return data
+
     def update_deepesdl_collection(self, deepesdl_collection_full_path) -> dict:
         """Append child and theme-related links to the DeepESDL collection."""
         with open(deepesdl_collection_full_path, encoding="utf-8") as f:
@@ -546,7 +610,7 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N
         # Add OSC extension metadata
         osc_extension = OscExtension.add_to(collection)
         # osc_project and osc_type are fixed constant values
-        osc_extension.osc_project = "deep-earth-system-data-lab"
+        osc_extension.osc_project = self.osc_project
         osc_extension.osc_type = "product"
         osc_extension.osc_status = self.osc_status
         osc_extension.osc_region = self.osc_region
@@ -623,9 +687,9 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N
         collection.add_link(
             Link(
                 rel="related",
-                target="../../projects/deep-earth-system-data-lab/collection.json",
+                target=f"../../projects/{self.osc_project}/collection.json",
                 media_type="application/json",
-                title="Project: DeepESDL",
+                title=f"Project: {self.format_string(self.osc_project)}",
             )
         )