change env names for STAC catalog and items for products

Author: TejasMorbagal

deep_code/tests/tools/test_publish.py

@@ -179,25 +179,25 @@ def test_publish_builds_pr_params(self, mock_wf, mock_ds):
     # S3 credential resolution
     # ------------------------------------------------------------------
 
-    def test_get_stac_s3_storage_options_prefers_xcube_env_vars(self):
+    def test_get_stac_s3_storage_options_prefers_stac_env_vars(self):
         env = {
-            "S3_USER_STORAGE_KEY": "xcube-key",
-            "S3_USER_STORAGE_SECRET": "xcube-secret",
+            "STAC_S3_KEY": "stac-key",
+            "STAC_S3_SECRET": "stac-secret",
             "AWS_ACCESS_KEY_ID": "aws-key",
             "AWS_SECRET_ACCESS_KEY": "aws-secret",
         }
         with patch.dict(os.environ, env):
             opts = self.publisher._get_stac_s3_storage_options()
-        self.assertEqual(opts["key"], "xcube-key")
-        self.assertEqual(opts["secret"], "xcube-secret")
+        self.assertEqual(opts["key"], "stac-key")
+        self.assertEqual(opts["secret"], "stac-secret")
         self.assertEqual(opts["s3_additional_kwargs"], {"ACL": ""})
 
     def test_get_stac_s3_storage_options_falls_back_to_aws_env_vars(self):
         env = {"AWS_ACCESS_KEY_ID": "aws-key", "AWS_SECRET_ACCESS_KEY": "aws-secret"}
         patched_env = {
             k: v
             for k, v in os.environ.items()
-            if k not in ("S3_USER_STORAGE_KEY", "S3_USER_STORAGE_SECRET")
+            if k not in ("STAC_S3_KEY", "STAC_S3_SECRET")
         }
         patched_env.update(env)
         with patch.dict(os.environ, patched_env, clear=True):
@@ -212,8 +212,8 @@ def test_get_stac_s3_storage_options_returns_acl_suppression_for_boto3_chain(sel
             for k, v in os.environ.items()
             if k
             not in (
-                "S3_USER_STORAGE_KEY",
-                "S3_USER_STORAGE_SECRET",
+                "STAC_S3_KEY",
+                "STAC_S3_SECRET",
                 "AWS_ACCESS_KEY_ID",
                 "AWS_SECRET_ACCESS_KEY",
             )

deep_code/tools/new.py

@@ -86,7 +86,7 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str:
             "#   {stac_catalog_s3_root}/{collection_id}/item.json  (STAC Item for the whole Zarr)\n"
             "# and adds a 'child' link from the OSC Collection to this S3 catalog.\n"
             "# S3 write credentials are resolved in order from:\n"
-            "#   1. S3_USER_STORAGE_KEY / S3_USER_STORAGE_SECRET env vars\n"
+            "#   1. STAC_S3_KEY / STAC_S3_SECRET env vars (STAC-specific, any bucket)\n"
             "#   2. AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars\n"
             "#   3. boto3 default chain (IAM role, ~/.aws/credentials)\n"
             "# stac_catalog_s3_root: s3://[YOUR-BUCKET]/stac/[collection-id]/\n"

deep_code/tools/publish.py

@@ -529,8 +529,9 @@ def _get_stac_s3_storage_options(self) -> dict:
 
         Priority (first match wins):
 
-        1. xcube user-storage env vars — ``S3_USER_STORAGE_KEY`` /
-           ``S3_USER_STORAGE_SECRET`` (already used by :func:`open_dataset`).
+        1. STAC-specific env vars — ``STAC_S3_KEY`` / ``STAC_S3_SECRET``.
+           Use these to target any S3 bucket independently of the xcube
+           user-storage bucket.
         2. Standard AWS env vars — ``AWS_ACCESS_KEY_ID`` /
            ``AWS_SECRET_ACCESS_KEY``.
         3. boto3 default credential chain — IAM role attached to the
@@ -539,10 +540,8 @@ def _get_stac_s3_storage_options(self) -> dict:
            to this chain automatically; no secrets are required in code.
         """
 
-        key = os.environ.get("S3_USER_STORAGE_KEY") or os.environ.get(
-            "AWS_ACCESS_KEY_ID"
-        )
-        secret = os.environ.get("S3_USER_STORAGE_SECRET") or os.environ.get(
+        key = os.environ.get("STAC_S3_KEY") or os.environ.get("AWS_ACCESS_KEY_ID")
+        secret = os.environ.get("STAC_S3_SECRET") or os.environ.get(
             "AWS_SECRET_ACCESS_KEY"
         )
         # s3_additional_kwargs={"ACL": ""} suppresses the ACL header that s3fs

docs/configuration.md

@@ -2,18 +2,30 @@
 
 ## Dataset config (YAML)
 ```yaml
+# Required
 dataset_id: your-dataset.zarr
 collection_id: your-collection
-osc_themes: [cryosphere]
-osc_region: global
-dataset_status: completed   # or ongoing/planned
 license_type: CC-BY-4.0
+
+# Optional
+osc_themes: [cryosphere]        # must match slugs at opensciencedata.esa.int/themes/catalog
+osc_region: global
+dataset_status: completed       # ongoing | completed | planned (default: ongoing)
 documentation_link: https://example.com/docs
-access_link: s3://bucket/your-dataset.zarr
+access_link: s3://bucket/your-dataset.zarr   # defaults to s3://deep-esdl-public/{dataset_id}
+
+# CF parameter overrides (list of {name, units, ...} dicts)
+cf_parameter:
+  - name: sea_surface_temperature
+    units: kelvin
 
 # Optional: publish a STAC Catalog + Item next to the data on S3.
 # When set, a lightweight STAC hierarchy (catalog.json → item.json) is written
 # directly to S3 and a "via" link is added to the OSC collection pointing to it.
+# S3 write credentials are resolved in order:
+#   1. STAC_S3_KEY / STAC_S3_SECRET  (STAC-specific, any bucket)
+#   2. AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY
+#   3. boto3 default chain (IAM role, ~/.aws/credentials)
 stac_catalog_s3_root: s3://bucket/stac/your-collection/
 ```
 
@@ -29,6 +41,7 @@ stac_catalog_s3_root: s3://bucket/stac/your-collection/
 | `dataset_status` | No | One of `ongoing`, `completed`, or `planned` (default: `ongoing`). |
 | `access_link` | No | Public S3 URL of the Zarr store. Defaults to `s3://deep-esdl-public/{dataset_id}`. |
 | `documentation_link` | No | URL to dataset documentation. |
+| `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). |
 | `stac_catalog_s3_root` | No | S3 root for the dataset-level STAC Catalog/Item. See [STAC Catalog on S3](#stac-catalog-on-s3). |
 
 ### STAC Catalog on S3
@@ -52,12 +65,14 @@ The OSC collection gains a `via` link to `catalog.json` so STAC-aware clients
 can discover the data path. `rel="child"` is intentionally avoided because the
 OSC validator requires every `child` link to resolve inside the metadata repository.
 
-S3 credentials are resolved in this order: `S3_USER_STORAGE_KEY` /
-`S3_USER_STORAGE_SECRET` env vars, then `AWS_ACCESS_KEY_ID` /
-`AWS_SECRET_ACCESS_KEY`, then the boto3 default chain (IAM role, `~/.aws/credentials`).
+S3 credentials for writing the STAC catalog are resolved in this order:
+`STAC_S3_KEY` / `STAC_S3_SECRET` env vars (STAC-specific, can target any bucket),
+then `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`,
+then the boto3 default chain (IAM role, `~/.aws/credentials`).
 
 ## Workflow config (YAML)
 ```yaml
+# Required
 workflow_id: your-workflow
 properties:
   title: "My workflow"
@@ -69,6 +84,8 @@ properties:
     name: deepesdl-xcube-1.8.3
     python_version: 3.11
     env_file: https://example.com/environment.yml
+
+# Optional
 jupyter_notebook_url: https://github.com/org/repo/path/to/notebook.ipynb
 contact:
   - name: Jane Doe
@@ -77,6 +94,26 @@ contact:
       - rel: about
         type: text/html
         href: https://example.org
+links:
+  - rel: related
+    type: text/html
+    href: https://example.com/related-resource
+    title: Related resource
 ```
 
+### Field reference
+
+| Field | Required | Description |
+|---|---|---|
+| `workflow_id` | Yes | Unique identifier for the workflow (spaces converted to hyphens, lowercased). |
+| `properties.title` | Yes | Human-readable title. |
+| `properties.description` | No | Short summary of what the workflow does. |
+| `properties.keywords` | No | List of keyword strings. |
+| `properties.themes` | No | List of OSC theme slugs. |
+| `properties.license` | No | License identifier (e.g. `proprietary`, `CC-BY-4.0`). |
+| `properties.jupyter_kernel_info` | No | Kernel name, Python version, and environment file URL. |
+| `jupyter_notebook_url` | No | Link to the source notebook on GitHub. When omitted, kernel and application links are skipped. |
+| `contact` | No | List of contact objects with `name`, `organization`, and `links`. |
+| `links` | No | Additional OGC API record links (e.g. `related`, `describedby`). |
+
 More templates and examples live in `dataset_config.yaml`, `workflow_config.yaml`, and `example-config/`.