Fix: add pydantic check on var_template_shape for zarr

Author: lbesnard

aodn_cloud_optimised/bin/generic_cloud_optimised_creation.py

@@ -456,6 +456,9 @@ class ZarrSchemaTransformation(BaseModel):
         default_factory=dict,
         description="Global attributes to modify. Supports 'delete' and 'set' keys.",
     )
+    var_template_shape: Union[str, List[str]] = Field(
+        ..., description="Variable name(s) used to define the template shape."
+    )
     dimensions: Optional[dict[str, dict[str, Any]]] = None
     dataset_sort_by: Optional[list[str]] = None
     vars_incompatible_with_region: Optional[list[str]] = None
@@ -638,6 +641,31 @@ def validate_add_variables(cls, value):
 
         return value
 
+    @model_validator(mode="after")
+    def validate_var_template_shape(self) -> "ZarrSchemaTransformation":
+        # 1. Ensure it's not empty
+        if not self.var_template_shape:
+            raise ValueError("var_template_shape cannot be empty.")
+
+        # 2. Coerce to a list for easier validation logic
+        vars_to_check = (
+            [self.var_template_shape]
+            if isinstance(self.var_template_shape, str)
+            else self.var_template_shape
+        )
+
+        # 3. Check that every variable exists in the dataset_schema
+        schema_keys = self.dataset_schema.keys()
+        missing = [v for v in vars_to_check if v not in schema_keys]
+
+        if missing:
+            raise ValueError(
+                f"var_template_shape variable(s) {missing} not found in 'schema'. "
+                f"Available variables: {list(schema_keys)}"
+            )
+
+        return self
+
     # @model_validator(mode="after")
     # def validate_gattrs_to_variable_dimensions(self) -> "DatasetConfig":
     #     if self.gattrs_to_variables:

aodn_cloud_optimised/config/dataset/satellite_optical_water_type_1day_snpp.json

@@ -40,6 +40,14 @@
     "batch_size": 60
   },
   "metadata_uuid": "8f44f09a-07a3-4f9b-ba26-2fc4983fef69",
+  "schema_transformation": {
+    "global_attributes": {
+      "set": {
+        "title": ""
+      }
+    },
+    "var_template_shape": "owtd_csiro"
+  },
   "schema": {
     "time": {
       "type": "timestamp[ns]",

aodn_cloud_optimised/config/dataset/satellite_sst_1day_aqua.json

@@ -9,7 +9,7 @@
         "type": "files",
         "s3_uri": "s3://imos-data/IMOS/SRS/OC/gridded/aqua/P1D/",
         "filter": [
-          ".*sst\\.nc$"
+          ".*\\.aust\\.sst\\.nc"
         ],
         "year_range": [
           2002,
@@ -40,6 +40,14 @@
     "batch_size": 60
   },
   "metadata_uuid": "468f550b-bbd8-42b1-91b5-313d5898d9e6",
+  "schema_transformation": {
+    "global_attributes": {
+      "set": {
+        "title": ""
+      }
+    },
+    "var_template_shape": "sst"
+  },
   "schema": {
     "time": {
       "type": "timestamp[ns]",

aodn_cloud_optimised/config/dataset/satellite_sst_1day_snpp.json

@@ -9,7 +9,7 @@
         "type": "files",
         "s3_uri": "s3://imos-data/IMOS/SRS/OC/gridded/snpp/P1D/",
         "filter": [
-          ".*sst\\.nc$"
+          ".*\\.aust\\.sst\\.nc"
         ],
         "year_range": [
           2022,
@@ -40,6 +40,14 @@
     "batch_size": 60
   },
   "metadata_uuid": "8e00907f-716d-4080-b212-534fcd78a602",
+  "schema_transformation": {
+    "global_attributes": {
+      "set": {
+        "title": ""
+      }
+    },
+    "var_template_shape": "sst"
+  },
   "schema": {
     "time": {
       "type": "timestamp[ns]",