Merge pull request #238 from aodn/ZarrDatasets

Zarr datasets

Author: lbesnard

aodn_cloud_optimised/bin/create_dataset_config.py

@@ -132,6 +132,15 @@ def generate_template_value(schema):
             - dict: An OrderedDict with keys and template values based on the object properties' schema.
             - None: If the schema type is not recognized.
     """
+    # Handle anyOf / oneOf by picking the first subschema
+    if "anyOf" in schema:
+        return generate_template_value(schema["anyOf"][0])
+    if "oneOf" in schema:
+        return generate_template_value(schema["oneOf"][0])
+    if "allOf" in schema:
+        # merge all subschemas (simplest: just pick the first for now)
+        return generate_template_value(schema["allOf"][0])
+
     schema_type = schema["type"]
 
     if schema_type == "string":

aodn_cloud_optimised/bin/satellite_ghrsst_l3s_1day_daytime_multi_sensor_australia.py

@@ -0,0 +1 @@
+generic_launcher.py

aodn_cloud_optimised/bin/satellite_ghrsst_l3s_3day_daytime_multi_sensor_australia.py

@@ -0,0 +1 @@
+generic_launcher.py

aodn_cloud_optimised/bin/satellite_ghrsst_l3s_3day_nighttime_multi_sensor_australia.py

@@ -0,0 +1 @@
+generic_launcher.py

aodn_cloud_optimised/bin/satellite_ghrsst_l3s_6day_daynighttime_multi_sensor_australia.py

@@ -0,0 +1 @@
+generic_launcher.py

aodn_cloud_optimised/bin/satellite_ghrsst_l3s_6day_daytime_multi_sensor_australia.py

@@ -0,0 +1 @@
+generic_launcher.py

aodn_cloud_optimised/bin/satellite_ghrsst_l3s_6day_nighttime_multi_sensor_australia.py

@@ -0,0 +1 @@
+generic_launcher.py

aodn_cloud_optimised/config/dataset/satellite_ghrsst_l3s_1day_daynighttime_multi_sensor_australia.json

@@ -47,19 +47,19 @@
   "run_settings": {
     "coiled_cluster_options": {
       "n_workers": [
-        60,
-        180
+        35,
+        120
       ],
-      "scheduler_vm_types": "m7i.2xlarge",
-      "worker_vm_types": "m7i.2xlarge",
+      "scheduler_vm_types": "m7i-flex.large",
+      "worker_vm_types": "m7i-flex.large",
       "allow_ingress_from": "me",
       "compute_purchase_option": "spot_with_fallback",
       "worker_options": {
-        "nthreads": 8,
-        "memory_limit": "32GB"
+        "nthreads": 2,
+        "memory_limit": "100GB"
       }
     },
-    "batch_size": 100,
+    "batch_size": 60,
     "cluster": {
       "mode": "coiled",
       "restart_every_path": true
@@ -70,7 +70,7 @@
         "filter": [],
         "year_range": [
           2012,
-          2024
+          2025
         ]
       }
     ],

aodn_cloud_optimised/config/dataset/satellite_ghrsst_l3s_1day_daytime_multi_sensor_australia.json

@@ -0,0 +1,83 @@
+{
+  "dataset_name": "satellite_ghrsst_l3s_1day_daytime_multi_sensor_australia",
+  "logger_name": "satellite_ghrsst_l3s_1day_daytime_multi_sensor_australia",
+  "parent_config": "satellite_ghrsst_main.json",
+  "cloud_optimised_format": "zarr",
+  "run_settings": {
+    "paths": [
+      {
+        "type": "files",
+        "s3_uri": "s3://imos-data/IMOS/SRS/SST/ghrsst/L3SM-1d/day/",
+        "filter": [
+          ".*\\.nc$"
+        ],
+        "year_range": [
+          2012,
+          2025
+        ]
+      }
+    ],
+    "cluster": {
+      "mode": "coiled",
+      "restart_every_path": true
+    },
+    "clear_existing_data": true,
+    "raise_error": false,
+    "coiled_cluster_options": {
+      "n_workers": [
+        35,
+        100
+      ],
+      "scheduler_vm_types": "m7i-flex.xlarge",
+      "worker_vm_types": "m7i-flex.xlarge",
+      "allow_ingress_from": "me",
+      "compute_purchase_option": "spot_with_fallback",
+      "worker_options": {
+        "nthreads": 1,
+        "memory_limit": "100GB"
+      }
+    },
+    "batch_size": 60
+  },
+  "metadata_uuid": "52ad55a5-b537-4e10-8a81-22d9317c81f2",
+  "aws_opendata_registry": {
+    "Name": "Satellite - Sea surface temperature - Level 3 - Multi sensor - 1 day - Night time",
+    "Description": "This is a multi-sensor SSTskin L3S product for a single day-time period, derived using sea surface temperature retrievals from the VIIRS sensor on the Suomi-NPP satellite and JPSS series of satellites, and AVHRR sensor on the NOAA and Metop series of Polar-orbiting satellites. The sensors and satellite platforms contributing to each file are listed in the sensor and platform global attributes in the file header. The Multi-sensor L3S product is provided as a 0.02deg x 0.02deg cylindrical equidistant projected map over the region 70E to 170W, 20N to 70S. The quality level for each pixel was remapped using the original ACSPO VIIRS and AVHRR L3U quality levels and Sensor Specific Error Statistics (SSES), as described in Griffin et al. (2017) Appendix at http://imos.org.au/facilities/srs/sstproducts/sstdata0/sstdata-references/, before compositing single swaths from the sensors. Each grid cell contains the 1 day average of all the highest available quality SSTs that overlap with that cell, weighted by the area of overlap. Refer to the IMOS SST products web page at http://imos.org.au/sstproducts.html for further information.",
+    "Documentation": "https://catalogue-imos.aodn.org.au/geonetwork/srv/eng/catalog.search#/metadata/52ad55a5-b537-4e10-8a81-22d9317c81f2",
+    "Contact": "info@aodn.org.au",
+    "ManagedBy": "AODN",
+    "UpdateFrequency": "As Needed",
+    "Tags": [
+      "oceans",
+      "satellite imagery"
+    ],
+    "License": "http://creativecommons.org/licenses/by/4.0/",
+    "Resources": [
+      {
+        "Description": "Cloud Optimised AODN dataset of IMOS - Satellite Remote Sensing - SST - L3S - Multi Sensor - 1 day - day time - Australia",
+        "ARN": "arn:aws:s3:::aodn-cloud-optimised/satellite_ghrsst_l3s_1day_daytime_multi_sensor_australia.zarr",
+        "Region": "ap-southeast-2",
+        "Type": "S3 Bucket"
+      }
+    ],
+    "DataAtWork": {
+      "Tutorials": [
+        {
+          "Title": "Accessing IMOS - Satellite Remote Sensing - SST - L3S - Multi Sensor - 1 day - day time - Australia",
+          "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/satellite_ghrsst_l3s_1day_daytime_multi_sensor_australia.ipynb",
+          "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/satellite_ghrsst_l3s_1day_daytime_multi_sensor_australia.ipynb",
+          "AuthorName": "Laurent Besnard",
+          "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised"
+        },
+        {
+          "Title": "Accessing and search for any AODN dataset",
+          "URL": "https://github.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb",
+          "NotebookURL": "https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/GetAodnData.ipynb",
+          "AuthorName": "Laurent Besnard",
+          "AuthorURL": "https://github.com/aodn/aodn_cloud_optimised"
+        }
+      ]
+    },
+    "Citation": "The citation in a list of references is: \"IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access].\""
+  }
+}