Merge pull request #109 from aodn/ConfigChanges

Radar: update config and notebooks

Author: lbesnard

aodn_cloud_optimised/bin/radar_BonneyCoast_velocity_hourly_averaged_delayed_qc.py

@@ -3,7 +3,7 @@
 
 
 def main():
-    for i, year in enumerate(range(2007, 2025)):
+    for i, year in enumerate(range(2010, 2025)):
         command = [
             "generic_cloud_optimised_creation",
             "--paths",

aodn_cloud_optimised/bin/radar_CapricornBunkerGroup_velocity_hourly_averaged_delayed_qc.py

@@ -3,7 +3,7 @@
 
 
 def main():
-    for i, year in enumerate(range(2007, 2025)):
+    for i, year in enumerate(range(2009, 2025)):
         command = [
             "generic_cloud_optimised_creation",
             "--paths",

aodn_cloud_optimised/bin/radar_CoffsHarbour_velocity_hourly_averaged_delayed_qc.py

@@ -3,7 +3,7 @@
 
 
 def main():
-    for i, year in enumerate(range(2007, 2025)):
+    for i, year in enumerate(range(2012, 2025)):
         command = [
             "generic_cloud_optimised_creation",
             "--paths",

aodn_cloud_optimised/bin/radar_Newcastle_velocity_hourly_averaged_delayed_qc.py

@@ -3,7 +3,7 @@
 
 
 def main():
-    for i, year in enumerate(range(2007, 2025)):
+    for i, year in enumerate(range(2017, 2025)):
         command = [
             "generic_cloud_optimised_creation",
             "--paths",

aodn_cloud_optimised/bin/radar_NorthWestShelf_velocity_hourly_averaged_delayed_qc.py

@@ -3,7 +3,7 @@
 
 
 def main():
-    for i, year in enumerate(range(2007, 2025)):
+    for i, year in enumerate(range(2022, 2025)):
         command = [
             "generic_cloud_optimised_creation",
             "--paths",
@@ -17,8 +17,8 @@ def main():
         ]
 
         # Add --clear-existing-data for the first iteration only
-        if i == 0:
-            command.append("--clear-existing-data")
+        # if i == 0:
+        #    command.append("--clear-existing-data")
 
         # Run the command
         subprocess.run(command, check=True)

aodn_cloud_optimised/bin/radar_TurquoiseCoast_velocity_hourly_averaged_delayed_qc.py

@@ -4,7 +4,7 @@
 
 def main():
     # starts in 2009, no data in 2013, 2014
-    for i, year in enumerate(range(2014, 2025)):
+    for i, year in enumerate(range(2009, 2025)):
         command = [
             "generic_cloud_optimised_creation",
             "--paths",
@@ -18,8 +18,8 @@ def main():
         ]
 
         # Add --clear-existing-data for the first iteration only
-        # if i == 0:
-        #     command.append("--clear-existing-data")
+        if i == 0:
+            command.append("--clear-existing-data")
 
         # Run the command
         subprocess.run(command, check=True)

aodn_cloud_optimised/config/dataset/radar_velocity_hourly_averaged_delayed_qc_main.json

@@ -5,7 +5,7 @@
   "coiled_cluster_options": {
     "n_workers": [
       12,
-      80
+      120
     ],
     "scheduler_vm_types": "m7i.large",
     "worker_vm_types": "m7i.xlarge",
@@ -16,7 +16,7 @@
       "memory_limit": "64GB"
     }
   },
-  "batch_size": 3000,
+  "batch_size": 6000,
   "var_template_shape": "UCUR",
   "vars_to_drop_no_common_dimension": [
     "I",

aodn_cloud_optimised/config/dataset/radar_velocity_hourly_averaged_delayed_qc_no_I_J_version_main.json

@@ -5,7 +5,7 @@
   "coiled_cluster_options": {
     "n_workers": [
       12,
-      80
+      140
     ],
     "scheduler_vm_types": "m7i.large",
     "worker_vm_types": "m7i.xlarge",
@@ -16,7 +16,7 @@
       "memory_limit": "64GB"
     }
   },
-  "batch_size": 3000,
+  "batch_size": 6000,
   "var_template_shape": "UCUR",
   "vars_to_drop_no_common_dimension": [
     "LATITUDE",

aodn_cloud_optimised/config/dataset/radar_wave_delayed_qc_no_I_J_version_main.json

@@ -5,7 +5,7 @@
   "coiled_cluster_options": {
     "n_workers": [
       12,
-      80
+      140
     ],
     "scheduler_vm_types": "m7i.large",
     "worker_vm_types": "m7i.xlarge",

aodn_cloud_optimised/config/dataset/radar_wind_delayed_qc_no_I_J_version_main.json

@@ -5,7 +5,7 @@
   "coiled_cluster_options": {
     "n_workers": [
       12,
-      80
+      140
     ],
     "scheduler_vm_types": "m7i.large",
     "worker_vm_types": "m7i.xlarge",