re-implement testing resource

Author: thommodin

test_aodn_cloud_optimised/resources/seabird_v1.json

@@ -0,0 +1,275 @@
+{
+  "dataset_name": "seabird_v1",
+  "logger_name": "seabird_v1",
+  "cloud_optimised_format": "parquet",
+  "run_settings": {
+    "paths": [
+      {
+        "s3_uri": "s3://imos-data/seabird",
+        "filter": [
+          "seabird_v1_2025-10-21T02:33:33.parquet"
+        ]
+      }
+    ],
+    "cluster": {
+      "mode": "local",
+      "restart_every_path": false
+    },
+    "clear_existing_data": true,
+    "raise_error": false,
+    "coiled_cluster_options": {
+      "n_workers": [
+        1,
+        20
+      ],
+      "scheduler_vm_types": "m7i-flex.large",
+      "worker_vm_types": "m7i-flex.large",
+      "allow_ingress_from": "me",
+      "compute_purchase_option": "spot_with_fallback",
+      "worker_options": {
+        "nthreads": 4,
+        "memory_limit": "8GB"
+      }
+    },
+    "batch_size": 5,
+    "force_previous_parquet_deletion": false
+  },
+  "metadata_uuid": null,
+  "schema": {
+    "index": {
+      "type": "int64"
+    },
+    "id": {
+      "type": "string"
+    },
+    "modified": {
+      "type": "timestamp[ms]"
+    },
+    "bibliographicCitation": {
+      "type": "string"
+    },
+    "institutionCode": {
+      "type": "string"
+    },
+    "collectionCode": {
+      "type": "string"
+    },
+    "basisOfRecord": {
+      "type": "string"
+    },
+    "occurrenceID": {
+      "type": "string"
+    },
+    "catalogNumber": {
+      "type": "string"
+    },
+    "recordedBy": {
+      "type": "string"
+    },
+    "individualCount": {
+      "type": "int64"
+    },
+    "organismQuantity": {
+      "type": "string"
+    },
+    "organismQuantityType": {
+      "type": "string"
+    },
+    "sex": {
+      "type": "string"
+    },
+    "lifeStage": {
+      "type": "string"
+    },
+    "occurrenceStatus": {
+      "type": "string"
+    },
+    "occurrenceRemarks": {
+      "type": "string"
+    },
+    "organismID": {
+      "type": "string"
+    },
+    "fieldNumber": {
+      "type": "string"
+    },
+    "eventDate": {
+      "type": "date32[day]"
+    },
+    "country": {
+      "type": "string"
+    },
+    "stateProvince": {
+      "type": "string"
+    },
+    "locality": {
+      "type": "string"
+    },
+    "minimumDepthInMeters": {
+      "type": "int64"
+    },
+    "maximumDepthInMeters": {
+      "type": "int64"
+    },
+    "decimalLatitude": {
+      "type": "double"
+    },
+    "decimalLongitude": {
+      "type": "double"
+    },
+    "coordinateUncertaintyInMeters": {
+      "type": "int64"
+    },
+    "coordinatePrecision": {
+      "type": "double"
+    },
+    "footprintWKT": {
+      "type": "string"
+    },
+    "identifiedBy": {
+      "type": "string"
+    },
+    "scientificNameID": {
+      "type": "string"
+    },
+    "scientificName": {
+      "type": "string"
+    },
+    "valid_authority": {
+      "type": "string"
+    },
+    "kingdom": {
+      "type": "string"
+    },
+    "phylum": {
+      "type": "string"
+    },
+    "class": {
+      "type": "string"
+    },
+    "order": {
+      "type": "string"
+    },
+    "family": {
+      "type": "string"
+    },
+    "genus": {
+      "type": "string"
+    },
+    "specificEpithet": {
+      "type": "string"
+    },
+    "taxonRank": {
+      "type": "string"
+    },
+    "scientificNameAuthorship": {
+      "type": "string"
+    },
+    "vernacularName": {
+      "type": "string"
+    },
+    "survey_type": {
+      "type": "string"
+    },
+    "organisation": {
+      "type": "string"
+    },
+    "dataset_id": {
+      "type": "string"
+    },
+    "Wind Direction": {
+      "type": "double"
+    },
+    "Air Temperature": {
+      "type": "double"
+    },
+    "Wind Speed": {
+      "type": "double"
+    },
+    "Depth": {
+      "type": "double"
+    },
+    "Air Pressure": {
+      "type": "double"
+    },
+    "Sea State": {
+      "type": "string"
+    },
+    "Salinity": {
+      "type": "double"
+    },
+    "Cloud Cover": {
+      "type": "string"
+    }
+  },
+  "aws_opendata_registry": {
+    "Name": "",
+    "Description": "",
+    "Documentation": "",
+    "Contact": "",
+    "ManagedBy": "",
+    "UpdateFrequency": "As Needed",
+    "Tags": [],
+    "License": "http://creativecommons.org/licenses/by/4.0/",
+    "Resources": [
+      {
+        "Description": "",
+        "ARN": "arn:aws:s3:::aodn-cloud-optimised/seabird_v1.parquet",
+        "Region": "ap-southeast-2",
+        "Type": "S3 Bucket"
+      }
+    ],
+    "DataAtWork": {
+      "Tutorials": []
+    },
+    "Citation": ""
+  },
+  "schema_transformation": {
+    "drop_variables": [],
+    "add_variables": {
+      "filename": {
+        "source": "@filename",
+        "schema": {
+          "type": "string",
+          "units": "1",
+          "long_name": "Filename of the source file"
+        }
+      },
+      "timestamp": {
+        "source": "@partitioning:time_extent",
+        "schema": {
+          "type": "int64",
+          "units": "1",
+          "long_name": "Partition timestamp"
+        }
+      },
+      "polygon": {
+        "source": "@partitioning:spatial_extent",
+        "schema": {
+          "type": "string",
+          "units": "1",
+          "long_name": "Spatial partition polygon"
+        }
+      }
+    },
+    "partitioning": [
+      {
+        "source_variable": "timestamp",
+        "type": "time_extent",
+        "time_extent": {
+          "time_varname": "eventDate",
+          "partition_period": "Y"
+        }
+      },
+      {
+        "source_variable": "polygon",
+        "type": "spatial_extent",
+        "spatial_extent": {
+          "lat_varname": "decimalLatitude",
+          "lon_varname": "decimalLongitude",
+          "spatial_resolution": 30
+        }
+      }
+    ]
+  }
+}