Skip to content

Commit 686fe73

Browse files
khannakarishmakhannakarishma
committed
AMSA Config File
1 parent f035396 commit 686fe73

File tree

6 files changed

+3805
-0
lines changed

6 files changed

+3805
-0
lines changed

‎aodn_cloud_optimised/bin/amsa_vessel_tracking.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
generic_launcher.py

‎aodn_cloud_optimised/config/dataset/amsa_vessel_tracking.json‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
{
2+
"dataset_name": "amsa_vessel_tracking",
3+
"logger_name": "amsa_vessel_tracking",
4+
"cloud_optimised_format": "parquet",
5+
"run_settings": {
6+
"paths": [
7+
{
8+
"type": "parquet",
9+
"partitioning": null,
10+
"s3_uri": "s3://aodn-dataflow-dev/karishma.khanna/AMSA/Optimised/"
11+
}
12+
],
13+
"cluster": {
14+
"mode": "coiled",
15+
"restart_every_path": false
16+
},
17+
"clear_existing_data": true,
18+
"raise_error": false,
19+
"coiled_cluster_options": {
20+
"n_workers": [
21+
1,
22+
20
23+
],
24+
"scheduler_vm_types": "m7i-flex.large",
25+
"worker_vm_types": "m7i-flex.large",
26+
"allow_ingress_from": "me",
27+
"compute_purchase_option": "spot_with_fallback",
28+
"worker_options": {
29+
"nthreads": 4,
30+
"memory_limit": "32GB"
31+
}
32+
},
33+
"batch_size": 1,
34+
"force_previous_parquet_deletion": true
35+
},
36+
"metadata_uuid": null,
37+
"schema": {
38+
"CRAFT_ID": {
39+
"type": "int64",
40+
"nullable": "True",
41+
"long_name": "vessel_identifier"
42+
},
43+
"LAT": {
44+
"type": "double",
45+
"nullable": "True",
46+
"standard_name": "latitude"
47+
},
48+
"LON": {
49+
"type": "double",
50+
"nullable": "True",
51+
"standard_name": "longitude"
52+
},
53+
"TIMESTAMP": {
54+
"type": "timestamp[ms, tz=UTC]",
55+
"nullable": "True",
56+
"long_name": "vessel_position_report_time"
57+
},
58+
"COURSE": {
59+
"type": "double",
60+
"nullable": "True",
61+
"long_name": "vessel_course_over_ground"
62+
},
63+
"SPEED": {
64+
"type": "double",
65+
"nullable": "True",
66+
"long_name": "vessel_speed_over_ground"
67+
},
68+
"TYPE": {
69+
"type": "large_string",
70+
"nullable": "True",
71+
"long_name": "vessel_type"
72+
},
73+
"SUBTYPE": {
74+
"type": "large_string",
75+
"nullable": "True",
76+
"long_name": "vessel_subtype"
77+
},
78+
"LENGTH": {
79+
"type": "int16",
80+
"nullable": "True",
81+
"long_name": "vessel_length"
82+
},
83+
"BEAM": {
84+
"type": "int16",
85+
"nullable": "True",
86+
"long_name": "vessel_beam"
87+
},
88+
"DRAUGHT": {
89+
"type": "double",
90+
"nullable": "True",
91+
"long_name": "vessel_draught"
92+
},
93+
"H3_INDEX": {
94+
"type": "large_string",
95+
"nullable": "True",
96+
"long_name": "h3_spatial_index"
97+
},
98+
"SOURCE_FILE_NAME": {
99+
"type": "large_string",
100+
"nullable": "True",
101+
"long_name": "input_file_name"
102+
},
103+
"PROCESSED_DATE": {
104+
"type": "timestamp[ms, tz=UTC]",
105+
"nullable": "True",
106+
"long_name": "data_processing_date"
107+
}
108+
},
109+
"aws_opendata_registry": {
110+
"Name": "AMSA Vessel Tracking Data",
111+
"Description": "Vessel tracking data from the Australian Maritime Safety Authority (AMSA). This dataset contains Automatic Identification System (AIS) data showing the location, speed, and course of vessels within the Australian Search and Rescue Region. The data has been cloud-optimised (Parquet) for efficient analysis.",
112+
"Documentation": "FILL UP MANUALLY - CHECK DOCUMENTATION",
113+
"Contact": "FILL UP MANUALLY - CHECK DOCUMENTATION",
114+
"ManagedBy": "FILL UP MANUALLY - CHECK DOCUMENTATION",
115+
"UpdateFrequency": "FILL UP MANUALLY - CHECK DOCUMENTATION",
116+
"Tags": [
117+
"FILL UP MANUALLY - CHECK DOCUMENTATION"
118+
],
119+
"License": "FILL UP MANUALLY - CHECK DOCUMENTATION",
120+
"Resources": [
121+
{
122+
"Description": "FILL UP MANUALLY - CHECK DOCUMENTATION",
123+
"ARN": "FILL UP MANUALLY - CHECK DOCUMENTATION",
124+
"Region": "FILL UP MANUALLY - CHECK DOCUMENTATION",
125+
"Type": "FILL UP MANUALLY - CHECK DOCUMENTATION",
126+
"Explore": [
127+
"FILL UP MANUALLY - CHECK DOCUMENTATION"
128+
]
129+
}
130+
],
131+
"DataAtWork": {
132+
"Tutorials": [
133+
{
134+
"Title": "FILL UP MANUALLY - CHECK DOCUMENTATION",
135+
"URL": "FILL UP MANUALLY - CHECK DOCUMENTATION",
136+
"Services": "FILL UP MANUALLY - CHECK DOCUMENTATION",
137+
"AuthorName": "FILL UP MANUALLY - CHECK DOCUMENTATION",
138+
"AuthorURL": "FILL UP MANUALLY - CHECK DOCUMENTATION"
139+
}
140+
],
141+
"Tools & Applications": [
142+
{
143+
"Title": "FILL UP MANUALLY - CHECK DOCUMENTATION",
144+
"URL": "FILL UP MANUALLY - CHECK DOCUMENTATION",
145+
"AuthorName": "FILL UP MANUALLY - CHECK DOCUMENTATION",
146+
"AuthorURL": "FILL UP MANUALLY - CHECK DOCUMENTATION"
147+
}
148+
],
149+
"Publications": [
150+
{
151+
"Title": "FILL UP MANUALLY - CHECK DOCUMENTATION",
152+
"URL": "FILL UP MANUALLY - CHECK DOCUMENTATION",
153+
"AuthorName": "FILL UP MANUALLY - CHECK DOCUMENTATION"
154+
}
155+
]
156+
}
157+
},
158+
"schema_transformation": {
159+
"drop_variables": [],
160+
"add_variables": {
161+
"filename": {
162+
"source": "@filename",
163+
"schema": {
164+
"type": "string",
165+
"units": "1",
166+
"long_name": "Filename of the source file"
167+
}
168+
},
169+
"timestamp": {
170+
"source": "@partitioning:time_extent",
171+
"schema": {
172+
"type": "int64",
173+
"units": "1",
174+
"long_name": "Partition timestamp"
175+
}
176+
},
177+
"polygon": {
178+
"source": "@partitioning:spatial_extent",
179+
"schema": {
180+
"type": "string",
181+
"units": "1",
182+
"long_name": "Spatial partition polygon"
183+
}
184+
}
185+
},
186+
"partitioning": [
187+
{
188+
"source_variable": "timestamp",
189+
"type": "time_extent",
190+
"time_extent": {
191+
"time_varname": "TIMESTAMP",
192+
"partition_period": "M"
193+
}
194+
},
195+
{
196+
"source_variable": "polygon",
197+
"type": "spatial_extent",
198+
"spatial_extent": {
199+
"lat_varname": "LAT",
200+
"lon_varname": "LON",
201+
"spatial_resolution": 10
202+
}
203+
}
204+
],
205+
"global_attributes": {
206+
"delete": [],
207+
"set": {
208+
"title": "AMSA Vessel Tracking Data - Cloud Optimised",
209+
"principal_investigator": "Australian Maritime Safety Authority (AMSA)"
210+
}
211+
}
212+
}
213+
}

‎notebooks/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ More information available [here](https://github.com/conda-forge/miniforge?tab=r
5656

5757
# AODN Notebooks directly loadable into Google Colab
5858

59+
- [amsa_vessel_tracking.ipynb](https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/amsa_vessel_tracking.ipynb)
5960
- [animal_acoustic_tracking_delayed_qc.ipynb](https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_acoustic_tracking_delayed_qc.ipynb)
6061
- [animal_ctd_satellite_relay_tagging_delayed_qc.ipynb](https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/animal_ctd_satellite_relay_tagging_delayed_qc.ipynb)
6162
- [argo.ipynb](https://githubtocolab.com/aodn/aodn_cloud_optimised/blob/main/notebooks/argo.ipynb)

0 commit comments

Comments
 (0)